| message | InvalidEnvironment:
[...]InvalidEnvironment:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/code.py", line 271, in main
robot_ci.write_topic_and_expect_zero("seed", config.seed)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 143, in write_topic_and_expect_zero
msgs = read_reply(self.fpout, timeout=timeout, nickname=self.nickname)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 309, in read_reply
raise RemoteNodeAborted(msg)
zuper_nodes.structures.RemoteNodeAborted: The remote node "ego0" aborted with the following error:
error in ego0 |Unexpected error:
|
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1365, in _do_call
|| return fn(*args)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1349, in _run_fn
|| return self._call_tf_sessionrun(options, feed_dict, fetch_list,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1441, in _call_tf_sessionrun
|| return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
|| tensorflow.python.framework.errors_impl.ResourceExhaustedError: OOM when allocating tensor with shape[11,11,32,256] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc
|| [[{{node default_policy/conv_value_3/kernel/Initializer/random_uniform/RandomUniform}}]]
|| Hint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.
||
||
|| During handling of the above exception, another exception occurred:
||
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| File "/submission/model.py", line 55, in __init__
|| self.model = PPOTrainer(config=config["rllib_config"])
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| Trainer.__init__(self, config, env, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| super().__init__(config, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| self._setup(copy.deepcopy(self.config))
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| self._init(self.config, self.env_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| self.workers = self._make_workers(env_creator, self._policy,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| return WorkerSet(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| self._local_worker = self._make_worker(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| worker = cls(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| self._build_policy_map(policy_dict, policy_config)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| policy_map[name] = cls(obs_space, act_space, merged_conf)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| DynamicTFPolicy.__init__(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| self._initialize_loss()
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 381, in _initialize_loss
|| self._sess.run(tf.global_variables_initializer())
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 957, in run
|| result = self._run(None, fetches, feed_dict, options_ptr,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1180, in _run
|| results = self._do_run(handle, final_targets, final_fetches,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1358, in _do_run
|| return self._do_call(_run_fn, feeds, fetches, targets, options,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1384, in _do_call
|| raise type(e)(node_def, op, message)
|| tensorflow.python.framework.errors_impl.ResourceExhaustedError: OOM when allocating tensor with shape[11,11,32,256] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc
|| [[node default_policy/conv_value_3/kernel/Initializer/random_uniform/RandomUniform (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:83) ]]
|| Hint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.
||
||
|| Original stack trace for 'default_policy/conv_value_3/kernel/Initializer/random_uniform/RandomUniform':
|| File "solution.py", line 127, in <module>
|| main()
|| File "solution.py", line 123, in main
|| wrap_direct(node=node, protocol=protocol)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/interface.py", line 24, in wrap_direct
|| run_loop(node, protocol, args)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 243, in run_loop
|| loop(node_name, fi, fo, node, protocol, tin, tout, config=config, fi_desc=fin, fo_desc=fout)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| File "/submission/model.py", line 55, in __init__
|| self.model = PPOTrainer(config=config["rllib_config"])
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| Trainer.__init__(self, config, env, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| super().__init__(config, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| self._setup(copy.deepcopy(self.config))
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| self._init(self.config, self.env_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| self.workers = self._make_workers(env_creator, self._policy,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| return WorkerSet(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| self._local_worker = self._make_worker(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| worker = cls(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| self._build_policy_map(policy_dict, policy_config)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| policy_map[name] = cls(obs_space, act_space, merged_conf)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| DynamicTFPolicy.__init__(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 147, in __init__
|| self.model = ModelCatalog.get_model_v2(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/catalog.py", line 347, in get_model_v2
|| return wrapper(obs_space, action_space, num_outputs, model_config,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py", line 83, in __init__
|| last_layer = tf.keras.layers.Conv2D(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 757, in __call__
|| self._maybe_build(inputs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 2098, in _maybe_build
|| self.build(input_shapes)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 197, in build
|| self.kernel = self.add_weight(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 431, in add_weight
|| variable = self._add_variable_with_custom_getter(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/training/tracking/base.py", line 745, in _add_variable_with_custom_getter
|| new_variable = getter(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_utils.py", line 133, in make_variable
|| return tf_variables.VariableV1(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/variables.py", line 260, in __call__
|| return cls._variable_v1_call(*args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/variables.py", line 206, in _variable_v1_call
|| return previous_getter(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/variables.py", line 199, in <lambda>
|| previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/variable_scope.py", line 2583, in default_variable_creator
|| return resource_variable_ops.ResourceVariable(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/variables.py", line 264, in __call__
|| return super(VariableMetaclass, cls).__call__(*args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/resource_variable_ops.py", line 1507, in __init__
|| self._init_from_args(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/resource_variable_ops.py", line 1651, in _init_from_args
|| initial_value() if init_from_fn else initial_value,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/init_ops.py", line 518, in __call__
|| return random_ops.random_uniform(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/dispatch.py", line 201, in wrapper
|| return target(*args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/random_ops.py", line 301, in random_uniform
|| result = gen_random_ops.random_uniform(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_random_ops.py", line 742, in random_uniform
|| _, _, _op, _outputs = _op_def_library._apply_op_helper(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
|| op = g._create_op_internal(op_type_name, inputs, dtypes=None,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 3477, in _create_op_internal
|| ret = Operation(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
|| self._traceback = tf_stack.extract_stack()
||
||
|| The above exception was the direct cause of the following exception:
||
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 339, in loop
|| raise Exception(msg) from e
|| Exception: Exception while calling the node's init() function.
||
|| | Traceback (most recent call last):
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1365, in _do_call
|| | return fn(*args)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1349, in _run_fn
|| | return self._call_tf_sessionrun(options, feed_dict, fetch_list,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1441, in _call_tf_sessionrun
|| | return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
|| | tensorflow.python.framework.errors_impl.ResourceExhaustedError: OOM when allocating tensor with shape[11,11,32,256] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc
|| | [[{{node default_policy/conv_value_3/kernel/Initializer/random_uniform/RandomUniform}}]]
|| | Hint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.
|| |
|| |
|| | During handling of the above exception, another exception occurred:
|| |
|| | Traceback (most recent call last):
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| | call_if_fun_exists(node, "init", context=context_data)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| | f(**kwargs)
|| | File "solution.py", line 29, in init
|| | self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| | File "/submission/model.py", line 55, in __init__
|| | self.model = PPOTrainer(config=config["rllib_config"])
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| | Trainer.__init__(self, config, env, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| | super().__init__(config, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| | self._setup(copy.deepcopy(self.config))
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| | self._init(self.config, self.env_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| | self.workers = self._make_workers(env_creator, self._policy,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| | return WorkerSet(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| | self._local_worker = self._make_worker(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| | worker = cls(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| | self._build_policy_map(policy_dict, policy_config)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| | policy_map[name] = cls(obs_space, act_space, merged_conf)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| | DynamicTFPolicy.__init__(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| | self._initialize_loss()
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 381, in _initialize_loss
|| | self._sess.run(tf.global_variables_initializer())
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 957, in run
|| | result = self._run(None, fetches, feed_dict, options_ptr,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1180, in _run
|| | results = self._do_run(handle, final_targets, final_fetches,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1358, in _do_run
|| | return self._do_call(_run_fn, feeds, fetches, targets, options,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1384, in _do_call
|| | raise type(e)(node_def, op, message)
|| | tensorflow.python.framework.errors_impl.ResourceExhaustedError: OOM when allocating tensor with shape[11,11,32,256] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc
|| | [[node default_policy/conv_value_3/kernel/Initializer/random_uniform/RandomUniform (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:83) ]]
|| | Hint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.
|| |
|| |
|| | Original stack trace for 'default_policy/conv_value_3/kernel/Initializer/random_uniform/RandomUniform':
|| | File "solution.py", line 127, in <module>
|| | main()
|| | File "solution.py", line 123, in main
|| | wrap_direct(node=node, protocol=protocol)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/interface.py", line 24, in wrap_direct
|| | run_loop(node, protocol, args)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 243, in run_loop
|| | loop(node_name, fi, fo, node, protocol, tin, tout, config=config, fi_desc=fin, fo_desc=fout)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| | call_if_fun_exists(node, "init", context=context_data)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| | f(**kwargs)
|| | File "solution.py", line 29, in init
|| | self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| | File "/submission/model.py", line 55, in __init__
|| | self.model = PPOTrainer(config=config["rllib_config"])
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| | Trainer.__init__(self, config, env, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| | super().__init__(config, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| | self._setup(copy.deepcopy(self.config))
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| | self._init(self.config, self.env_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| | self.workers = self._make_workers(env_creator, self._policy,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| | return WorkerSet(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| | self._local_worker = self._make_worker(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| | worker = cls(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| | self._build_policy_map(policy_dict, policy_config)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| | policy_map[name] = cls(obs_space, act_space, merged_conf)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| | DynamicTFPolicy.__init__(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 147, in __init__
|| | self.model = ModelCatalog.get_model_v2(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/catalog.py", line 347, in get_model_v2
|| | return wrapper(obs_space, action_space, num_outputs, model_config,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py", line 83, in __init__
|| | last_layer = tf.keras.layers.Conv2D(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 757, in __call__
|| | self._maybe_build(inputs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 2098, in _maybe_build
|| | self.build(input_shapes)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 197, in build
|| | self.kernel = self.add_weight(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 431, in add_weight
|| | variable = self._add_variable_with_custom_getter(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/training/tracking/base.py", line 745, in _add_variable_with_custom_getter
|| | new_variable = getter(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_utils.py", line 133, in make_variable
|| | return tf_variables.VariableV1(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/variables.py", line 260, in __call__
|| | return cls._variable_v1_call(*args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/variables.py", line 206, in _variable_v1_call
|| | return previous_getter(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/variables.py", line 199, in <lambda>
|| | previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/variable_scope.py", line 2583, in default_variable_creator
|| | return resource_variable_ops.ResourceVariable(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/variables.py", line 264, in __call__
|| | return super(VariableMetaclass, cls).__call__(*args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/resource_variable_ops.py", line 1507, in __init__
|| | self._init_from_args(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/resource_variable_ops.py", line 1651, in _init_from_args
|| | initial_value() if init_from_fn else initial_value,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/init_ops.py", line 518, in __call__
|| | return random_ops.random_uniform(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/dispatch.py", line 201, in wrapper
|| | return target(*args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/random_ops.py", line 301, in random_uniform
|| | result = gen_random_ops.random_uniform(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_random_ops.py", line 742, in random_uniform
|| | _, _, _op, _outputs = _op_def_library._apply_op_helper(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
|| | op = g._create_op_internal(op_type_name, inputs, dtypes=None,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 3477, in _create_op_internal
|| | ret = Operation(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
|| | self._traceback = tf_stack.extract_stack()
|| |
|| |
||
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/duckietown_challenges/cie_concrete.py", line 681, in scoring_context
yield cie
File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/experiment_manager.py", line 60, in go
wrap(cie)
File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/experiment_manager.py", line 33, in wrap
asyncio.run(main(cie, logdir, attempts), debug=True)
File "/usr/local/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/code.py", line 277, in main
raise InvalidEnvironment(msg) from e
duckietown_challenges.exceptions.InvalidEnvironment: Detected out of CUDA memory:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/code.py", line 271, in main
robot_ci.write_topic_and_expect_zero("seed", config.seed)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 143, in write_topic_and_expect_zero
msgs = read_reply(self.fpout, timeout=timeout, nickname=self.nickname)
File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 309, in read_reply
raise RemoteNodeAborted(msg)
zuper_nodes.structures.RemoteNodeAborted: The remote node "ego0" aborted with the following error:
error in ego0 |Unexpected error:
|
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1365, in _do_call
|| return fn(*args)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1349, in _run_fn
|| return self._call_tf_sessionrun(options, feed_dict, fetch_list,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1441, in _call_tf_sessionrun
|| return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
|| tensorflow.python.framework.errors_impl.ResourceExhaustedError: OOM when allocating tensor with shape[11,11,32,256] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc
|| [[{{node default_policy/conv_value_3/kernel/Initializer/random_uniform/RandomUniform}}]]
|| Hint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.
||
||
|| During handling of the above exception, another exception occurred:
||
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| File "/submission/model.py", line 55, in __init__
|| self.model = PPOTrainer(config=config["rllib_config"])
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| Trainer.__init__(self, config, env, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| super().__init__(config, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| self._setup(copy.deepcopy(self.config))
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| self._init(self.config, self.env_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| self.workers = self._make_workers(env_creator, self._policy,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| return WorkerSet(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| self._local_worker = self._make_worker(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| worker = cls(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| self._build_policy_map(policy_dict, policy_config)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| policy_map[name] = cls(obs_space, act_space, merged_conf)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| DynamicTFPolicy.__init__(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| self._initialize_loss()
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 381, in _initialize_loss
|| self._sess.run(tf.global_variables_initializer())
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 957, in run
|| result = self._run(None, fetches, feed_dict, options_ptr,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1180, in _run
|| results = self._do_run(handle, final_targets, final_fetches,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1358, in _do_run
|| return self._do_call(_run_fn, feeds, fetches, targets, options,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1384, in _do_call
|| raise type(e)(node_def, op, message)
|| tensorflow.python.framework.errors_impl.ResourceExhaustedError: OOM when allocating tensor with shape[11,11,32,256] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc
|| [[node default_policy/conv_value_3/kernel/Initializer/random_uniform/RandomUniform (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:83) ]]
|| Hint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.
||
||
|| Original stack trace for 'default_policy/conv_value_3/kernel/Initializer/random_uniform/RandomUniform':
|| File "solution.py", line 127, in <module>
|| main()
|| File "solution.py", line 123, in main
|| wrap_direct(node=node, protocol=protocol)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/interface.py", line 24, in wrap_direct
|| run_loop(node, protocol, args)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 243, in run_loop
|| loop(node_name, fi, fo, node, protocol, tin, tout, config=config, fi_desc=fin, fo_desc=fout)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| call_if_fun_exists(node, "init", context=context_data)
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| f(**kwargs)
|| File "solution.py", line 29, in init
|| self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| File "/submission/model.py", line 55, in __init__
|| self.model = PPOTrainer(config=config["rllib_config"])
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| Trainer.__init__(self, config, env, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| super().__init__(config, logger_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| self._setup(copy.deepcopy(self.config))
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| self._init(self.config, self.env_creator)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| self.workers = self._make_workers(env_creator, self._policy,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| return WorkerSet(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| self._local_worker = self._make_worker(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| worker = cls(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| self._build_policy_map(policy_dict, policy_config)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| policy_map[name] = cls(obs_space, act_space, merged_conf)
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| DynamicTFPolicy.__init__(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 147, in __init__
|| self.model = ModelCatalog.get_model_v2(
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/catalog.py", line 347, in get_model_v2
|| return wrapper(obs_space, action_space, num_outputs, model_config,
|| File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py", line 83, in __init__
|| last_layer = tf.keras.layers.Conv2D(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 757, in __call__
|| self._maybe_build(inputs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 2098, in _maybe_build
|| self.build(input_shapes)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 197, in build
|| self.kernel = self.add_weight(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 431, in add_weight
|| variable = self._add_variable_with_custom_getter(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/training/tracking/base.py", line 745, in _add_variable_with_custom_getter
|| new_variable = getter(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_utils.py", line 133, in make_variable
|| return tf_variables.VariableV1(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/variables.py", line 260, in __call__
|| return cls._variable_v1_call(*args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/variables.py", line 206, in _variable_v1_call
|| return previous_getter(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/variables.py", line 199, in <lambda>
|| previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/variable_scope.py", line 2583, in default_variable_creator
|| return resource_variable_ops.ResourceVariable(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/variables.py", line 264, in __call__
|| return super(VariableMetaclass, cls).__call__(*args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/resource_variable_ops.py", line 1507, in __init__
|| self._init_from_args(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/resource_variable_ops.py", line 1651, in _init_from_args
|| initial_value() if init_from_fn else initial_value,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/init_ops.py", line 518, in __call__
|| return random_ops.random_uniform(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/dispatch.py", line 201, in wrapper
|| return target(*args, **kwargs)
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/random_ops.py", line 301, in random_uniform
|| result = gen_random_ops.random_uniform(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_random_ops.py", line 742, in random_uniform
|| _, _, _op, _outputs = _op_def_library._apply_op_helper(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
|| op = g._create_op_internal(op_type_name, inputs, dtypes=None,
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 3477, in _create_op_internal
|| ret = Operation(
|| File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
|| self._traceback = tf_stack.extract_stack()
||
||
|| The above exception was the direct cause of the following exception:
||
|| Traceback (most recent call last):
|| File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 339, in loop
|| raise Exception(msg) from e
|| Exception: Exception while calling the node's init() function.
||
|| | Traceback (most recent call last):
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1365, in _do_call
|| | return fn(*args)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1349, in _run_fn
|| | return self._call_tf_sessionrun(options, feed_dict, fetch_list,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1441, in _call_tf_sessionrun
|| | return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
|| | tensorflow.python.framework.errors_impl.ResourceExhaustedError: OOM when allocating tensor with shape[11,11,32,256] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc
|| | [[{{node default_policy/conv_value_3/kernel/Initializer/random_uniform/RandomUniform}}]]
|| | Hint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.
|| |
|| |
|| | During handling of the above exception, another exception occurred:
|| |
|| | Traceback (most recent call last):
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| | call_if_fun_exists(node, "init", context=context_data)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| | f(**kwargs)
|| | File "solution.py", line 29, in init
|| | self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| | File "/submission/model.py", line 55, in __init__
|| | self.model = PPOTrainer(config=config["rllib_config"])
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| | Trainer.__init__(self, config, env, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| | super().__init__(config, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| | self._setup(copy.deepcopy(self.config))
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| | self._init(self.config, self.env_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| | self.workers = self._make_workers(env_creator, self._policy,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| | return WorkerSet(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| | self._local_worker = self._make_worker(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| | worker = cls(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| | self._build_policy_map(policy_dict, policy_config)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| | policy_map[name] = cls(obs_space, act_space, merged_conf)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| | DynamicTFPolicy.__init__(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 237, in __init__
|| | self._initialize_loss()
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 381, in _initialize_loss
|| | self._sess.run(tf.global_variables_initializer())
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 957, in run
|| | result = self._run(None, fetches, feed_dict, options_ptr,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1180, in _run
|| | results = self._do_run(handle, final_targets, final_fetches,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1358, in _do_run
|| | return self._do_call(_run_fn, feeds, fetches, targets, options,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py", line 1384, in _do_call
|| | raise type(e)(node_def, op, message)
|| | tensorflow.python.framework.errors_impl.ResourceExhaustedError: OOM when allocating tensor with shape[11,11,32,256] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc
|| | [[node default_policy/conv_value_3/kernel/Initializer/random_uniform/RandomUniform (defined at /usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py:83) ]]
|| | Hint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.
|| |
|| |
|| | Original stack trace for 'default_policy/conv_value_3/kernel/Initializer/random_uniform/RandomUniform':
|| | File "solution.py", line 127, in <module>
|| | main()
|| | File "solution.py", line 123, in main
|| | wrap_direct(node=node, protocol=protocol)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/interface.py", line 24, in wrap_direct
|| | run_loop(node, protocol, args)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 243, in run_loop
|| | loop(node_name, fi, fo, node, protocol, tin, tout, config=config, fi_desc=fin, fo_desc=fout)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 322, in loop
|| | call_if_fun_exists(node, "init", context=context_data)
|| | File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
|| | f(**kwargs)
|| | File "solution.py", line 29, in init
|| | self.model = RLlibModel(SEED,experiment_idx=0,checkpoint_idx=0,logger=context)
|| | File "/submission/model.py", line 55, in __init__
|| | self.model = PPOTrainer(config=config["rllib_config"])
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 90, in __init__
|| | Trainer.__init__(self, config, env, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 455, in __init__
|| | super().__init__(config, logger_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/tune/trainable.py", line 174, in __init__
|| | self._setup(copy.deepcopy(self.config))
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 596, in _setup
|| | self._init(self.config, self.env_creator)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer_template.py", line 115, in _init
|| | self.workers = self._make_workers(env_creator, self._policy,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/agents/trainer.py", line 662, in _make_workers
|| | return WorkerSet(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 61, in __init__
|| | self._local_worker = self._make_worker(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/worker_set.py", line 237, in _make_worker
|| | worker = cls(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 360, in __init__
|| | self._build_policy_map(policy_dict, policy_config)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/evaluation/rollout_worker.py", line 842, in _build_policy_map
|| | policy_map[name] = cls(obs_space, act_space, merged_conf)
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/tf_policy_template.py", line 129, in __init__
|| | DynamicTFPolicy.__init__(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/policy/dynamic_tf_policy.py", line 147, in __init__
|| | self.model = ModelCatalog.get_model_v2(
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/catalog.py", line 347, in get_model_v2
|| | return wrapper(obs_space, action_space, num_outputs, model_config,
|| | File "/usr/local/lib/python3.8/dist-packages/ray/rllib/models/tf/visionnet_v2.py", line 83, in __init__
|| | last_layer = tf.keras.layers.Conv2D(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 757, in __call__
|| | self._maybe_build(inputs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 2098, in _maybe_build
|| | self.build(input_shapes)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 197, in build
|| | self.kernel = self.add_weight(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 431, in add_weight
|| | variable = self._add_variable_with_custom_getter(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/training/tracking/base.py", line 745, in _add_variable_with_custom_getter
|| | new_variable = getter(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/base_layer_utils.py", line 133, in make_variable
|| | return tf_variables.VariableV1(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/variables.py", line 260, in __call__
|| | return cls._variable_v1_call(*args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/variables.py", line 206, in _variable_v1_call
|| | return previous_getter(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/variables.py", line 199, in <lambda>
|| | previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/variable_scope.py", line 2583, in default_variable_creator
|| | return resource_variable_ops.ResourceVariable(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/variables.py", line 264, in __call__
|| | return super(VariableMetaclass, cls).__call__(*args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/resource_variable_ops.py", line 1507, in __init__
|| | self._init_from_args(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/resource_variable_ops.py", line 1651, in _init_from_args
|| | initial_value() if init_from_fn else initial_value,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/init_ops.py", line 518, in __call__
|| | return random_ops.random_uniform(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/dispatch.py", line 201, in wrapper
|| | return target(*args, **kwargs)
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/random_ops.py", line 301, in random_uniform
|| | result = gen_random_ops.random_uniform(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_random_ops.py", line 742, in random_uniform
|| | _, _, _op, _outputs = _op_def_library._apply_op_helper(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/op_def_library.py", line 742, in _apply_op_helper
|| | op = g._create_op_internal(op_type_name, inputs, dtypes=None,
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 3477, in _create_op_internal
|| | ret = Operation(
|| | File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
|| | self._traceback = tf_stack.extract_stack()
|| |
|| |
||
|
András Kalapos 🇭🇺