Job 47903

InvalidEnvironment:
Traceback (most recent call last):
  File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/code.py", line 242, in main
    robot_ci.write_topic_and_expect_zero("seed", config.seed)
  File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 143, in write_topic_and_expect_zero
    msgs = read_reply(self.fpout, timeout=timeout, nickname=self.nickname)
  File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 309, in read_reply
    raise RemoteNodeAborted(msg)
zuper_nodes.structures.RemoteNodeAborted: The remote node "ego0" aborted with the following error:

error in ego0 |Unexpected error:
              |
              || Traceback (most recent call last):
              ||   File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 320, in loop
              ||     call_if_fun_exists(node, "init", context=context_data)
              ||   File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
              ||     f(**kwargs)
              ||   File "solution.py", line 29, in init
              ||     self.model = DDPG(state_dim=self.preprocessor.shape, action_dim=2, max_action=1, net_type="cnn")
              ||   File "/submission/model.py", line 153, in __init__
              ||     self.actor = ActorCNN(action_dim, max_action).to(device)
              ||   File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 612, in to
              ||     return self._apply(convert)
              ||   File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 359, in _apply
              ||     module._apply(fn)
              ||   File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 381, in _apply
              ||     param_applied = fn(param)
              ||   File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 610, in convert
              ||     return t.to(device, dtype if t.is_floating_point() else None, non_blocking)
              || RuntimeError: CUDA error: out of memory
              ||

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/local/lib/python3.8/site-packages/duckietown_challenges/cie_concrete.py", line 681, in scoring_context
    yield cie
  File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/experiment_manager.py", line 68, in go
    wrap(cie)
  File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/experiment_manager.py", line 34, in wrap
    asyncio.run(main(cie, logdir, attempts), debug=True)
  File "/usr/local/lib/python3.8/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/usr/local/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
    return future.result()
  File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/code.py", line 248, in main
    raise InvalidEnvironment(msg) from e
duckietown_challenges.exceptions.InvalidEnvironment: Detected out of CUDA memory:

Traceback (most recent call last):
  File "/usr/local/lib/python3.8/site-packages/duckietown_experiment_manager/code.py", line 242, in main
    robot_ci.write_topic_and_expect_zero("seed", config.seed)
  File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 143, in write_topic_and_expect_zero
    msgs = read_reply(self.fpout, timeout=timeout, nickname=self.nickname)
  File "/usr/local/lib/python3.8/site-packages/zuper_nodes_wrapper/wrapper_outside.py", line 309, in read_reply
    raise RemoteNodeAborted(msg)
zuper_nodes.structures.RemoteNodeAborted: The remote node "ego0" aborted with the following error:

error in ego0 |Unexpected error:
              |
              || Traceback (most recent call last):
              ||   File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/wrapper.py", line 320, in loop
              ||     call_if_fun_exists(node, "init", context=context_data)
              ||   File "/usr/local/lib/python3.8/dist-packages/zuper_nodes_wrapper/utils.py", line 21, in call_if_fun_exists
              ||     f(**kwargs)
              ||   File "solution.py", line 29, in init
              ||     self.model = DDPG(state_dim=self.preprocessor.shape, action_dim=2, max_action=1, net_type="cnn")
              ||   File "/submission/model.py", line 153, in __init__
              ||     self.actor = ActorCNN(action_dim, max_action).to(device)
              ||   File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 612, in to
              ||     return self._apply(convert)
              ||   File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 359, in _apply
              ||     module._apply(fn)
              ||   File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 381, in _apply
              ||     param_applied = fn(param)
              ||   File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 610, in convert
              ||     return t.to(device, dtype if t.is_floating_point() else None, non_blocking)
              || RuntimeError: CUDA error: out of memory
              ||