Duckietown Challenges Home Challenges Submissions

Submission 2876

Submission2876
Competingyes
Challengeaido2-LF-sim-testing
UserAndrea Daniele 🇮🇹
Date submitted
Last status update
Completecomplete
DetailsEvaluation is complete.
Sisters
Result💚
Jobsstep1-simulation: 22726
Next
User labelminimal_agent (Python 3)
Admin priority50
Blessingn/a
User priority50

The highlights are available only to the owner and the admins.

Evaluation jobs for this submission

Show only up-to-date jobs
Job IDstepstatusup to datedate starteddate completeddurationmessage
22726step1-simulationsuccessyes0:39:15
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
driven_lanedir_consec_median0.6202968959903497
survival_time_median6.8999999999999835
deviation-center-line_median0.2968973039399582
in-drivable-lane_median0


other stats
agent_compute-ego_max0.17951467054353343
agent_compute-ego_mean0.17523637157914354
agent_compute-ego_median0.1754371519569013
agent_compute-ego_min0.1709729512532552
deviation-center-line_max1.2578907011919125
deviation-center-line_mean0.392559795052945
deviation-center-line_min0.20837582250812192
deviation-heading_max1.7338651399303977
deviation-heading_mean0.922434565822337
deviation-heading_median0.801279189692507
deviation-heading_min0.48740885281046914
driven_any_max1.7560498171261352
driven_any_mean0.8746411422725543
driven_any_median0.9886852216355494
driven_any_min0.278862430643389
driven_lanedir_consec_max1.7523714592470048
driven_lanedir_consec_mean0.6813457664348396
driven_lanedir_consec_min0.2720172605900939
driven_lanedir_max1.7523714592470048
driven_lanedir_mean0.6813457664348396
driven_lanedir_median0.6202968959903497
driven_lanedir_min0.2720172605900939
in-drivable-lane_max6.400000000000029
in-drivable-lane_mean1.1100000000000017
in-drivable-lane_min0
per-episodes
details{"ETHZ_autolab_technical_track-0-0": {"driven_any": 0.9900915586687412, "sim_physics": 0.13650447173084287, "survival_time": 6.949999999999983, "driven_lanedir": 0.6591401084474304, "sim_render-ego": 0.07181741007798009, "in-drivable-lane": 1.999999999999993, "agent_compute-ego": 0.1754371519569013, "deviation-heading": 0.801279189692507, "set_robot_commands": 0.11236827665095708, "deviation-center-line": 0.33899838427143014, "driven_lanedir_consec": 0.6591401084474304, "sim_compute_sim_state": 0.04360895191165183, "sim_compute_performance-ego": 0.08044795338198435, "sim_compute_robot_state-ego": 0.08770399127932761}, "ETHZ_autolab_technical_track-1-0": {"driven_any": 1.0058141307648691, "sim_physics": 0.13196887459073747, "survival_time": 6.999999999999983, "driven_lanedir": 0.8945129127998346, "sim_render-ego": 0.07161608593804496, "in-drivable-lane": 0.34999999999999876, "agent_compute-ego": 0.17664319106510706, "deviation-heading": 1.7338651399303977, "set_robot_commands": 0.11364908899579727, "deviation-center-line": 0.49658151961950325, "driven_lanedir_consec": 0.8945129127998346, "sim_compute_sim_state": 0.043769797257014686, "sim_compute_performance-ego": 0.08209801401410784, "sim_compute_robot_state-ego": 0.08811301674161638}, "ETHZ_autolab_technical_track-2-0": {"driven_any": 0.278862430643389, "sim_physics": 0.1463927745819092, "survival_time": 2.25, "driven_lanedir": 0.2720172605900939, "sim_render-ego": 0.07111065652635362, "in-drivable-lane": 0, "agent_compute-ego": 0.1709729512532552, "deviation-heading": 0.5056416838713088, "set_robot_commands": 0.1112990591261122, "deviation-center-line": 0.23430699853227116, "driven_lanedir_consec": 0.2720172605900939, "sim_compute_sim_state": 0.044500838385687934, "sim_compute_performance-ego": 0.07822379006279839, "sim_compute_robot_state-ego": 0.08496520784166124}, "ETHZ_autolab_technical_track-3-0": {"driven_any": 0.6407319399837514, "sim_physics": 0.1440693710161292, "survival_time": 4.599999999999992, "driven_lanedir": 0.6257610897668884, "sim_render-ego": 0.0726075846215953, "in-drivable-lane": 0, "agent_compute-ego": 0.17623048761616583, "deviation-heading": 0.9772760265854514, "set_robot_commands": 0.113432923088903, "deviation-center-line": 0.36802347786147127, "driven_lanedir_consec": 0.6257610897668884, "sim_compute_sim_state": 0.04447536623996237, "sim_compute_performance-ego": 0.08176000221915868, "sim_compute_robot_state-ego": 0.09470012136127638}, "ETHZ_autolab_technical_track-4-0": {"driven_any": 1.5927758413436612, "sim_physics": 0.1300266344253331, "survival_time": 10.95000000000002, "driven_lanedir": 0.6151282935769209, "sim_render-ego": 0.07000101538009296, "in-drivable-lane": 6.400000000000029, "agent_compute-ego": 0.1726576149735821, "deviation-heading": 0.638184343010141, "set_robot_commands": 0.1118219258034066, "deviation-center-line": 0.4584544318695165, "driven_lanedir_consec": 0.6151282935769209, "sim_compute_sim_state": 0.04312937444747855, "sim_compute_performance-ego": 0.08103551385609527, "sim_compute_robot_state-ego": 0.08970087521696744}, "ETHZ_autolab_technical_track-5-0": {"driven_any": 0.4747777342207285, "sim_physics": 0.1470543418611799, "survival_time": 3.4999999999999956, "driven_lanedir": 0.45069691160064007, "sim_render-ego": 0.07169316155569894, "in-drivable-lane": 0, "agent_compute-ego": 0.17481568881443568, "deviation-heading": 1.1234646030837343, "set_robot_commands": 0.1128300768988473, "deviation-center-line": 0.24614915791869676, "driven_lanedir_consec": 0.45069691160064007, "sim_compute_sim_state": 0.04552114009857178, "sim_compute_performance-ego": 0.07991392612457275, "sim_compute_robot_state-ego": 0.0901557275227138}, "ETHZ_autolab_technical_track-6-0": {"driven_any": 1.019681949804902, "sim_physics": 0.14934488921098307, "survival_time": 7.099999999999983, "driven_lanedir": 0.5497294169147833, "sim_render-ego": 0.06900491680897458, "in-drivable-lane": 2.9999999999999893, "agent_compute-ego": 0.17200321043041392, "deviation-heading": 1.017147490050159, "set_robot_commands": 0.1103101179633342, "deviation-center-line": 0.2968973039399582, "driven_lanedir_consec": 0.5497294169147833, "sim_compute_sim_state": 0.0434120473727374, "sim_compute_performance-ego": 0.08120952693509384, "sim_compute_robot_state-ego": 0.08820957868871554}, "ETHZ_autolab_technical_track-7-0": {"driven_any": 0.292561402657028, "sim_physics": 0.1353435568187548, "survival_time": 2.3, "driven_lanedir": 0.28556003912644146, "sim_render-ego": 0.06870046387548032, "in-drivable-lane": 0, "agent_compute-ego": 0.17732009680374808, "deviation-heading": 0.5055513317884642, "set_robot_commands": 0.11016585515893024, "deviation-center-line": 0.2291230953178777, "driven_lanedir_consec": 0.28556003912644146, "sim_compute_sim_state": 0.046542058820309845, "sim_compute_performance-ego": 0.07893885736880095, "sim_compute_robot_state-ego": 0.0893018038376518}, "ETHZ_autolab_technical_track-8-0": {"driven_any": 0.6284115088436346, "sim_physics": 0.1343188789155748, "survival_time": 4.499999999999992, "driven_lanedir": 0.6202968959903497, "sim_render-ego": 0.07169016732109917, "in-drivable-lane": 0, "agent_compute-ego": 0.1739970154232449, "deviation-heading": 0.6992938214416804, "set_robot_commands": 0.1117610428068373, "deviation-center-line": 0.292496185158671, "driven_lanedir_consec": 0.6202968959903497, "sim_compute_sim_state": 0.04405180878109402, "sim_compute_performance-ego": 0.07946018642849392, "sim_compute_robot_state-ego": 0.08971006605360243}, "ETHZ_autolab_technical_track-9-0": {"driven_any": 0.9886852216355494, "sim_physics": 0.14295135891955832, "survival_time": 6.8999999999999835, "driven_lanedir": 0.6459976804857221, "sim_render-ego": 0.07109204755313155, "in-drivable-lane": 1.7499999999999938, "agent_compute-ego": 0.1782268752222476, "deviation-heading": 1.4412214371425691, "set_robot_commands": 0.11043745538462764, "deviation-center-line": 0.3337191779486854, "driven_lanedir_consec": 0.6459976804857221, "sim_compute_sim_state": 0.04329431229743405, "sim_compute_performance-ego": 0.0818491310313128, "sim_compute_robot_state-ego": 0.09003292132115019}, "ETHZ_autolab_technical_track-10-0": {"driven_any": 1.5766678237160747, "sim_physics": 0.1411201275816751, "survival_time": 10.90000000000002, "driven_lanedir": 1.2238020245203944, "sim_render-ego": 0.07174633292976869, "in-drivable-lane": 1.8000000000000256, "agent_compute-ego": 0.17749898805530792, "deviation-heading": 1.6250699730411842, "set_robot_commands": 0.11294995207305349, "deviation-center-line": 0.5732131458770182, "driven_lanedir_consec": 1.2238020245203944, "sim_compute_sim_state": 0.044044763670055144, "sim_compute_performance-ego": 0.08184697868627146, "sim_compute_robot_state-ego": 0.0873519477494266}, "ETHZ_autolab_technical_track-11-0": {"driven_any": 0.4896825830686729, "sim_physics": 0.14152860310342577, "survival_time": 3.599999999999995, "driven_lanedir": 0.4827572868140654, "sim_render-ego": 0.0729701386557685, "in-drivable-lane": 0, "agent_compute-ego": 0.17677511109246147, "deviation-heading": 0.5911477230687963, "set_robot_commands": 0.1138222184446123, "deviation-center-line": 0.29144378866191073, "driven_lanedir_consec": 0.4827572868140654, "sim_compute_sim_state": 0.0460200740231408, "sim_compute_performance-ego": 0.07947545581393772, "sim_compute_robot_state-ego": 0.0861211551560296}, "ETHZ_autolab_technical_track-12-0": {"driven_any": 0.9934601655841236, "sim_physics": 0.15669773980010326, "survival_time": 6.949999999999983, "driven_lanedir": 0.7638704215325345, "sim_render-ego": 0.07260146587015055, "in-drivable-lane": 1.3499999999999952, "agent_compute-ego": 0.17951467054353343, "deviation-heading": 0.9502563901618318, "set_robot_commands": 0.11307434726962085, "deviation-center-line": 0.26272373511713126, "driven_lanedir_consec": 0.7638704215325345, "sim_compute_sim_state": 0.044994560077036026, "sim_compute_performance-ego": 0.08310384201488906, "sim_compute_robot_state-ego": 0.09469847884967172}, "ETHZ_autolab_technical_track-13-0": {"driven_any": 0.3913630260270548, "sim_physics": 0.1325865882938191, "survival_time": 2.9499999999999975, "driven_lanedir": 0.378544695109488, "sim_render-ego": 0.07944631980637372, "in-drivable-lane": 0, "agent_compute-ego": 0.1744601888171697, "deviation-heading": 0.7397104816563612, "set_robot_commands": 0.11294071957216424, "deviation-center-line": 0.20837582250812192, "driven_lanedir_consec": 0.378544695109488, "sim_compute_sim_state": 0.0448162798154152, "sim_compute_performance-ego": 0.07812969563371044, "sim_compute_robot_state-ego": 0.08322379548670882}, "ETHZ_autolab_technical_track-14-0": {"driven_any": 1.7560498171261352, "sim_physics": 0.13521481351733702, "survival_time": 12.050000000000036, "driven_lanedir": 1.7523714592470048, "sim_render-ego": 0.07172126987663048, "in-drivable-lane": 0, "agent_compute-ego": 0.17199233161957928, "deviation-heading": 0.48740885281046914, "set_robot_commands": 0.1085341590074088, "deviation-center-line": 1.2578907011919125, "driven_lanedir_consec": 1.7523714592470048, "sim_compute_sim_state": 0.044225985578481584, "sim_compute_performance-ego": 0.07882340320413043, "sim_compute_robot_state-ego": 0.08771322970568392}}
set_robot_commands_max0.1138222184446123
set_robot_commands_mean0.11195981454964082
set_robot_commands_median0.11236827665095708
set_robot_commands_min0.1085341590074088
sim_compute_performance-ego_max0.08310384201488906
sim_compute_performance-ego_mean0.08042108511835719
sim_compute_performance-ego_median0.08044795338198435
sim_compute_performance-ego_min0.07812969563371044
sim_compute_robot_state-ego_max0.09470012136127638
sim_compute_robot_state-ego_mean0.08878012778748022
sim_compute_robot_state-ego_median0.08820957868871554
sim_compute_robot_state-ego_min0.08322379548670882
sim_compute_sim_state_max0.046542058820309845
sim_compute_sim_state_mean0.04442715725173809
sim_compute_sim_state_median0.044225985578481584
sim_compute_sim_state_min0.04312937444747855
sim_physics_max0.15669773980010326
sim_physics_mean0.1403415349578242
sim_physics_median0.1411201275816751
sim_physics_min0.1300266344253331
sim_render-ego_max0.07944631980637372
sim_render-ego_mean0.07185460245314292
sim_render-ego_median0.07169316155569894
sim_render-ego_min0.06870046387548032
simulation-passed1
survival_time_max12.050000000000036
survival_time_mean6.166666666666666
survival_time_min2.25
No reset possible
20972step1-simulationsuccessno0:34:10
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
No reset possible
20953step1-simulationhost-errorno0:00:57
Uncaught exception: [...]
Uncaught exception:
Traceback (most recent call last):
  File "/project/src/duckietown_challenges_runner/runner.py", line 483, in get_cr
    submission_id=submission_id, timeout_sec=timeout_sec)
  File "/usr/lib/python3.6/contextlib.py", line 88, in __exit__
    next(self.gen)
  File "/project/src/duckietown_challenges_runner/runner.py", line 343, in setup_logging
    convert(f_stdout)
  File "/project/src/duckietown_challenges_runner/runner.py", line 332, in convert
    data = open(log).read().strip()
FileNotFoundError: [Errno 2] No such file or directory: '/tmp/duckietown/DT18/evaluator/executions/aido2-LF-sim-testing/submission2876/step1-simulation-ip-172-31-42-167-7227-job20953/logs/challenges-runner/stdout.log'
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
No reset possible
20284step1-simulationsuccessno0:12:53
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
No reset possible
20267step1-simulationhost-errorno0:02:26
Uncaught exception: [...]
Uncaught exception:
Traceback (most recent call last):
  File "/project/src/duckietown_challenges_runner/runner.py", line 990, in run_docker
    return subprocess.check_output(cmd0, cwd=cwd, stderr=sys.stderr)
  File "/usr/lib/python3.6/subprocess.py", line 336, in check_output
    **kwargs).stdout
  File "/usr/lib/python3.6/subprocess.py", line 418, in run
    output=stdout, stderr=stderr)
subprocess.CalledProcessError: Command '['docker-compose', '-p', 'job20267-487756', 'ps', '-q', 'solution']' returned non-zero exit status 1.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/project/src/duckietown_challenges_runner/runner.py", line 478, in get_cr
    submission_id=submission_id, timeout_sec=timeout_sec)
  File "/project/src/duckietown_challenges_runner/runner.py", line 573, in run_single
    write_logs(wd, project, services=config['services'])
  File "/project/src/duckietown_challenges_runner/runner.py", line 899, in write_logs
    container_id = get_id_for_service(wd, project, service)
  File "/project/src/duckietown_challenges_runner/runner.py", line 585, in get_id_for_service
    o = run_docker(wd, project, cmd, get_output=True)
  File "/project/src/duckietown_challenges_runner/runner.py", line 998, in run_docker
    raise DockerComposeFail(msg) from e
duckietown_challenges_runner.runner.DockerComposeFail: Could not run ['docker-compose', '-p', 'job20267-487756', 'ps', '-q', 'solution']:

   >  Command '['docker-compose', '-p', 'job20267-487756', 'ps', '-q', 'solution']' returned non-zero exit status 1.

stdout | 

stderr | 
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
No reset possible
20266step1-simulationhost-errorno0:02:27
Uncaught exception: [...]
Uncaught exception:
Traceback (most recent call last):
  File "/project/src/duckietown_challenges_runner/runner.py", line 990, in run_docker
    return subprocess.check_output(cmd0, cwd=cwd, stderr=sys.stderr)
  File "/usr/lib/python3.6/subprocess.py", line 336, in check_output
    **kwargs).stdout
  File "/usr/lib/python3.6/subprocess.py", line 418, in run
    output=stdout, stderr=stderr)
subprocess.CalledProcessError: Command '['docker-compose', '-p', 'job20266-376130', 'ps', '-q', 'solution']' returned non-zero exit status 255.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/project/src/duckietown_challenges_runner/runner.py", line 478, in get_cr
    submission_id=submission_id, timeout_sec=timeout_sec)
  File "/project/src/duckietown_challenges_runner/runner.py", line 573, in run_single
    write_logs(wd, project, services=config['services'])
  File "/project/src/duckietown_challenges_runner/runner.py", line 899, in write_logs
    container_id = get_id_for_service(wd, project, service)
  File "/project/src/duckietown_challenges_runner/runner.py", line 585, in get_id_for_service
    o = run_docker(wd, project, cmd, get_output=True)
  File "/project/src/duckietown_challenges_runner/runner.py", line 998, in run_docker
    raise DockerComposeFail(msg) from e
duckietown_challenges_runner.runner.DockerComposeFail: Could not run ['docker-compose', '-p', 'job20266-376130', 'ps', '-q', 'solution']:

   >  Command '['docker-compose', '-p', 'job20266-376130', 'ps', '-q', 'solution']' returned non-zero exit status 255.

stdout | 

stderr | 
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
No reset possible
19206step1-simulationsuccessno0:13:36
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
No reset possible
19183step1-simulationerrorno0:03:59
InvalidEvaluator: Tr [...]
InvalidEvaluator:
Traceback (most recent call last):
  File "experiment_manager.py", line 173, in main
    make_video1(fn, out_video)
  File "/project/src/aido-protocols/src/aido_schemas/utils_video.py", line 56, in make_video1
    pg('video_aido', dict(filename=fn, output=output))
  File "/project/src/procgraph/src/procgraph/scripts/pgmain.py", line 244, in pg
    model.init()
  File "/project/src/procgraph/src/procgraph/core/model.py", line 218, in init
    block.init()
  File "/project/src/aido-protocols/src/aido_schemas/utils_video.py", line 16, in init
    self.log = read_simulator_log_cbor(self.get_config('filename'))
  File "/project/src/aido-protocols/src/aido_schemas/utils_drawing.py", line 179, in read_simulator_log_cbor
    render_time = read_perfomance(ld)
  File "/project/src/aido-protocols/src/aido_schemas/utils_drawing.py", line 74, in read_perfomance
    for i, ob in enumerate(read_topic2(ld, 'timing_information')):
  File "/project/src/aido-protocols/src/aido_schemas/utils_drawing.py", line 49, in read_topic2
    for ob in ld.objects:
AttributeError: 'str' object has no attribute 'objects'

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/project/src/duckietown-challenges/src/duckietown_challenges/cie_concrete.py", line 610, in scoring_context
    yield cie
  File "experiment_manager.py", line 435, in <module>
    wrap(cie)
  File "experiment_manager.py", line 423, in wrap
    main(cie, logdir, attempts)
  File "experiment_manager.py", line 186, in main
    raise dc.InvalidEvaluator(msg) from e
duckietown_challenges.exceptions.InvalidEvaluator: Anomalous error while running episodes:

 > Traceback (most recent call last):
 >   File "experiment_manager.py", line 173, in main
 >     make_video1(fn, out_video)
 >   File "/project/src/aido-protocols/src/aido_schemas/utils_video.py", line 56, in make_video1
 >     pg('video_aido', dict(filename=fn, output=output))
 >   File "/project/src/procgraph/src/procgraph/scripts/pgmain.py", line 244, in pg
 >     model.init()
 >   File "/project/src/procgraph/src/procgraph/core/model.py", line 218, in init
 >     block.init()
 >   File "/project/src/aido-protocols/src/aido_schemas/utils_video.py", line 16, in init
 >     self.log = read_simulator_log_cbor(self.get_config('filename'))
 >   File "/project/src/aido-protocols/src/aido_schemas/utils_drawing.py", line 179, in read_simulator_log_cbor
 >     render_time = read_perfomance(ld)
 >   File "/project/src/aido-protocols/src/aido_schemas/utils_drawing.py", line 74, in read_perfomance
 >     for i, ob in enumerate(read_topic2(ld, 'timing_information')):
 >   File "/project/src/aido-protocols/src/aido_schemas/utils_drawing.py", line 49, in read_topic2
 >     for ob in ld.objects:
 > AttributeError: 'str' object has no attribute 'objects'
 > 
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
No reset possible
18944step1-simulationsuccessno0:16:07
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
No reset possible
18934step1-simulationsuccessno0:15:01
Artefacts hidden. If you are the author, please login using the top-right link or use the dashboard.
No reset possible