The results below are organized as follows:
- each table displays the solver’s performance for individual problem instances for the given task under different time limits
- table values are normalized scores for each evaluated problem as outlined in Evaluation Criteria
PR
overall
| Problem |
20sec |
1200sec |
3600sec |
| Alchemy_11 |
0.0 (nan) |
96.7 (606.3) |
96.7 (606.3) |
| CSP_11 |
0.0 (nan) |
86.3 (14.7) |
86.3 (14.7) |
| CSP_12 |
86.2 (17.3) |
86.2 (17.3) |
86.2 (17.3) |
| CSP_13 |
0.0 (nan) |
86.5 (17.0) |
86.5 (17.0) |
| DBN_11 |
0.0 (nan) |
99.9 (58.5) |
99.9 (58.5) |
| DBN_12 |
0.0 (nan) |
100.0 (63.1) |
100.0 (63.1) |
| DBN_13 |
0.0 (nan) |
99.6 (66.1) |
99.6 (66.1) |
| DBN_14 |
0.0 (nan) |
100.0 (151.2) |
100.0 (151.2) |
| DBN_15 |
0.0 (nan) |
80.3 (150.7) |
80.3 (150.7) |
| DBN_16 |
0.0 (nan) |
99.9 (166.1) |
99.9 (166.1) |
| Grids_11 |
65.2 (184.9) |
65.2 (184.9) |
65.2 (184.9) |
| Grids_12 |
49.8 (349.7) |
49.8 (349.7) |
49.8 (349.7) |
| Grids_13 |
64.7 (364.2) |
64.7 (364.2) |
64.7 (364.2) |
| Grids_14 |
81.4 (522.2) |
81.4 (522.2) |
81.4 (522.2) |
| Grids_15 |
0.0 (nan) |
95.5 (294.7) |
95.5 (294.7) |
| Grids_16 |
0.0 (nan) |
66.4 (719.3) |
66.4 (719.3) |
| Grids_17 |
0.0 (nan) |
50.1 (1471.1) |
50.1 (1471.1) |
| Grids_18 |
0.0 (nan) |
74.5 (2084.4) |
74.5 (2084.4) |
| ObjectDetection_11 |
0.0 (nan) |
94.8 (-75.0) |
94.8 (-75.0) |
| ObjectDetection_12 |
0.0 (nan) |
92.1 (-67.6) |
92.1 (-67.6) |
| ObjectDetection_13 |
0.0 (nan) |
93.1 (-70.6) |
93.1 (-70.6) |
| ObjectDetection_14 |
0.0 (nan) |
90.9 (-64.3) |
90.9 (-64.3) |
| ObjectDetection_15 |
0.0 (nan) |
94.6 (-51.9) |
94.6 (-51.9) |
| ObjectDetection_16 |
0.0 (nan) |
93.8 (-34.5) |
93.8 (-34.5) |
| ObjectDetection_17 |
0.0 (nan) |
93.7 (-29.9) |
93.7 (-29.9) |
| ObjectDetection_18 |
0.0 (nan) |
85.6 (-37.3) |
85.6 (-37.3) |
| ObjectDetection_19 |
0.0 (nan) |
61.6 (-39.2) |
61.6 (-39.2) |
| ObjectDetection_20 |
0.0 (nan) |
71.8 (-35.5) |
71.8 (-35.5) |
| ObjectDetection_21 |
0.0 (nan) |
89.1 (-24.5) |
89.1 (-24.5) |
| ObjectDetection_22 |
0.0 (nan) |
91.3 (-36.8) |
91.3 (-36.8) |
| ObjectDetection_23 |
0.0 (nan) |
92.0 (-31.7) |
92.0 (-31.7) |
| ObjectDetection_24 |
0.0 (nan) |
92.3 (-26.5) |
92.3 (-26.5) |
| ObjectDetection_25 |
0.0 (nan) |
96.0 (-18.7) |
96.0 (-18.7) |
| ObjectDetection_26 |
0.0 (nan) |
92.1 (-33.8) |
92.1 (-33.8) |
| ObjectDetection_27 |
0.0 (nan) |
93.8 (-29.9) |
93.8 (-29.9) |
| ObjectDetection_28 |
0.0 (nan) |
98.3 (-46.8) |
98.3 (-46.8) |
| ObjectDetection_29 |
0.0 (nan) |
95.6 (-50.9) |
95.6 (-50.9) |
| ObjectDetection_30 |
0.0 (nan) |
88.0 (-52.6) |
88.0 (-52.6) |
| ObjectDetection_31 |
0.0 (nan) |
92.5 (-44.2) |
92.5 (-44.2) |
| ObjectDetection_32 |
0.0 (nan) |
88.7 (-46.8) |
88.7 (-46.8) |
| ObjectDetection_33 |
0.0 (nan) |
89.2 (-44.2) |
89.2 (-44.2) |
| ObjectDetection_34 |
0.0 (nan) |
98.0 (-35.4) |
98.0 (-35.4) |
| ObjectDetection_35 |
0.0 (nan) |
97.2 (-37.6) |
97.2 (-37.6) |
| ObjectDetection_36 |
0.0 (nan) |
97.3 (-30.7) |
97.3 (-30.7) |
| ObjectDetection_37 |
0.0 (nan) |
96.1 (-30.0) |
96.1 (-30.0) |
| ObjectDetection_38 |
0.0 (nan) |
96.3 (-75.1) |
96.3 (-75.1) |
| ObjectDetection_39 |
0.0 (nan) |
95.1 (-70.3) |
95.1 (-70.3) |
| ObjectDetection_40 |
0.0 (nan) |
95.2 (-72.4) |
95.2 (-72.4) |
| ObjectDetection_41 |
0.0 (nan) |
0.0 (-73.7) |
0.0 (-73.7) |
| ObjectDetection_42 |
0.0 (nan) |
97.3 (-45.2) |
97.3 (-45.2) |
| ObjectDetection_43 |
0.0 (nan) |
95.7 (-39.5) |
95.7 (-39.5) |
| ObjectDetection_44 |
0.0 (nan) |
94.7 (-41.1) |
94.7 (-41.1) |
| ObjectDetection_45 |
0.0 (nan) |
94.6 (-39.0) |
94.6 (-39.0) |
| ObjectDetection_46 |
0.0 (nan) |
0.0 (-43.0) |
0.0 (-43.0) |
| ObjectDetection_47 |
0.0 (nan) |
0.0 (-45.8) |
0.0 (-45.8) |
| ObjectDetection_48 |
0.0 (nan) |
0.0 (-38.0) |
0.0 (-38.0) |
| ObjectDetection_49 |
0.0 (nan) |
0.0 (-40.1) |
0.0 (-40.1) |
| ObjectDetection_50 |
0.0 (nan) |
32.2 (-37.1) |
32.2 (-37.1) |
| ObjectDetection_51 |
0.0 (nan) |
97.6 (-53.4) |
97.6 (-53.4) |
| ObjectDetection_52 |
0.0 (nan) |
97.4 (-57.0) |
97.4 (-57.0) |
| ObjectDetection_53 |
0.0 (nan) |
97.5 (-47.8) |
97.5 (-47.8) |
| ObjectDetection_54 |
0.0 (nan) |
98.2 (-41.5) |
98.2 (-41.5) |
| ObjectDetection_55 |
0.0 (nan) |
98.2 (-34.1) |
98.2 (-34.1) |
| ObjectDetection_56 |
0.0 (nan) |
97.9 (-36.3) |
97.9 (-36.3) |
| ObjectDetection_57 |
0.0 (nan) |
97.8 (-33.4) |
97.8 (-33.4) |
| ObjectDetection_58 |
0.0 (nan) |
95.4 (-41.7) |
95.4 (-41.7) |
| ObjectDetection_59 |
0.0 (nan) |
93.6 (-35.9) |
93.6 (-35.9) |
| ObjectDetection_60 |
0.0 (nan) |
92.9 (-35.3) |
92.9 (-35.3) |
| ObjectDetection_61 |
0.0 (nan) |
98.4 (-23.2) |
98.4 (-23.2) |
| ObjectDetection_62 |
0.0 (nan) |
98.7 (-24.8) |
98.7 (-24.8) |
| ObjectDetection_63 |
0.0 (nan) |
98.2 (-19.9) |
98.2 (-19.9) |
| ObjectDetection_64 |
0.0 (nan) |
98.4 (-21.3) |
98.4 (-21.3) |
| ObjectDetection_65 |
0.0 (nan) |
98.1 (-19.7) |
98.1 (-19.7) |
| ObjectDetection_66 |
0.0 (nan) |
95.9 (-47.1) |
95.9 (-47.1) |
| ObjectDetection_67 |
0.0 (nan) |
95.3 (-40.7) |
95.3 (-40.7) |
| ObjectDetection_68 |
0.0 (nan) |
95.6 (-42.7) |
95.6 (-42.7) |
| ObjectDetection_69 |
0.0 (nan) |
95.3 (-40.3) |
95.3 (-40.3) |
| ObjectDetection_70 |
96.6 (-31.3) |
96.6 (-31.3) |
96.6 (-31.3) |
| ObjectDetection_71 |
0.0 (nan) |
97.3 (-33.7) |
97.3 (-33.7) |
| ObjectDetection_72 |
0.0 (nan) |
93.0 (-26.5) |
93.0 (-26.5) |
| ObjectDetection_73 |
0.0 (nan) |
93.1 (-25.7) |
93.1 (-25.7) |
| ObjectDetection_74 |
0.0 (nan) |
98.7 (-31.8) |
98.7 (-31.8) |
| ObjectDetection_75 |
0.0 (nan) |
98.5 (-33.8) |
98.5 (-33.8) |
| Pedigree_11 |
0.0 (nan) |
96.0 (-18.2) |
96.0 (-18.2) |
| Pedigree_12 |
0.0 (nan) |
96.4 (-12.1) |
96.4 (-12.1) |
| Pedigree_13 |
0.0 (nan) |
92.5 (-16.7) |
92.5 (-16.7) |
| Promedus_11 |
0.0 (nan) |
96.0 (-8.6) |
96.0 (-8.6) |
| Promedus_12 |
0.0 (nan) |
93.6 (-3.3) |
93.6 (-3.3) |
| Promedus_13 |
0.0 (nan) |
97.4 (-4.6) |
97.4 (-4.6) |
| Promedus_14 |
96.6 (-8.0) |
96.6 (-8.0) |
96.6 (-8.0) |
| Promedus_15 |
91.4 (-3.8) |
91.4 (-3.8) |
91.4 (-3.8) |
| Promedus_16 |
0.0 (nan) |
99.3 (-7.0) |
99.3 (-7.0) |
| Promedus_17 |
0.0 (nan) |
99.6 (-9.5) |
99.6 (-9.5) |
| Promedus_18 |
0.0 (nan) |
99.9 (-4.7) |
99.9 (-4.7) |
| Promedus_19 |
0.0 (nan) |
100.0 (-4.3) |
100.0 (-4.3) |
| Promedus_20 |
0.0 (nan) |
94.1 (-7.3) |
94.1 (-7.3) |
| Promedus_21 |
0.0 (nan) |
88.9 (-5.8) |
88.9 (-5.8) |
| Promedus_22 |
93.1 (-2.6) |
93.1 (-2.6) |
93.1 (-2.6) |
| Promedus_23 |
0.0 (nan) |
94.9 (-11.5) |
94.9 (-11.5) |
| Promedus_24 |
0.0 (nan) |
99.8 (-5.9) |
99.8 (-5.9) |
| Promedus_25 |
0.0 (nan) |
98.4 (-9.5) |
98.4 (-9.5) |
| Promedus_26 |
0.0 (nan) |
98.8 (-7.3) |
98.8 (-7.3) |
| Promedus_27 |
91.4 (-8.4) |
91.4 (-8.4) |
91.4 (-8.4) |
| Promedus_28 |
0.0 (nan) |
95.0 (-8.3) |
95.0 (-8.3) |
| Promedus_29 |
99.2 (-10.5) |
99.2 (-10.5) |
99.2 (-10.5) |
| Promedus_30 |
95.3 (-22.2) |
95.3 (-22.2) |
95.3 (-22.2) |
| Promedus_31 |
0.0 (nan) |
89.3 (-2.0) |
89.3 (-2.0) |
| Promedus_32 |
0.0 (nan) |
95.4 (-2.3) |
95.4 (-2.3) |
| Promedus_33 |
0.0 (nan) |
99.9 (-2.8) |
99.9 (-2.8) |
| Promedus_34 |
0.0 (nan) |
90.2 (-3.3) |
90.2 (-3.3) |
| Promedus_35 |
88.6 (-1.9) |
88.6 (-1.9) |
88.6 (-1.9) |
| Promedus_36 |
0.0 (nan) |
88.6 (-1.9) |
88.6 (-1.9) |
| Promedus_37 |
0.0 (nan) |
96.4 (-4.3) |
96.4 (-4.3) |
| Promedus_38 |
0.0 (nan) |
95.5 (-5.2) |
95.5 (-5.2) |
| Segmentation_11 |
0.0 (nan) |
88.4 (-25.9) |
88.4 (-25.9) |
| Segmentation_12 |
0.0 (nan) |
100.0 (-10.3) |
100.0 (-10.3) |
| Segmentation_13 |
0.0 (nan) |
94.5 (-34.4) |
94.5 (-34.4) |
| Segmentation_14 |
0.0 (nan) |
99.1 (-39.7) |
99.1 (-39.7) |
| Segmentation_15 |
0.0 (nan) |
49.7 (-31.6) |
49.7 (-31.6) |
| Segmentation_16 |
0.0 (nan) |
98.3 (-38.5) |
98.3 (-38.5) |
| grid10x10.f10 |
49.8 (349.7) |
49.8 (349.7) |
49.8 (349.7) |
| linkage_12 |
0.0 (nan) |
97.4 (-71.7) |
97.4 (-71.7) |
| linkage_13 |
0.0 (nan) |
97.9 (-77.3) |
97.9 (-77.3) |
| linkage_14 |
0.0 (nan) |
92.6 (-32.0) |
92.6 (-32.0) |
| linkage_16 |
0.0 (nan) |
82.2 (-41.3) |
82.2 (-41.3) |
| linkage_18 |
0.0 (nan) |
95.5 (-81.7) |
95.5 (-81.7) |
| linkage_19 |
0.0 (nan) |
94.1 (-62.6) |
94.1 (-62.6) |
| linkage_21 |
0.0 (nan) |
93.8 (-31.4) |
93.8 (-31.4) |
| linkage_22 |
0.0 (nan) |
93.7 (-57.2) |
93.7 (-57.2) |
| linkage_23 |
0.0 (nan) |
85.2 (-125.3) |
85.2 (-125.3) |
| linkage_24 |
0.0 (nan) |
95.0 (-87.0) |
95.0 (-87.0) |
| linkage_26 |
0.0 (nan) |
95.3 (-118.8) |
95.3 (-118.8) |
| linkage_27 |
0.0 (nan) |
90.5 (-67.7) |
90.5 (-67.7) |
MAR
overall
| Problem |
20sec |
1200sec |
3600sec |
| Alchemy_11 |
0.0 |
96.8 |
96.8 |
| CSP_11 |
0.0 |
78.3 |
78.3 |
| CSP_12 |
77.5 |
77.5 |
77.5 |
| CSP_13 |
0.0 |
75.1 |
75.1 |
| DBN_11 |
0.0 |
17.5 |
17.5 |
| DBN_12 |
0.0 |
84.9 |
84.9 |
| DBN_13 |
0.0 |
0.0 |
0.0 |
| DBN_14 |
0.0 |
100.0 |
100.0 |
| DBN_15 |
0.0 |
0.0 |
0.0 |
| DBN_16 |
0.0 |
68.0 |
68.0 |
| Grids_11 |
0.1 |
0.1 |
0.1 |
| Grids_12 |
36.5 |
36.5 |
36.5 |
| Grids_13 |
13.2 |
13.2 |
13.2 |
| Grids_14 |
0.0 |
0.0 |
0.0 |
| Grids_15 |
0.0 |
23.4 |
23.4 |
| Grids_16 |
0.0 |
0.0 |
0.0 |
| Grids_17 |
0.0 |
0.0 |
0.0 |
| Grids_18 |
0.0 |
0.0 |
0.0 |
| ObjectDetection_11 |
0.0 |
87.0 |
87.0 |
| ObjectDetection_12 |
0.0 |
80.6 |
80.6 |
| ObjectDetection_13 |
0.0 |
82.1 |
82.1 |
| ObjectDetection_14 |
0.0 |
76.1 |
76.1 |
| ObjectDetection_15 |
0.0 |
91.6 |
91.6 |
| ObjectDetection_16 |
0.0 |
89.3 |
89.3 |
| ObjectDetection_17 |
0.0 |
90.8 |
90.8 |
| ObjectDetection_18 |
0.0 |
52.9 |
52.9 |
| ObjectDetection_19 |
0.0 |
9.1 |
9.1 |
| ObjectDetection_20 |
0.0 |
35.4 |
35.4 |
| ObjectDetection_21 |
0.0 |
82.4 |
82.4 |
| ObjectDetection_22 |
0.0 |
86.1 |
86.1 |
| ObjectDetection_23 |
0.0 |
83.5 |
83.5 |
| ObjectDetection_24 |
0.0 |
84.6 |
84.6 |
| ObjectDetection_25 |
0.0 |
94.8 |
94.8 |
| ObjectDetection_26 |
0.0 |
91.3 |
91.3 |
| ObjectDetection_27 |
0.0 |
93.2 |
93.2 |
| ObjectDetection_28 |
0.0 |
98.7 |
98.7 |
| ObjectDetection_29 |
0.0 |
93.7 |
93.7 |
| ObjectDetection_30 |
0.0 |
86.1 |
86.1 |
| ObjectDetection_31 |
0.0 |
91.5 |
91.5 |
| ObjectDetection_32 |
0.0 |
86.5 |
86.5 |
| ObjectDetection_33 |
0.0 |
86.4 |
86.4 |
| ObjectDetection_34 |
0.0 |
98.3 |
98.3 |
| ObjectDetection_35 |
0.0 |
97.3 |
97.3 |
| ObjectDetection_36 |
0.0 |
97.7 |
97.7 |
| ObjectDetection_37 |
0.0 |
96.6 |
96.6 |
| ObjectDetection_38 |
0.0 |
92.7 |
92.7 |
| ObjectDetection_39 |
0.0 |
88.6 |
88.6 |
| ObjectDetection_40 |
0.0 |
86.5 |
86.5 |
| ObjectDetection_41 |
0.0 |
0.0 |
0.0 |
| ObjectDetection_42 |
0.0 |
96.5 |
96.5 |
| ObjectDetection_43 |
0.0 |
96.2 |
96.2 |
| ObjectDetection_44 |
0.0 |
94.3 |
94.3 |
| ObjectDetection_45 |
0.0 |
93.5 |
93.5 |
| ObjectDetection_46 |
0.0 |
0.0 |
0.0 |
| ObjectDetection_47 |
0.0 |
0.0 |
0.0 |
| ObjectDetection_48 |
0.0 |
0.0 |
0.0 |
| ObjectDetection_49 |
0.0 |
0.0 |
0.0 |
| ObjectDetection_50 |
0.0 |
0.0 |
0.0 |
| ObjectDetection_51 |
0.0 |
96.1 |
96.1 |
| ObjectDetection_52 |
0.0 |
94.8 |
94.8 |
| ObjectDetection_53 |
0.0 |
94.6 |
94.6 |
| ObjectDetection_54 |
0.0 |
97.5 |
97.5 |
| ObjectDetection_55 |
0.0 |
97.2 |
97.2 |
| ObjectDetection_56 |
0.0 |
96.7 |
96.7 |
| ObjectDetection_57 |
0.0 |
95.8 |
95.8 |
| ObjectDetection_58 |
0.0 |
92.2 |
92.2 |
| ObjectDetection_59 |
0.0 |
93.4 |
93.4 |
| ObjectDetection_60 |
0.0 |
92.3 |
92.3 |
| ObjectDetection_61 |
0.0 |
97.0 |
97.0 |
| ObjectDetection_62 |
0.0 |
97.6 |
97.6 |
| ObjectDetection_63 |
0.0 |
96.7 |
96.7 |
| ObjectDetection_64 |
0.0 |
96.8 |
96.8 |
| ObjectDetection_65 |
0.0 |
96.3 |
96.3 |
| ObjectDetection_66 |
93.4 |
93.4 |
93.4 |
| ObjectDetection_67 |
0.0 |
92.8 |
92.8 |
| ObjectDetection_68 |
0.0 |
92.4 |
92.4 |
| ObjectDetection_69 |
0.0 |
92.5 |
92.5 |
| ObjectDetection_70 |
78.3 |
78.3 |
78.3 |
| ObjectDetection_71 |
0.0 |
88.6 |
88.6 |
| ObjectDetection_72 |
0.0 |
57.0 |
57.0 |
| ObjectDetection_73 |
0.0 |
56.1 |
56.1 |
| ObjectDetection_74 |
0.0 |
97.9 |
97.9 |
| ObjectDetection_75 |
0.0 |
98.4 |
98.4 |
| Pedigree_11 |
0.0 |
0.0 |
0.0 |
| Pedigree_12 |
0.0 |
0.0 |
0.0 |
| Pedigree_13 |
0.0 |
0.0 |
0.0 |
| Promedus_11 |
0.0 |
0.0 |
0.0 |
| Promedus_12 |
0.0 |
6.1 |
6.1 |
| Promedus_13 |
0.0 |
0.0 |
0.0 |
| Promedus_14 |
0.0 |
0.0 |
0.0 |
| Promedus_15 |
13.6 |
13.6 |
13.6 |
| Promedus_16 |
0.0 |
23.8 |
23.8 |
| Promedus_17 |
0.0 |
0.0 |
0.0 |
| Promedus_18 |
0.0 |
17.7 |
17.7 |
| Promedus_19 |
0.0 |
33.6 |
33.6 |
| Promedus_20 |
0.0 |
0.0 |
0.0 |
| Promedus_21 |
0.0 |
14.6 |
14.6 |
| Promedus_22 |
33.6 |
33.6 |
33.6 |
| Promedus_23 |
0.0 |
4.8 |
4.8 |
| Promedus_24 |
0.0 |
29.5 |
29.5 |
| Promedus_25 |
0.0 |
18.2 |
18.2 |
| Promedus_26 |
0.0 |
8.5 |
8.5 |
| Promedus_27 |
9.7 |
9.7 |
9.7 |
| Promedus_28 |
0.0 |
0.0 |
0.0 |
| Promedus_29 |
17.9 |
17.9 |
17.9 |
| Promedus_30 |
18.6 |
18.6 |
18.6 |
| Promedus_31 |
0.0 |
13.9 |
13.9 |
| Promedus_32 |
0.0 |
16.0 |
16.0 |
| Promedus_33 |
0.0 |
10.8 |
10.8 |
| Promedus_34 |
0.0 |
8.5 |
8.5 |
| Promedus_35 |
10.6 |
10.6 |
10.6 |
| Promedus_36 |
0.0 |
10.6 |
10.6 |
| Promedus_37 |
0.0 |
0.7 |
0.7 |
| Promedus_38 |
0.0 |
2.4 |
2.4 |
| Segmentation_11 |
0.0 |
38.1 |
38.1 |
| Segmentation_12 |
0.0 |
100.0 |
100.0 |
| Segmentation_13 |
0.0 |
50.8 |
50.8 |
| Segmentation_14 |
0.0 |
89.9 |
89.9 |
| Segmentation_15 |
0.0 |
48.6 |
48.6 |
| Segmentation_16 |
0.0 |
80.7 |
80.7 |
| linkage_12 |
0.0 |
6.7 |
6.7 |
| linkage_13 |
0.0 |
0.0 |
0.0 |
| linkage_14 |
0.0 |
86.0 |
86.0 |