The results below are organized as follows:
- each table below displays various solvers’ performance for individual problem instances for the given task and time limit
- table values are normalized scores for each evaluated problem as outlined in Evaluation Criteria
overall
20sec
| Problem |
baseline |
ibia-mar |
lbp-mar |
uai14-mar |
| 1aie |
0 |
100.0 |
100.0 |
100.0 |
| 1bxv |
0 |
100.0 |
100.0 |
100.0 |
| 1g6x |
0 |
99.9 |
100.0 |
100.0 |
| 1jer |
0 |
100.0 |
100.0 |
100.0 |
| 1npl |
0 |
100.0 |
100.0 |
100.0 |
| 1qt9 |
0 |
89.9 |
100.0 |
98.0 |
| 1rfs |
0 |
91.7 |
100.0 |
100.0 |
| 2rta |
0 |
100.0 |
100.0 |
100.0 |
| BN_12 |
0 |
99.0 |
97.5 |
0.0 |
| BN_13 |
0 |
99.5 |
98.7 |
0.0 |
| BN_15 |
0 |
99.7 |
97.7 |
99.4 |
| BN_31 |
0 |
100.0 |
87.6 |
99.3 |
| blockmap_10_01-0009 |
0 |
0.0 |
0.0 |
0.0 |
| blockmap_10_02-0009 |
0 |
0.0 |
0.0 |
0.0 |
| blockmap_10_03-0009 |
0 |
0.0 |
0.0 |
0.0 |
| blockmap_10_03-0010 |
0 |
0.0 |
0.0 |
0.0 |
| blockmap_15_01-0008 |
0 |
0.0 |
0.0 |
0.0 |
| blockmap_15_02-0008 |
0 |
0.0 |
0.0 |
0.0 |
| blockmap_15_03-0010 |
0 |
0.0 |
0.0 |
0.0 |
| blockmap_20_01-0009 |
0 |
0.0 |
0.0 |
0.0 |
| bwt3ac.wcsp |
0 |
97.0 |
47.2 |
0.0 |
| fs-07 |
0 |
98.5 |
0.0 |
0.0 |
| mastermind_03_08_04-0000 |
0 |
96.2 |
57.6 |
74.7 |
| mastermind_03_08_04-0001 |
0 |
67.9 |
87.6 |
99.1 |
| mastermind_03_08_04-0002 |
0 |
87.3 |
80.0 |
99.6 |
| mastermind_03_08_04-0003 |
0 |
63.1 |
75.8 |
98.2 |
| mastermind_03_08_04-0007 |
0 |
100.0 |
94.4 |
0.0 |
| mastermind_03_08_04-0008 |
0 |
100.0 |
87.7 |
0.0 |
| mastermind_03_08_04-0010 |
0 |
100.0 |
81.5 |
0.0 |
| mastermind_03_08_04-0012 |
0 |
0.0 |
0.0 |
55.3 |
| mastermind_03_08_04-0013 |
0 |
0.0 |
2.5 |
63.2 |
| mastermind_03_08_04-0014 |
0 |
0.0 |
13.0 |
56.0 |
| mastermind_03_08_04-0015 |
0 |
0.0 |
21.0 |
64.7 |
| mastermind_03_08_05-0001 |
0 |
0.0 |
86.6 |
0.0 |
| mastermind_03_08_05-0003 |
0 |
0.0 |
92.2 |
0.0 |
| mastermind_03_08_05-0009 |
0 |
0.0 |
74.5 |
55.2 |
| mastermind_04_08_03-0000 |
0 |
0.0 |
71.9 |
99.6 |
| mastermind_04_08_03-0011 |
0 |
100.0 |
37.9 |
47.0 |
| mastermind_04_08_03-0012 |
0 |
100.0 |
46.2 |
62.6 |
| mastermind_04_08_03-0013 |
0 |
100.0 |
47.4 |
62.2 |
| mastermind_04_08_03-0014 |
0 |
100.0 |
18.8 |
34.3 |
| mastermind_04_08_03-0015 |
0 |
100.0 |
7.8 |
41.4 |
| mastermind_04_08_04-0001 |
0 |
73.5 |
85.0 |
98.1 |
| mastermind_04_08_04-0002 |
0 |
100.0 |
62.7 |
75.6 |
| mastermind_04_08_04-0003 |
0 |
100.0 |
69.5 |
77.7 |
| mastermind_04_08_04-0004 |
0 |
100.0 |
63.3 |
75.0 |
| mastermind_04_08_04-0005 |
0 |
83.4 |
89.7 |
99.0 |
| mastermind_05_08_03-0001 |
0 |
84.3 |
79.2 |
99.6 |
| mastermind_05_08_03-0002 |
0 |
82.0 |
72.4 |
99.4 |
| mastermind_05_08_03-0003 |
0 |
82.1 |
67.2 |
99.2 |
| mastermind_05_08_03-0004 |
0 |
88.5 |
82.5 |
99.6 |
| mastermind_05_08_03-0009 |
0 |
100.0 |
57.4 |
0.0 |
| mastermind_05_08_03-0011 |
0 |
0.0 |
42.5 |
74.8 |
| mastermind_05_08_03-0012 |
0 |
0.0 |
64.9 |
73.8 |
| mastermind_05_08_03-0013 |
0 |
0.0 |
53.9 |
67.5 |
| mastermind_05_08_03-0015 |
0 |
0.0 |
27.8 |
0.0 |
| mastermind_06_08_03-0002 |
0 |
86.6 |
69.2 |
99.4 |
| mastermind_06_08_03-0003 |
0 |
89.1 |
79.8 |
99.8 |
| mastermind_06_08_03-0005 |
0 |
100.0 |
79.3 |
100.0 |
| mastermind_06_08_03-0009 |
0 |
100.0 |
58.8 |
0.0 |
| mastermind_10_08_03-0008 |
0 |
67.8 |
30.5 |
30.5 |
| mastermind_10_08_03-0009 |
0 |
84.5 |
77.7 |
77.6 |
| myciel5g_3.wcsp |
0 |
88.9 |
100.0 |
100.0 |
| or_chain_102.fg |
0 |
0.0 |
10.3 |
13.3 |
| or_chain_106.fg |
0 |
0.0 |
91.2 |
88.8 |
| or_chain_107.fg |
0 |
0.0 |
68.2 |
57.2 |
| or_chain_12.fg |
0 |
0.0 |
7.0 |
17.7 |
| or_chain_128.fg |
0 |
0.0 |
71.4 |
45.5 |
| or_chain_132.fg |
0 |
0.0 |
37.0 |
97.5 |
| or_chain_138.fg |
0 |
0.0 |
95.7 |
96.1 |
| or_chain_140.fg |
0 |
0.0 |
0.0 |
66.6 |
| or_chain_149.fg |
0 |
0.0 |
97.6 |
97.9 |
| or_chain_15.fg |
0 |
0.0 |
91.6 |
84.7 |
| or_chain_150.fg |
0 |
0.0 |
37.1 |
50.1 |
| or_chain_153.fg |
0 |
0.0 |
0.0 |
12.6 |
| or_chain_155.fg |
0 |
0.0 |
93.1 |
93.9 |
| or_chain_161.fg |
0 |
0.0 |
46.8 |
32.2 |
| or_chain_186.fg |
0 |
0.0 |
84.1 |
67.2 |
| or_chain_188.fg |
0 |
0.0 |
45.3 |
71.8 |
| or_chain_198.fg |
0 |
0.0 |
99.2 |
99.2 |
| or_chain_209.fg |
0 |
0.0 |
74.5 |
87.5 |
| or_chain_242.fg |
0 |
0.0 |
56.9 |
58.0 |
| or_chain_4.fg |
0 |
0.0 |
0.1 |
24.3 |
| or_chain_53.fg |
0 |
0.0 |
99.7 |
99.7 |
| or_chain_61.fg |
0 |
0.0 |
82.2 |
82.9 |
| or_chain_64.fg |
0 |
0.0 |
0.0 |
0.0 |
| or_chain_90.fg |
0 |
0.0 |
45.6 |
35.9 |
| queen5_5_3.wcsp |
0 |
88.9 |
100.0 |
100.0 |
1200sec
| Problem |
baseline |
ibia-mar |
lbp-mar |
uai14-mar |
| 1aie |
0 |
100.0 |
100.0 |
100.0 |
| 1bxv |
0 |
100.0 |
100.0 |
100.0 |
| 1g6x |
0 |
100.0 |
100.0 |
100.0 |
| 1jer |
0 |
100.0 |
100.0 |
100.0 |
| 1npl |
0 |
100.0 |
100.0 |
100.0 |
| 1qt9 |
0 |
100.0 |
100.0 |
99.9 |
| 1rfs |
0 |
100.0 |
100.0 |
100.0 |
| 2rta |
0 |
100.0 |
100.0 |
100.0 |
| BN_12 |
0 |
100.0 |
97.5 |
99.6 |
| BN_13 |
0 |
99.7 |
98.7 |
99.7 |
| BN_15 |
0 |
100.0 |
97.7 |
0.0 |
| BN_31 |
0 |
100.0 |
87.6 |
99.9 |
| blockmap_10_01-0009 |
0 |
100.0 |
0.0 |
0.0 |
| blockmap_10_02-0009 |
0 |
100.0 |
0.0 |
0.0 |
| blockmap_10_03-0009 |
0 |
100.0 |
0.0 |
0.0 |
| blockmap_10_03-0010 |
0 |
100.0 |
0.0 |
0.0 |
| blockmap_15_01-0008 |
0 |
100.0 |
0.0 |
0.0 |
| blockmap_15_02-0008 |
0 |
100.0 |
0.0 |
100.0 |
| blockmap_15_03-0010 |
0 |
100.0 |
0.0 |
100.0 |
| blockmap_20_01-0009 |
0 |
0.0 |
0.0 |
99.8 |
| bwt3ac.wcsp |
0 |
99.9 |
47.2 |
72.0 |
| fs-07 |
0 |
98.3 |
0.0 |
100.0 |
| mastermind_03_08_04-0000 |
0 |
97.9 |
57.6 |
98.9 |
| mastermind_03_08_04-0001 |
0 |
67.9 |
87.6 |
99.2 |
| mastermind_03_08_04-0002 |
0 |
87.3 |
80.0 |
99.6 |
| mastermind_03_08_04-0003 |
0 |
63.1 |
75.8 |
98.1 |
| mastermind_03_08_04-0007 |
0 |
100.0 |
94.4 |
100.0 |
| mastermind_03_08_04-0008 |
0 |
100.0 |
87.8 |
100.0 |
| mastermind_03_08_04-0010 |
0 |
100.0 |
81.5 |
0.0 |
| mastermind_03_08_04-0012 |
0 |
90.2 |
0.0 |
42.3 |
| mastermind_03_08_04-0013 |
0 |
91.3 |
1.9 |
14.0 |
| mastermind_03_08_04-0014 |
0 |
87.2 |
9.1 |
42.2 |
| mastermind_03_08_04-0015 |
0 |
92.8 |
6.2 |
88.7 |
| mastermind_03_08_05-0001 |
0 |
77.9 |
86.6 |
98.5 |
| mastermind_03_08_05-0003 |
0 |
81.7 |
92.2 |
99.4 |
| mastermind_03_08_05-0009 |
0 |
100.0 |
74.5 |
0.0 |
| mastermind_04_08_03-0000 |
0 |
86.9 |
71.9 |
99.3 |
| mastermind_04_08_03-0011 |
0 |
100.0 |
37.9 |
100.0 |
| mastermind_04_08_03-0012 |
0 |
100.0 |
46.8 |
100.0 |
| mastermind_04_08_03-0013 |
0 |
100.0 |
47.6 |
100.0 |
| mastermind_04_08_03-0014 |
0 |
100.0 |
18.8 |
100.0 |
| mastermind_04_08_03-0015 |
0 |
100.0 |
7.8 |
100.0 |
| mastermind_04_08_04-0001 |
0 |
73.5 |
85.0 |
97.7 |
| mastermind_04_08_04-0002 |
0 |
100.0 |
62.7 |
99.5 |
| mastermind_04_08_04-0003 |
0 |
100.0 |
69.5 |
99.6 |
| mastermind_04_08_04-0004 |
0 |
100.0 |
63.3 |
99.6 |
| mastermind_04_08_04-0005 |
0 |
83.4 |
89.7 |
99.1 |
| mastermind_05_08_03-0001 |
0 |
84.3 |
79.2 |
99.8 |
| mastermind_05_08_03-0002 |
0 |
82.0 |
72.4 |
99.5 |
| mastermind_05_08_03-0003 |
0 |
82.1 |
67.2 |
99.0 |
| mastermind_05_08_03-0004 |
0 |
88.5 |
82.5 |
99.6 |
| mastermind_05_08_03-0009 |
0 |
100.0 |
57.4 |
73.5 |
| mastermind_05_08_03-0011 |
0 |
100.0 |
42.8 |
97.3 |
| mastermind_05_08_03-0012 |
0 |
100.0 |
64.9 |
97.6 |
| mastermind_05_08_03-0013 |
0 |
100.0 |
53.8 |
94.1 |
| mastermind_05_08_03-0015 |
0 |
100.0 |
28.0 |
97.6 |
| mastermind_06_08_03-0002 |
0 |
86.6 |
69.2 |
99.3 |
| mastermind_06_08_03-0003 |
0 |
89.1 |
79.8 |
99.7 |
| mastermind_06_08_03-0005 |
0 |
100.0 |
79.3 |
100.0 |
| mastermind_06_08_03-0009 |
0 |
100.0 |
58.8 |
72.7 |
| mastermind_10_08_03-0008 |
0 |
67.8 |
30.5 |
0.0 |
| mastermind_10_08_03-0009 |
0 |
84.5 |
77.7 |
77.7 |
| myciel5g_3.wcsp |
0 |
88.9 |
100.0 |
100.0 |
| or_chain_102.fg |
0 |
76.4 |
10.3 |
54.5 |
| or_chain_106.fg |
0 |
44.4 |
91.2 |
72.5 |
| or_chain_107.fg |
0 |
35.5 |
68.2 |
55.7 |
| or_chain_12.fg |
0 |
0.0 |
7.0 |
90.2 |
| or_chain_128.fg |
0 |
0.0 |
71.4 |
77.3 |
| or_chain_132.fg |
0 |
42.1 |
37.0 |
97.6 |
| or_chain_138.fg |
0 |
47.1 |
95.7 |
99.2 |
| or_chain_140.fg |
0 |
0.0 |
0.0 |
0.0 |
| or_chain_149.fg |
0 |
86.2 |
97.6 |
100.0 |
| or_chain_15.fg |
0 |
65.4 |
91.6 |
85.0 |
| or_chain_150.fg |
0 |
55.6 |
37.1 |
54.2 |
| or_chain_153.fg |
0 |
67.4 |
0.0 |
89.2 |
| or_chain_155.fg |
0 |
85.5 |
93.1 |
93.9 |
| or_chain_161.fg |
0 |
0.0 |
46.8 |
76.4 |
| or_chain_186.fg |
0 |
65.2 |
84.1 |
63.6 |
| or_chain_188.fg |
0 |
0.0 |
45.3 |
100.0 |
| or_chain_198.fg |
0 |
38.6 |
99.2 |
99.2 |
| or_chain_209.fg |
0 |
46.8 |
74.5 |
99.9 |
| or_chain_242.fg |
0 |
31.6 |
56.9 |
78.5 |
| or_chain_4.fg |
0 |
72.7 |
0.1 |
57.0 |
| or_chain_53.fg |
0 |
48.2 |
99.7 |
99.7 |
| or_chain_61.fg |
0 |
0.0 |
82.2 |
86.6 |
| or_chain_64.fg |
0 |
0.0 |
0.0 |
96.9 |
| or_chain_90.fg |
0 |
47.6 |
45.6 |
100.0 |
| queen5_5_3.wcsp |
0 |
88.9 |
100.0 |
100.0 |
3600sec
| Problem |
baseline |
ibia-mar |
lbp-mar |
uai14-mar |
| 1aie |
0 |
100.0 |
100.0 |
100.0 |
| 1bxv |
0 |
100.0 |
100.0 |
100.0 |
| 1g6x |
0 |
100.0 |
100.0 |
100.0 |
| 1jer |
0 |
100.0 |
100.0 |
100.0 |
| 1npl |
0 |
100.0 |
100.0 |
100.0 |
| 1qt9 |
0 |
100.0 |
100.0 |
99.9 |
| 1rfs |
0 |
100.0 |
100.0 |
100.0 |
| 2rta |
0 |
100.0 |
100.0 |
100.0 |
| BN_12 |
0 |
100.0 |
97.5 |
99.6 |
| BN_13 |
0 |
99.7 |
98.7 |
99.7 |
| BN_15 |
0 |
100.0 |
97.7 |
0.0 |
| BN_31 |
0 |
100.0 |
87.6 |
100.0 |
| blockmap_10_01-0009 |
0 |
100.0 |
0.0 |
0.0 |
| blockmap_10_02-0009 |
0 |
100.0 |
0.0 |
0.0 |
| blockmap_10_03-0009 |
0 |
100.0 |
0.0 |
100.0 |
| blockmap_10_03-0010 |
0 |
100.0 |
0.0 |
0.0 |
| blockmap_15_01-0008 |
0 |
100.0 |
0.0 |
0.0 |
| blockmap_15_02-0008 |
0 |
100.0 |
0.0 |
100.0 |
| blockmap_15_03-0010 |
0 |
100.0 |
0.0 |
100.0 |
| blockmap_20_01-0009 |
0 |
0.0 |
0.0 |
99.8 |
| bwt3ac.wcsp |
0 |
99.9 |
47.2 |
64.0 |
| fs-07 |
0 |
98.3 |
0.0 |
100.0 |
| mastermind_03_08_04-0000 |
0 |
97.9 |
57.6 |
99.8 |
| mastermind_03_08_04-0001 |
0 |
67.9 |
87.6 |
99.2 |
| mastermind_03_08_04-0002 |
0 |
87.3 |
80.0 |
99.6 |
| mastermind_03_08_04-0003 |
0 |
63.1 |
75.8 |
98.1 |
| mastermind_03_08_04-0007 |
0 |
100.0 |
94.4 |
100.0 |
| mastermind_03_08_04-0008 |
0 |
100.0 |
87.8 |
100.0 |
| mastermind_03_08_04-0010 |
0 |
100.0 |
81.5 |
0.0 |
| mastermind_03_08_04-0012 |
0 |
90.2 |
0.0 |
61.5 |
| mastermind_03_08_04-0013 |
0 |
91.3 |
1.9 |
74.3 |
| mastermind_03_08_04-0014 |
0 |
87.2 |
9.1 |
66.0 |
| mastermind_03_08_04-0015 |
0 |
92.8 |
6.2 |
95.4 |
| mastermind_03_08_05-0001 |
0 |
77.9 |
86.6 |
78.1 |
| mastermind_03_08_05-0003 |
0 |
81.7 |
92.2 |
81.7 |
| mastermind_03_08_05-0009 |
0 |
100.0 |
74.5 |
0.0 |
| mastermind_04_08_03-0000 |
0 |
86.9 |
71.9 |
99.3 |
| mastermind_04_08_03-0011 |
0 |
100.0 |
37.9 |
100.0 |
| mastermind_04_08_03-0012 |
0 |
100.0 |
46.8 |
100.0 |
| mastermind_04_08_03-0013 |
0 |
100.0 |
47.6 |
100.0 |
| mastermind_04_08_03-0014 |
0 |
100.0 |
18.8 |
100.0 |
| mastermind_04_08_03-0015 |
0 |
100.0 |
7.8 |
100.0 |
| mastermind_04_08_04-0001 |
0 |
73.5 |
85.0 |
74.3 |
| mastermind_04_08_04-0002 |
0 |
100.0 |
62.7 |
68.3 |
| mastermind_04_08_04-0003 |
0 |
100.0 |
69.5 |
99.9 |
| mastermind_04_08_04-0004 |
0 |
100.0 |
63.3 |
99.9 |
| mastermind_04_08_04-0005 |
0 |
83.4 |
89.7 |
83.4 |
| mastermind_05_08_03-0001 |
0 |
84.3 |
79.2 |
99.9 |
| mastermind_05_08_03-0002 |
0 |
82.0 |
72.4 |
99.5 |
| mastermind_05_08_03-0003 |
0 |
82.1 |
67.2 |
99.0 |
| mastermind_05_08_03-0004 |
0 |
88.5 |
82.5 |
99.6 |
| mastermind_05_08_03-0009 |
0 |
100.0 |
57.4 |
73.5 |
| mastermind_05_08_03-0011 |
0 |
100.0 |
42.8 |
54.6 |
| mastermind_05_08_03-0012 |
0 |
100.0 |
64.9 |
99.5 |
| mastermind_05_08_03-0013 |
0 |
100.0 |
53.8 |
80.6 |
| mastermind_05_08_03-0015 |
0 |
100.0 |
28.0 |
59.3 |
| mastermind_06_08_03-0002 |
0 |
86.6 |
69.2 |
86.6 |
| mastermind_06_08_03-0003 |
0 |
89.1 |
79.8 |
89.1 |
| mastermind_06_08_03-0005 |
0 |
100.0 |
79.3 |
100.0 |
| mastermind_06_08_03-0009 |
0 |
100.0 |
58.8 |
73.8 |
| mastermind_10_08_03-0008 |
0 |
67.8 |
30.5 |
0.0 |
| mastermind_10_08_03-0009 |
0 |
84.5 |
77.7 |
77.7 |
| myciel5g_3.wcsp |
0 |
88.9 |
100.0 |
100.0 |
| or_chain_102.fg |
0 |
76.4 |
10.3 |
64.2 |
| or_chain_106.fg |
0 |
77.1 |
91.2 |
92.8 |
| or_chain_107.fg |
0 |
35.5 |
68.2 |
63.6 |
| or_chain_12.fg |
0 |
0.0 |
7.0 |
100.0 |
| or_chain_128.fg |
0 |
0.0 |
71.4 |
94.7 |
| or_chain_132.fg |
0 |
42.1 |
37.0 |
99.1 |
| or_chain_138.fg |
0 |
47.1 |
95.7 |
99.9 |
| or_chain_140.fg |
0 |
0.0 |
0.0 |
5.0 |
| or_chain_149.fg |
0 |
86.2 |
97.6 |
97.9 |
| or_chain_15.fg |
0 |
65.4 |
91.6 |
59.2 |
| or_chain_150.fg |
0 |
55.6 |
37.1 |
55.3 |
| or_chain_153.fg |
0 |
67.4 |
0.0 |
100.0 |
| or_chain_155.fg |
0 |
85.5 |
93.1 |
98.7 |
| or_chain_161.fg |
0 |
24.9 |
46.8 |
97.7 |
| or_chain_186.fg |
0 |
65.2 |
84.1 |
100.0 |
| or_chain_188.fg |
0 |
0.0 |
45.3 |
100.0 |
| or_chain_198.fg |
0 |
40.0 |
99.2 |
99.6 |
| or_chain_209.fg |
0 |
46.8 |
74.5 |
99.9 |
| or_chain_242.fg |
0 |
22.3 |
56.9 |
95.5 |
| or_chain_4.fg |
0 |
72.7 |
0.1 |
61.2 |
| or_chain_53.fg |
0 |
39.3 |
99.7 |
99.7 |
| or_chain_61.fg |
0 |
0.0 |
82.2 |
84.5 |
| or_chain_64.fg |
0 |
0.0 |
0.0 |
100.0 |
| or_chain_90.fg |
0 |
47.6 |
45.6 |
100.0 |
| queen5_5_3.wcsp |
0 |
88.9 |
100.0 |
100.0 |