From 020b04360e05edee63154c3aea2232379948a0b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lennart=20Sch=C3=A4permeier?= Date: Sat, 7 Feb 2026 15:47:57 +0100 Subject: [PATCH] Implement exact R2 indicator for bi-objective problems Squash of the following 30 commits: Add files for exact R2 indicator Basic bi-objective R2 indicator implementation Skip points not dominated by ideal ref and remove logging Provide basic documentation for exact R2 Add as contributor Add exact_r2 as news [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Remove empty section to make pre-commit-ci happy Rename exact_r2 -> r2_exact [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Optimizations in r2_exact computation Fix citation Skip another unnecessary check Improve edgecase handling Inline subfunction call [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Add functional tests for r2_exact [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Remove unused object Add exact R2 to metrics example [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Remove additional space (which messed up the layout) Use correct r2_exact function name Fix docstring formatting in test_r2_from_doc Update reference argument in r2_exact examples Fix precision of R2 exact function output Refactor _utility function and variable declarations [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Add mathematical description of R2 Remove () Add original source of R2 indicator --- README.md | 7 +- bibkeys.txt | 2 + c/Makefile | 2 + c/r2_exact.c | 94 ++++++++++++++++ c/r2_exact.h | 31 ++++++ python/README.md | 3 +- python/doc/source/REFERENCES.bib | 23 ++++ python/doc/source/index.rst | 2 +- .../source/reference/functions.metrics.rst | 40 +++++++ python/doc/source/whatsnew/index.rst | 2 +- python/examples/plot_metrics.py | 13 ++- python/pyproject.toml | 1 + python/src/moocore/__init__.py | 2 + python/src/moocore/_ffi_build.py | 2 + python/src/moocore/_moocore.py | 105 ++++++++++++++++++ python/src/moocore/libmoocore.h | 2 + python/tests/test_moocore.py | 49 ++++++++ r/inst/REFERENCES.bib | 23 ++++ 18 files changed, 392 insertions(+), 11 deletions(-) create mode 100644 c/r2_exact.c create mode 100644 c/r2_exact.h diff --git a/README.md b/README.md index c9f05d56c..4ccc84d2d 100644 --- a/README.md +++ b/README.md @@ -15,10 +15,11 @@ [Manuel López-Ibáñez](https://lopez-ibanez.eu), [Carlos M. Fonseca](https://eden.dei.uc.pt/~cmfonsec/), [Luís Paquete](https://eden.dei.uc.pt/~paquete/), - Andreia P. Guerreiro - Mickaël Binois. + Andreia P. Guerreiro, + Mickaël Binois, Leonardo C.T. Bezerra, - Fergus Rooney. + Fergus Rooney, + [Lennart Schäpermeier](https://schaepermeier.github.io). --------------------------------------- diff --git a/bibkeys.txt b/bibkeys.txt index 62b82f152..9a94163be 100644 --- a/bibkeys.txt +++ b/bibkeys.txt @@ -17,6 +17,7 @@ GruFon2009:emaa Grunert01 GueFon2017hv4d GueFonPaq2021hv +HanJas1998 HerWer1987tabucol IshMasTanNoj2015igd Jen03 @@ -31,6 +32,7 @@ Molchanov2005theory Muller1959sphere RubMel1998simulation SchEsqLarCoe2012tec +SchKer2025r2v2 VelLam1998gp WuAza2001metrics ZhoZhaJin2009igdx diff --git a/c/Makefile b/c/Makefile index d29a0e63d..caf154304 100644 --- a/c/Makefile +++ b/c/Makefile @@ -62,6 +62,7 @@ SRCS = avl.c \ eafdiff.c \ eaf_main.c \ epsilon.c \ + r2_exact.c \ hv3dplus.c \ hv4d.c \ hv.c \ @@ -91,6 +92,7 @@ HDRS = avl.h \ cvector.h \ eaf.h \ epsilon.h \ + r2_exact.h \ gcc_attribs.h \ hv.h \ hvapprox.h \ diff --git a/c/r2_exact.c b/c/r2_exact.c new file mode 100644 index 000000000..a52fecd01 --- /dev/null +++ b/c/r2_exact.c @@ -0,0 +1,94 @@ +#include +#include +#include +#include "common.h" +#include "sort.h" + +// Computes (two times) the utility of axis-parallel segment between y1, y2, and y2p. +static double _utility(const double y1, const double y2, const double y2p) +{ + if (y1 == 0) { + return 0; + } + + double w = y2 / (y1 + y2); + double wp; + if (y2p == DBL_MAX) { + wp = 1; + } else { + wp = y2p / (y1 + y2p); + } + + return y1 * (wp * wp - w * w); +} + +double r2_exact(const double * restrict data, size_t n, dimension_t dim, + const double * restrict ref) +{ + if (unlikely(n == 0)) return -1; + + ASSUME(dim == 2); + + // p is sorted by f1 (primarily), then f2 (secondarily) + const double **p = generate_sorted_doublep_2d(data, &n, DBL_MAX); + + if (unlikely(!p)) return -1; + + size_t j = 0; + + // skip points not dominated by "ideal" ref point + while (j < n && p[j][0] < ref[0]) { + j++; + } + + // no points to evaluate + if (unlikely(j == n)) { + if (p[j - 1][1] <= ref[1]) return 0.0; // ideal ref. is dominated. + return DBL_MAX; // ideal ref. is nondominated --> worst possible value + } + + double prev_y1 = p[j][0] - ref[0]; + double prev_y2 = p[j][1] - ref[1]; + + if (prev_y2 < 0) { + // ideal ref. is dominated. + return 0.0; + } + + // first element + double r2_exact = _utility(prev_y1, prev_y2, DBL_MAX); + // printf("y2 segment: %f, %f, MAX: %f\n", p[j][0] - ref[0], p[j][1] - ref[1], _utility(p[j][0] - ref[0], p[j][1] - ref[1], DBL_MAX)); + + while(j < n - 1) { + j++; + double y1 = p[j][0] - ref[0]; + double y2 = p[j][1] - ref[1]; + + // skip anything that's not dominated by ref + if (y2 < 0) continue; + + // printf("%f\n", y1); + // printf("%f\n", y2); + + if (y2 < prev_y2) { + // pj not dominated by previous non-dominated pj + r2_exact += _utility(prev_y2, prev_y1, y1) + _utility(y1, y2, prev_y2); + + // printf("y1 segment: %f, %f, %f: %f\n", prev_y2, prev_y1, y1, _utility(prev_y2, prev_y1, y1)); + // printf("y2 segment: %f, %f, %f: %f\n", y1, y2, prev_y2, _utility(y1, y2, prev_y2)); + + prev_y1 = y1; + prev_y2 = y2; + } + } + + // last element + r2_exact += _utility(prev_y2, prev_y1, DBL_MAX); + // printf("y1 segment: %f, %f, MAX: %f\n", prev_y2, prev_y1, _utility(prev_y2, prev_y1, DBL_MAX)); + + // we omitted a 1/2 in the computation thus far: + r2_exact = 0.5 * r2_exact; + + free(p); + return r2_exact; +} diff --git a/c/r2_exact.h b/c/r2_exact.h new file mode 100644 index 000000000..ff3f8d697 --- /dev/null +++ b/c/r2_exact.h @@ -0,0 +1,31 @@ +/************************************************************************* + + r2_exact.h + + --------------------------------------------------------------------- + + Copyright (C) 2026 + Lennart Schäpermeier + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at https://mozilla.org/MPL/2.0/. + + ---------------------------------------------------------------------- + + +*************************************************************************/ +#ifndef R2_EXACT_H_ +#define R2_EXACT_H_ + +#include +#include "libmoocore-config.h" + +// C++ needs to know that types and declarations are C, not C++. +BEGIN_C_DECLS + +MOOCORE_API double r2_exact(const double * restrict data, size_t n, dimension_t d, const double * restrict ref); + +END_C_DECLS + +#endif // R2_EXACT_H_ diff --git a/python/README.md b/python/README.md index c15963df7..cec7da625 100644 --- a/python/README.md +++ b/python/README.md @@ -14,7 +14,8 @@ **Contributors:** [Manuel López-Ibáñez](https://lopez-ibanez.eu), - Fergus Rooney. + Fergus Rooney, + [Lennart Schäpermeier](https://schaepermeier.github.io). --------------------------------------- diff --git a/python/doc/source/REFERENCES.bib b/python/doc/source/REFERENCES.bib index 8398ff5cb..abbaf25ea 100644 --- a/python/doc/source/REFERENCES.bib +++ b/python/doc/source/REFERENCES.bib @@ -361,6 +361,17 @@ @article{SchEsqLarCoe2012tec doi = {10.1109/TEVC.2011.2161872} } +@article{SchKer2025r2v2, + author = {Sch{\"a}permeier, Lennart and Pascal Kerschke }, + title = {{R2} v2: The {Pareto}-compliant {R2} Indicator for Better + Benchmarking in Bi-objective Optimization}, + journal = {Evolutionary Computation}, + year = 2025, + pages = {1--17}, + month = oct, + doi = {10.1162/evco.a.366} +} + @article{WuAza2001metrics, author = {J. Wu and S. Azam}, title = {Metrics for Quality Assessment of a Multiobjective Design @@ -604,6 +615,18 @@ @incollection{Grunert01 paper.} } +@techreport{HanJas1998, + author = { Michael Pilegaard Hansen and Andrzej Jaszkiewicz }, + title = {Evaluating the quality of approximations to the non-dominated + set}, + institution = {Institute of Mathematical Modelling, Technical University of + Denmark}, + year = 1998, + number = {IMM-REP-1998-7}, + address = {Lyngby, Denmark}, + annote = {Proposed R2 indicator} +} + @incollection{IshMasTanNoj2015igd, booktitle = { Evolutionary Multi-criterion Optimization, EMO 2015 Part {I}}, address = { Heidelberg, Germany}, diff --git a/python/doc/source/index.rst b/python/doc/source/index.rst index 12c517ce2..9e5c57014 100644 --- a/python/doc/source/index.rst +++ b/python/doc/source/index.rst @@ -28,7 +28,7 @@ The goal of the **moocore** project (https://github.com/multi-objective/moocore/ * :ref:`Generate and transform nondominated sets `. * :ref:`Identify and filter dominated vectors `. -* :ref:`Quality metrics ` such as (weighted) hypervolume, epsilon, IGD+, etc. +* :ref:`Quality metrics ` such as (weighted) hypervolume, epsilon, IGD+, (exact) R2, etc. * :ref:`Computation of the Empirical Attainment Function `. The empirical attainment function (EAF) describes the probabilistic distribution of the outcomes obtained by a stochastic algorithm in the objective space. Most critical functionality is implemented in C, with the R and Python packages providing convenient interfaces to the C code. diff --git a/python/doc/source/reference/functions.metrics.rst b/python/doc/source/reference/functions.metrics.rst index 1a0f102c0..ebb28b1f2 100644 --- a/python/doc/source/reference/functions.metrics.rst +++ b/python/doc/source/reference/functions.metrics.rst @@ -190,6 +190,46 @@ Approximating the hypervolume metric Computing the hypervolume can be time consuming, thus several approaches have been proposed in the literature to approximate its value via Monte-Carlo or quasi-Monte-Carlo sampling :cite:p:`DenZha2019approxhv`. These methods are implemented in :func:`whv_hype` and :func:`hv_approx`. +.. _r2_indicator: + +(Exact) R2 Indicator +============================================================================= + +.. autosummary:: + :toctree: generated/ + + r2_exact + +The unary R2 indicator is a quality indicator for a set :math:`A \subset \mathbb{R}^m` +w.r.t. an ideal or utopian reference point :math:`\vec{r} \in \mathbb{R}^m`. +It was originally proposed by :cite:t:`HanJas1998` and is defined as the +expected Tchebycheff utility under a uniform distribution of weight vectors +(w.l.o.g. assuming minimization): + +.. math:: + R2(A) := \int_{w \in W} \min_{a \in A} \left\{ \max_{i=1,\dots,m} w_i (a_i - r_i) \right\} \, dw, + +where :math:`W` denotes the uniform distribution across weights: + +.. math:: + W = \{w \in \mathbb R^m \mid w_i \geq 0, \sum_{i=1}^m w_i = 1\}. + +The R2 indicator is to be minimized and has an optimal value of 0 when +:math:`\vec{r} \in A`. + +The exact R2 indicator is strongly Pareto-compliant, i.e., compatible with Pareto-optimality: + +.. math:: + \forall A, B \subset \mathbb R^m: A \prec B \Rightarrow R2(A) < R2(B). + +Given an ideal or utopian reference ponint, which is available in most scenarios, +all non-dominated solutions always contribute to the value of the exact R2 indicator. +However, it is scale-dependent and care should be taken such that all objectives +contribute approximately equally to the indicator, e.g., by normalizing the +Pareto front to the unit hypercube. + +:func:`r2_exact` computes the exact R2 indicator for bi-objective solution +sets in :math:`\mathcal O(N \log N)` following :cite:t:`SchKer2025r2v2`. Bibliography ============ diff --git a/python/doc/source/whatsnew/index.rst b/python/doc/source/whatsnew/index.rst index 3ed6d8fb3..4980549a5 100644 --- a/python/doc/source/whatsnew/index.rst +++ b/python/doc/source/whatsnew/index.rst @@ -8,7 +8,7 @@ Version 0.3.0 (development) --------------------------- - `moarchiving `_ is now included in the benchmarks. - +- :func:`~moocore.r2_exact` implements the exact computation of the R2 indicator for bi-objective solution sets. Version 0.2.0 (10/01/2026) --------------------------- diff --git a/python/examples/plot_metrics.py b/python/examples/plot_metrics.py index 8c4bced2e..3e98633a0 100644 --- a/python/examples/plot_metrics.py +++ b/python/examples/plot_metrics.py @@ -98,10 +98,11 @@ # %% # # However, both :func:`moocore.igd` and :func:`moocore.avg_hausdorff_dist` -# incorrectly measure B as better than A, whereas :func:`moocore.igd_plus` and -# :func:`moocore.hypervolume` correctly measure A as better than B (remember -# that hypervolume must be maximized) and :func:`moocore.epsilon_additive` measures -# both as equally good (epsilon is weakly Pareto compliant). +# incorrectly measure B as better than A, whereas :func:`moocore.igd_plus`, +# :func:`moocore.r2_exact` and :func:`moocore.hypervolume` correctly measure A +# as better than B (remember that hypervolume must be maximized) and +# :func:`moocore.epsilon_additive` measures both as equally good +# (epsilon is weakly Pareto compliant). # pd.DataFrame( @@ -112,6 +113,7 @@ moocore.igd_plus(A, ref), moocore.epsilon_additive(A, ref), moocore.hypervolume(A, ref=10), + moocore.r2_exact(A, ref=0), ], B=[ moocore.igd(B, ref), @@ -119,7 +121,8 @@ moocore.igd_plus(B, ref), moocore.epsilon_additive(B, ref), moocore.hypervolume(B, ref=10), + moocore.r2_exact(B, ref=0), ], ), - index=["IGD", "Hausdorff", "IGD+", "eps+", "HV"], + index=["IGD", "Hausdorff", "IGD+", "eps+", "HV", "Exact R2"], ) diff --git a/python/pyproject.toml b/python/pyproject.toml index 9a87d1ffb..ffa59d881 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -23,6 +23,7 @@ license-files = [ "COPYRIGHTS" ] authors = [ { name = "Manuel López-Ibáñez", email = "manuel.lopez-ibanez@manchester.ac.uk" }, { name = "Fergus Rooney", email = "fergus.rooney@outlook.com" }, + { name = "Lennart Schäpermeier", email = "lennart.schaepermeier@uni-muenster.de" }, ] requires-python = ">=3.10" classifiers = [ diff --git a/python/src/moocore/__init__.py b/python/src/moocore/__init__.py index 6c5aa8efe..393b416b8 100644 --- a/python/src/moocore/__init__.py +++ b/python/src/moocore/__init__.py @@ -10,6 +10,7 @@ eafdiff, epsilon_additive, epsilon_mult, + r2_exact, filter_dominated, filter_dominated_within_sets, generate_ndset, @@ -69,6 +70,7 @@ "largest_eafdiff", "normalise", "pareto_rank", + "r2_exact", "read_datasets", "total_whv_rect", "vorob_dev", diff --git a/python/src/moocore/_ffi_build.py b/python/src/moocore/_ffi_build.py index 4a80a055e..2b4bf98d1 100644 --- a/python/src/moocore/_ffi_build.py +++ b/python/src/moocore/_ffi_build.py @@ -24,6 +24,7 @@ #include "nondominated.h" #include "epsilon.h" #include "eaf.h" +#include "r2_exact.h" #include "whv.h" #include "whv_hype.h" #include "hvapprox.h" @@ -33,6 +34,7 @@ "eaf.c", "eaf3d.c", "eafdiff.c", + "r2_exact.c", "hv.c", "hvapprox.c", "hv3dplus.c", diff --git a/python/src/moocore/_moocore.py b/python/src/moocore/_moocore.py index 42e8ab4a9..b6a10d943 100644 --- a/python/src/moocore/_moocore.py +++ b/python/src/moocore/_moocore.py @@ -2733,3 +2733,108 @@ def apply_within_sets( if not shorter: res = res.take(np.concatenate(idx).argsort(), axis=0) return res + + +def r2_exact( + data: ArrayLike, + /, + ref: ArrayLike, + *, + maximise: bool | Sequence[bool] = False, +) -> float: + r"""Exact R2 indicator. + + Computes the exact R2 indicator with respect to a given ideal/utopian reference point + assuming minimization of all objectives. + + .. seealso:: For details of the R2 indicator, see :ref:`r2_indicator`. + + Parameters + ---------- + data : + Numpy array of numerical values, where each row gives the coordinates of a point. + If the array is created from the :func:`read_datasets` function, remove the last column. + ref : + Reference point as a 1D vector. Must be same length as a single point in the ``data``. + maximise : + Whether the objectives must be maximised instead of minimised. + Either a single boolean value that applies to all objectives or a list of booleans, with one value per objective. + Also accepts a 1D numpy array with value 0/1 for each objective. + + Returns + ------- + A single numerical value, the exact R2 indicator. + + Notes + ----- + The current implementation exclusively supports bi-objective solution sets and runs in :math:`O(n \log n)`. + For more details on the computation of the exact R2 indicator refer to :footcite:t:`SchKer2025r2v2`. + + References + ---------- + .. footbibliography:: + + Examples + -------- + >>> dat = np.array([[5, 5], [4, 6], [2, 7], [7, 4]]) + >>> moocore.r2_exact(dat, ref=[0, 0]) + 2.594191919191919 + + This function assumes that objectives must be minimized by default. We can + easily specify maximization: + + >>> dat = np.array([[5, 5], [4, 6], [2, 7], [7, 4]]) + >>> moocore.r2_exact(dat, ref=[10, 10], maximise=True) + 2.519696969696969 + + Merge all the sets of a dataset by removing the set number column: + + >>> dat = moocore.get_dataset("input1.dat")[:, :-1] + >>> len(dat) + 100 + + Dominated points are ignored, so this: + + >>> moocore.r2_exact(dat, ref=0) + 0.3336076878950565 + + gives the same exact R2 value as this: + + >>> dat = moocore.filter_dominated(dat) + >>> len(dat) + 6 + >>> moocore.r2_exact(dat, ref=0) + 0.3336076878950565 + + """ + # Convert to numpy.array in case the user provides a list. We use + # np.asarray to convert it to floating-point, otherwise if a user inputs + # something like ref = np.array([10, 10]) then numpy would interpret it as + # an int array. + data, data_copied = asarray_maybe_copy(data) + nobj = data.shape[1] + # Make sure it is a 1D array of length nobj. + ref = array_1d_of_length_n(np.asarray(ref, dtype=float), nobj) + if nobj != ref.shape[0]: + raise ValueError( + f"data and ref need to have the same number of objectives ({nobj} != {ref.shape[0]})" + ) + + maximise = _parse_maximise(maximise, nobj) + # FIXME: Do this in C. + if maximise.any(): + if not data_copied: + data = data.copy() + data[:, maximise] = -data[:, maximise] + ref = ref.copy() + ref[maximise] = -ref[maximise] + + data_p, npoints, nobj = np2d_to_double_array( + data, ctype_shape=("size_t", "uint_fast8_t") + ) + ref_buf = ffi.from_buffer("double []", ref) + r2 = lib.r2_exact(data_p, npoints, nobj, ref_buf) + # if r2 < 0: + # raise MemoryError("memory allocation failed") + + return r2 diff --git a/python/src/moocore/libmoocore.h b/python/src/moocore/libmoocore.h index a94a66586..24b762874 100644 --- a/python/src/moocore/libmoocore.h +++ b/python/src/moocore/libmoocore.h @@ -16,6 +16,8 @@ double avg_Hausdorff_dist(const double * restrict data, size_t n, dimension_t d, // epsilon.h double epsilon_additive(const double * restrict data, size_t n, dimension_t d, const double * restrict ref, size_t ref_size, const bool * restrict maximise); double epsilon_mult(const double * restrict data, size_t n, dimension_t d, const double * restrict ref, size_t ref_size, const bool * restrict maximise); +// r2_exact.h +double r2_exact(const double * restrict data, size_t n, dimension_t d, const double * restrict ref); // nondominated.h size_t find_weakly_dominated_point(const double * restrict points, size_t n, dimension_t d, diff --git a/python/tests/test_moocore.py b/python/tests/test_moocore.py index b233150c4..6d4329f8c 100644 --- a/python/tests/test_moocore.py +++ b/python/tests/test_moocore.py @@ -555,3 +555,52 @@ def test_nondominated_sorting_3d(): [False, False, False, False, True], ] check_nondominated_sorting_fronts(five_fronts, five_fronts_expected) + + +class TestR2Exact: + """Test exact R2 computation.""" + + def test_r2_output(self): + """Checks the exact R2 calculation produces the correct value for some known simple test cases.""" + # perfect approximation of ideal ref yields 0.0 + r2 = moocore.r2_exact([[0, 0]], [0, 0]) + assert_allclose(r2, 0.0) + + # [1,1] in normalized objective space should yield r2 = 0.75 + r2 = moocore.r2_exact([[2, 2]], [1, 1]) + assert_allclose(r2, 0.75) + + # [[1,0],[0,1]] should yield r2 = 0.25 + r2 = moocore.r2_exact([[0, 1], [1, 0]], [0, 0]) + assert_allclose(r2, 0.25) + + r2 = moocore.r2_exact( + [[0, 1], [0.2, 0.8], [0.4, 0.6], [0.6, 0.4], [0.8, 0.2], [1, 0]], + [0, 0], + ) + assert_allclose(r2, 0.1833333333333333) + + # a closely sampled linear front should yield a value close to 1/6 + x = np.linspace(0, 1, 1000001) + pf = list(zip(x, (1 - x))) + r2 = moocore.r2_exact(pf, ref=[0, 0]) + assert_allclose(r2, 1 / 6, atol=1e-6) + + def test_r2_from_doc(self): + """Checks that the exact R2 examples from the documentation work as expected.""" + dat = np.array([[5, 5], [4, 6], [2, 7], [7, 4]]) + r2 = moocore.r2_exact(dat, ref=[0, 0]) + assert_allclose(r2, 2.5941919191919194) + + r2 = moocore.r2_exact(dat, ref=[10, 10], maximise=True) + assert_allclose(r2, 2.5196969696969695) + + dat = moocore.get_dataset("input1.dat")[:, :-1] + r2 = moocore.r2_exact(dat, ref=[0, 0]) + + assert_allclose(r2, 0.33360768789505657) + + dat = moocore.filter_dominated(dat) + moocore.r2_exact(dat, ref=[10, 10]) + + assert_allclose(r2, 0.33360768789505657) diff --git a/r/inst/REFERENCES.bib b/r/inst/REFERENCES.bib index 8398ff5cb..abbaf25ea 100644 --- a/r/inst/REFERENCES.bib +++ b/r/inst/REFERENCES.bib @@ -361,6 +361,17 @@ @article{SchEsqLarCoe2012tec doi = {10.1109/TEVC.2011.2161872} } +@article{SchKer2025r2v2, + author = {Sch{\"a}permeier, Lennart and Pascal Kerschke }, + title = {{R2} v2: The {Pareto}-compliant {R2} Indicator for Better + Benchmarking in Bi-objective Optimization}, + journal = {Evolutionary Computation}, + year = 2025, + pages = {1--17}, + month = oct, + doi = {10.1162/evco.a.366} +} + @article{WuAza2001metrics, author = {J. Wu and S. Azam}, title = {Metrics for Quality Assessment of a Multiobjective Design @@ -604,6 +615,18 @@ @incollection{Grunert01 paper.} } +@techreport{HanJas1998, + author = { Michael Pilegaard Hansen and Andrzej Jaszkiewicz }, + title = {Evaluating the quality of approximations to the non-dominated + set}, + institution = {Institute of Mathematical Modelling, Technical University of + Denmark}, + year = 1998, + number = {IMM-REP-1998-7}, + address = {Lyngby, Denmark}, + annote = {Proposed R2 indicator} +} + @incollection{IshMasTanNoj2015igd, booktitle = { Evolutionary Multi-criterion Optimization, EMO 2015 Part {I}}, address = { Heidelberg, Germany},