Fix Hodges-Lehmann distribution ratio calculation #111
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| package com.atlassian.performance.tools.report.api.distribution | ||
|
|
||
| import com.numericalmethod.suanshu.stats.test.rank.wilcoxon.WilcoxonRankSum | ||
| import org.apache.commons.math3.stat.descriptive.rank.Median | ||
| import org.apache.commons.math3.stat.ranking.NaNStrategy | ||
|
|
||
| class DistributionComparator private constructor( | ||
| private val baseline: DoubleArray, | ||
| private val experiment: DoubleArray, | ||
| /** | ||
| * A ratio by which experiment can be slower/faster than baseline and not considered as a regression/improvement | ||
| */ | ||
| private val tolerance: Double, | ||
| private val significance: Double | ||
| ) { | ||
|
|
||
|
|
||
| /** | ||
| * Performs a one-tailed Mann–Whitney U test to check whether experiment is slower than the baseline | ||
| * | ||
| * @return true if the experiment is slower than the baseline by more than tolerance, false otherwise | ||
| */ | ||
| private fun isExperimentRegressed(baselineMedian: Double): Boolean { | ||
| val mu = -tolerance * baselineMedian | ||
| return WilcoxonRankSum(baseline, experiment, mu).pValue1SidedLess < significance | ||
| } | ||
|
|
||
| private fun isExperimentImproved(baselineMedian: Double): Boolean { | ||
| val mu = -tolerance * baselineMedian | ||
| return WilcoxonRankSum(experiment, baseline, mu).pValue1SidedLess < significance | ||
| } | ||
|
|
||
| /** | ||
| * Pseudo-median: the median of the Walsh (pairwise) averages | ||
| */ | ||
| private fun pseudoMedian(array: DoubleArray): Double { | ||
| val n = array.size | ||
| val size = n * (n + 1) / 2 - n | ||
| val values = DoubleArray(size) | ||
| var k = 0 | ||
| for (i in 0 until n) { | ||
| for (j in i + 1 until n) { | ||
| values[k++] = (array[i] + array[j]) / 2 | ||
| } | ||
| } | ||
| return Median().evaluate(values) | ||
| } | ||
|
|
||
| private fun median(func: (xi: Double, yj: Double) -> Double): Double { | ||
| val values = DoubleArray(baseline.size * experiment.size) | ||
| var k = 0 | ||
| for (i in baseline.indices) { | ||
| for (j in experiment.indices) { | ||
| values[k++] = func(baseline[i], experiment[j]) | ||
| } | ||
| } | ||
| return Median().withNaNStrategy(NaNStrategy.MINIMAL).evaluate(values) | ||
|
Contributor
There was a problem hiding this comment. When used on a set of latency measurements, why would we inject fake negative infinities?
Contributor
Author
There was a problem hiding this comment. It's needed for cases when part of the distribution is the same and other part is different. Without it small difference will be calculated as whole distribution difference and instead of NaN you will get -0.45623836126629425 in attached example
Contributor
There was a problem hiding this comment. Yes, but the workaround injects fake extreme values. Both "before" and "after" seem wrong.
Contributor
Author
There was a problem hiding this comment. The only value of this PR is to have a |
||
| } | ||
|
|
||
| private fun shift(): Double { | ||
| return median { xi, yj -> yj - xi } | ||
| } | ||
|
|
||
| private fun ratio(): Double { | ||
| return median { xi, yj -> yj / xi } | ||
| } | ||
|
|
||
| /** | ||
| * Calculates the distance between two data sets based on the [Hodges-Lehmann estimator][]. | ||
| * [Hodges-Lehmann estimator]: https://en.wikipedia.org/wiki/Hodges%E2%80%93Lehmann_estimator | ||
| * https://aakinshin.net/hodges-lehmann-estimator/ | ||
| * https://github.com/AndreyAkinshin/perfolizer/blob/master/src/Perfolizer/Perfolizer/Mathematics/GenericEstimators/HodgesLehmannEstimator.cs | ||
| * | ||
| * Takes into account tolerance which answers the question "is change is big enough to matter?" | ||
| */ | ||
| fun compare(): DistributionComparison { | ||
| val experimentShift = shift() | ||
| val baselineMedian = pseudoMedian(baseline) | ||
| val experimentRatio = ratio() | ||
| val isExperimentImproved = isExperimentImproved(baselineMedian) | ||
| val isExperimentRegressed = isExperimentRegressed(baselineMedian) | ||
| return DistributionComparison( | ||
| experimentRelativeChange = experimentRatio - 1, | ||
| experimentShift = experimentShift, | ||
| isExperimentRegressed = isExperimentRegressed, | ||
| isExperimentImproved = isExperimentImproved | ||
| ) | ||
| } | ||
|
|
||
| class Builder( | ||
| private var baseline: DoubleArray, | ||
| private var experiment: DoubleArray | ||
| ) { | ||
| private var significance: Double = 0.05 | ||
| private var tolerance: Double = 0.01 | ||
|
|
||
| fun significance(significance: Double) = apply { this.significance = significance } | ||
| fun tolerance(tolerance: Double) = apply { this.tolerance = tolerance } | ||
| fun baseline(baseline: DoubleArray) = apply { this.baseline = baseline } | ||
| fun experiment(experiment: DoubleArray) = apply { this.experiment = experiment } | ||
|
|
||
| fun build() = DistributionComparator( | ||
| baseline = baseline, | ||
| experiment = experiment, | ||
| tolerance = tolerance, | ||
| significance = significance | ||
| ) | ||
|
|
||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| package com.atlassian.performance.tools.report.api.distribution | ||
|
|
||
| class DistributionComparison( | ||
| val experimentRelativeChange: Double, | ||
|
|
||
| val experimentShift: Double, | ||
| val isExperimentRegressed: Boolean, | ||
| val isExperimentImproved: Boolean | ||
|
Contributor
There was a problem hiding this comment. The |
||
| ) { | ||
|
|
||
| init { | ||
| if (isExperimentImproved && isExperimentRegressed) { | ||
| throw IllegalArgumentException("Experiment can't be both regressed and improved at the same time") | ||
| } | ||
| } | ||
|
|
||
| fun hasImpact() = isExperimentRegressed || isExperimentImproved | ||
|
|
||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.