variable results of ModelMetrics::auc on r-devel

[topepo]

When running the code on R4.0, the results are different each time:

library(ModelMetrics)
#> 
#> Attaching package: 'ModelMetrics'
#> The following object is masked from 'package:base':
#> 
#>     kappa
set.seed(52)
N = 100000
Actual = as.numeric(runif(N) > .5)
Predicted = as.numeric(runif(N))

actual = Actual
predicted = Predicted

ModelMetrics::auc(Actual, Predicted)
#> [1] -0.4122159

ModelMetrics::auc(Actual, Predicted)
#> [1] -0.4290166

ModelMetrics::auc(Actual, Predicted)
#> [1] -0.2943601

^{Created on 2019-12-31 by the reprex package (v0.3.0)}

Session info

devtools::session_info()
#> ─ Session info ───────────────────────────────────────────────────────────────
#>  setting  value                                             
#>  version  R Under development (unstable) (2019-12-29 r77627)
#>  os       macOS Catalina 10.15.1                            
#>  system   x86_64, darwin15.6.0                              
#>  ui       X11                                               
#>  language (EN)                                              
#>  collate  en_US.UTF-8                                       
#>  ctype    en_US.UTF-8                                       
#>  tz       America/New_York                                  
#>  date     2019-12-31                                        
#> 
#> ─ Packages ───────────────────────────────────────────────────────────────────
#>  package      * version    date       lib source                         
#>  assertthat     0.2.1      2019-03-21 [1] CRAN (R 4.0.0)                 
#>  backports      1.1.5      2019-10-02 [1] CRAN (R 4.0.0)                 
#>  callr          3.4.0      2019-12-09 [1] CRAN (R 4.0.0)                 
#>  cli            2.0.0      2019-12-09 [1] CRAN (R 4.0.0)                 
#>  crayon         1.3.4      2017-09-16 [1] CRAN (R 4.0.0)                 
#>  data.table     1.12.8     2019-12-09 [1] CRAN (R 4.0.0)                 
#>  desc           1.2.0      2018-05-01 [1] CRAN (R 4.0.0)                 
#>  devtools       2.2.2      2019-12-31 [1] Github (r-lib/devtools@03537c6)
#>  digest         0.6.23     2019-11-23 [1] CRAN (R 4.0.0)                 
#>  ellipsis       0.3.0      2019-09-20 [1] CRAN (R 4.0.0)                 
#>  evaluate       0.14       2019-05-28 [1] CRAN (R 4.0.0)                 
#>  fansi          0.4.0      2018-10-05 [1] CRAN (R 4.0.0)                 
#>  fs             1.3.1      2019-05-06 [1] CRAN (R 4.0.0)                 
#>  glue           1.3.1      2019-03-12 [1] CRAN (R 4.0.0)                 
#>  highr          0.8        2019-03-20 [1] CRAN (R 4.0.0)                 
#>  htmltools      0.4.0      2019-10-04 [1] CRAN (R 4.0.0)                 
#>  knitr          1.26       2019-11-12 [1] CRAN (R 4.0.0)                 
#>  magrittr       1.5        2014-11-22 [1] CRAN (R 4.0.0)                 
#>  memoise        1.1.0      2017-04-21 [1] CRAN (R 4.0.0)                 
#>  ModelMetrics * 1.2.2      2018-11-03 [1] CRAN (R 4.0.0)                 
#>  pkgbuild       1.0.6      2019-10-09 [1] CRAN (R 4.0.0)                 
#>  pkgload        1.0.2      2018-10-29 [1] CRAN (R 4.0.0)                 
#>  prettyunits    1.0.2      2015-07-13 [1] CRAN (R 4.0.0)                 
#>  processx       3.4.1      2019-07-18 [1] CRAN (R 4.0.0)                 
#>  ps             1.3.0      2018-12-21 [1] CRAN (R 4.0.0)                 
#>  R6             2.4.1      2019-11-12 [1] CRAN (R 4.0.0)                 
#>  Rcpp           1.0.3      2019-11-08 [1] CRAN (R 4.0.0)                 
#>  remotes        2.1.0      2019-06-24 [1] CRAN (R 4.0.0)                 
#>  rlang          0.4.2      2019-11-23 [1] CRAN (R 4.0.0)                 
#>  rmarkdown      2.0        2019-12-12 [1] CRAN (R 4.0.0)                 
#>  rprojroot      1.3-2      2018-01-03 [1] CRAN (R 4.0.0)                 
#>  sessioninfo    1.1.1      2018-11-05 [1] CRAN (R 4.0.0)                 
#>  stringi        1.4.3      2019-03-12 [1] CRAN (R 4.0.0)                 
#>  stringr        1.4.0      2019-02-10 [1] CRAN (R 4.0.0)                 
#>  testthat       2.3.1      2019-12-01 [1] CRAN (R 4.0.0)                 
#>  usethis        1.5.1.9000 2019-12-31 [1] Github (r-lib/usethis@b2e894e) 
#>  withr          2.1.2      2018-03-15 [1] CRAN (R 4.0.0)                 
#>  xfun           0.11       2019-11-12 [1] CRAN (R 4.0.0)                 
#>  yaml           2.2.0      2018-07-25 [1] CRAN (R 4.0.0)                 
#> 
#> [1] /Library/Frameworks/R.framework/Versions/4.0/Resources/library

on R 3.6.1:

library(ModelMetrics)
#> 
#> Attaching package: 'ModelMetrics'
#> The following object is masked from 'package:base':
#> 
#>     kappa
set.seed(52)
N = 100000
Actual = as.numeric(runif(N) > .5)
Predicted = as.numeric(runif(N))

actual = Actual
predicted = Predicted

ModelMetrics::auc(Actual, Predicted)
#> [1] 0.5030778

ModelMetrics::auc(Actual, Predicted)
#> [1] 0.5030778

ModelMetrics::auc(Actual, Predicted)
#> [1] 0.5030778

^{Created on 2019-12-31 by the reprex package (v0.3.0)}

Session info

devtools::session_info()
#> ─ Session info ───────────────────────────────────────────────────────────────
#>  setting  value                       
#>  version  R version 3.6.1 (2019-07-05)
#>  os       macOS Mojave 10.14.6        
#>  system   x86_64, darwin15.6.0        
#>  ui       X11                         
#>  language (EN)                        
#>  collate  en_US.UTF-8                 
#>  ctype    en_US.UTF-8                 
#>  tz       America/New_York            
#>  date     2019-12-31                  
#> 
#> ─ Packages ───────────────────────────────────────────────────────────────────
#>  package      * version    date       lib source                         
#>  assertthat     0.2.1      2019-03-21 [1] CRAN (R 3.6.0)                 
#>  backports      1.1.5      2019-10-02 [1] CRAN (R 3.6.0)                 
#>  callr          3.4.0      2019-12-09 [1] CRAN (R 3.6.0)                 
#>  cli            2.0.0      2019-12-09 [1] CRAN (R 3.6.0)                 
#>  crayon         1.3.4      2017-09-16 [1] CRAN (R 3.6.0)                 
#>  data.table     1.12.6     2019-10-18 [1] CRAN (R 3.6.0)                 
#>  desc           1.2.0      2018-05-01 [1] CRAN (R 3.6.0)                 
#>  devtools       2.2.1.9000 2019-10-31 [1] Github (r-lib/devtools@1da672a)
#>  digest         0.6.23     2019-11-23 [1] CRAN (R 3.6.1)                 
#>  ellipsis       0.3.0      2019-09-20 [1] CRAN (R 3.6.0)                 
#>  evaluate       0.14       2019-05-28 [1] CRAN (R 3.6.0)                 
#>  fansi          0.4.0      2018-10-05 [1] CRAN (R 3.6.0)                 
#>  fs             1.3.1      2019-05-06 [1] CRAN (R 3.6.0)                 
#>  glue           1.3.1      2019-03-12 [1] CRAN (R 3.6.0)                 
#>  highr          0.8        2019-03-20 [1] CRAN (R 3.6.0)                 
#>  htmltools      0.4.0      2019-10-04 [1] CRAN (R 3.6.0)                 
#>  knitr          1.26       2019-11-12 [1] CRAN (R 3.6.0)                 
#>  magrittr       1.5        2014-11-22 [1] CRAN (R 3.6.0)                 
#>  memoise        1.1.0      2017-04-21 [1] CRAN (R 3.6.0)                 
#>  ModelMetrics * 1.2.2      2018-11-03 [1] CRAN (R 3.6.0)                 
#>  pkgbuild       1.0.6      2019-10-09 [1] CRAN (R 3.6.0)                 
#>  pkgload        1.0.2      2018-10-29 [1] CRAN (R 3.6.0)                 
#>  prettyunits    1.0.2      2015-07-13 [1] CRAN (R 3.6.0)                 
#>  processx       3.4.1      2019-07-18 [1] CRAN (R 3.6.0)                 
#>  ps             1.3.0      2018-12-21 [1] CRAN (R 3.6.0)                 
#>  R6             2.4.1      2019-11-12 [1] CRAN (R 3.6.0)                 
#>  Rcpp           1.0.3      2019-11-08 [1] CRAN (R 3.6.0)                 
#>  remotes        2.1.0      2019-06-24 [1] CRAN (R 3.6.0)                 
#>  rlang          0.4.2      2019-11-23 [1] CRAN (R 3.6.0)                 
#>  rmarkdown      2.0        2019-12-12 [1] CRAN (R 3.6.0)                 
#>  rprojroot      1.3-2      2018-01-03 [1] CRAN (R 3.6.0)                 
#>  sessioninfo    1.1.1      2018-11-05 [1] CRAN (R 3.6.0)                 
#>  stringi        1.4.3      2019-03-12 [1] CRAN (R 3.6.0)                 
#>  stringr        1.4.0      2019-02-10 [1] CRAN (R 3.6.0)                 
#>  testthat       2.2.1      2019-07-25 [1] CRAN (R 3.6.0)                 
#>  usethis        1.5.1.9000 2019-12-18 [1] Github (r-lib/usethis@b2e894e) 
#>  withr          2.1.2      2018-03-15 [1] CRAN (R 3.6.0)                 
#>  xfun           0.11       2019-11-12 [1] CRAN (R 3.6.0)                 
#>  yaml           2.2.0      2018-07-25 [1] CRAN (R 3.6.0)                 
#> 
#> [1] /Library/Frameworks/R.framework/Versions/3.6/Resources/library

[JackStat]

Very strange. I will examine this week

[JackStat]

@topepo Do you have a resource for booting up a docker image with R 4.0? Maybe another way?

[JackStat]

I was not able to reproduce the issue. I booted up an image of R4.0 with ModelMetrics installed and ran your code. Here are the results

R Under development (unstable) (2020-01-28 r77738) -- "Unsuffered Consequences"
Copyright (C) 2020 The R Foundation for Statistical Computing
Platform: x86_64-pc-linux-gnu (64-bit)

R is free software and comes with ABSOLUTELY NO WARRANTY.
You are welcome to redistribute it under certain conditions.
Type 'license()' or 'licence()' for distribution details.

R is a collaborative project with many contributors.
Type 'contributors()' for more information and
'citation()' on how to cite R or R packages in publications.

Type 'demo()' for some demos, 'help()' for on-line help, or
'help.start()' for an HTML browser interface to help.
Type 'q()' to quit R.

> library(ModelMetrics)

Attaching package: ‘ModelMetrics’

The following object is masked from ‘package:base’:

    kappa

> set.seed(52)
> N = 100000
> Actual = as.numeric(runif(N) > .5)
> Predicted = as.numeric(runif(N))
> actual = Actual
> predicted = Predicted
>
> ModelMetrics::auc(Actual, Predicted)
[1] 0.5030778
> ModelMetrics::auc(Actual, Predicted)
[1] 0.5030778
> ModelMetrics::auc(Actual, Predicted)
[1] 0.5030778
> sessionInfo()
R Under development (unstable) (2020-01-28 r77738)
Platform: x86_64-pc-linux-gnu (64-bit)
Running under: Debian GNU/Linux 10 (buster)

Matrix products: default
BLAS/LAPACK: /usr/lib/x86_64-linux-gnu/libopenblasp-r0.3.5.so

locale:
 [1] LC_CTYPE=en_US.UTF-8       LC_NUMERIC=C
 [3] LC_TIME=en_US.UTF-8        LC_COLLATE=en_US.UTF-8
 [5] LC_MONETARY=en_US.UTF-8    LC_MESSAGES=C
 [7] LC_PAPER=en_US.UTF-8       LC_NAME=C
 [9] LC_ADDRESS=C               LC_TELEPHONE=C
[11] LC_MEASUREMENT=en_US.UTF-8 LC_IDENTIFICATION=C

attached base packages:
[1] stats     graphics  grDevices utils     datasets  methods   base

other attached packages:
[1] ModelMetrics_1.2.2.1

loaded via a namespace (and not attached):
[1] compiler_4.0.0    Rcpp_1.0.3        data.table_1.12.8

[JackStat]

@topepo I cannot reproduce the issue. Are you still observing this behavior?