From 80201739e2b17de3ac45dbecc8f72c7e29a436b4 Mon Sep 17 00:00:00 2001 From: Indrajeet Patil Date: Sun, 26 Mar 2023 10:38:19 +0200 Subject: [PATCH 01/10] Check if tests are coupled cf. https://github.com/r-lib/lintr/issues/1937 --- .../workflows/check-random-test-order.yaml | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 .github/workflows/check-random-test-order.yaml diff --git a/.github/workflows/check-random-test-order.yaml b/.github/workflows/check-random-test-order.yaml new file mode 100644 index 0000000000..ddfd4f75cc --- /dev/null +++ b/.github/workflows/check-random-test-order.yaml @@ -0,0 +1,74 @@ +# Run tests in random order +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + +name: check-random-test-order + +jobs: + check-random-test-order: + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + + steps: + - uses: actions/checkout@v3 + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: "devel" + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + pak-version: devel + extra-packages: | + local::. + + - name: Run Tests in Random Order + run: | + options(crayon.enabled = TRUE) + withr::local_envvar(TESTTHAT_PARALLEL = "FALSE") + library(cli) + library(glue) + library(testthat) + pkgload::load_all(".") + test_script_paths <- testthat::find_test_scripts("tests/testthat") + set.seed(123) + randomized_test_script_paths <- sample(test_script_paths) + any_test_failures <- FALSE + any_test_errors <- FALSE + test_path <- function(path) { + report <- as.data.frame(testthat::test_file(path, reporter = "silent")) + has_test_failures <- any(report$failed == 1L) + has_test_errors <- any(report$error == 1L) + if (has_test_failures) { + cli_alert_danger(glue("Tests in `{path}` are failing.")) + any_test_failures <<- TRUE + } + if (has_test_errors) { + cli_alert_danger(glue("There was error while running tests in `{path}`.")) + any_test_errors <<- TRUE + } + if (!has_test_failures && !has_test_errors) { + cli_alert_success(glue("All tests passing in `{path}`.")) + } + } + cli_rule() + cli_inform("Running tests in random order:") + cli_rule() + purrr::walk(randomized_test_script_paths, test_path) + cli_rule() + if (any_test_failures) { + cli_abort("Tests in some files are failing. Check the log.") + } + if (any_test_errors) { + cli_abort("There was error while running tests in some files. Check the log.") + } + if (!any_test_failures && !any_test_errors) { + cli_alert_success("Tests from all files are passing!") + } + cli_rule() + shell: Rscript {0} From d01a9e35e7b7c462ab3b1848f821a0e321cbe769 Mon Sep 17 00:00:00 2001 From: Indrajeet Patil Date: Sat, 15 Apr 2023 10:50:23 +0200 Subject: [PATCH 02/10] select a different seed each time --- .github/workflows/check-random-test-order.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check-random-test-order.yaml b/.github/workflows/check-random-test-order.yaml index ddfd4f75cc..fede8548dd 100644 --- a/.github/workflows/check-random-test-order.yaml +++ b/.github/workflows/check-random-test-order.yaml @@ -36,7 +36,9 @@ jobs: library(testthat) pkgload::load_all(".") test_script_paths <- testthat::find_test_scripts("tests/testthat") - set.seed(123) + seed <- sample.int(1e6, 1L) + cli_inform("Chosen seed for the current test run: {seed}") + set.seed(seed) randomized_test_script_paths <- sample(test_script_paths) any_test_failures <- FALSE any_test_errors <- FALSE From c26668d178a401aa2d3923fd7e3156473d0385a7 Mon Sep 17 00:00:00 2001 From: Indrajeet Patil Date: Sat, 15 Apr 2023 10:53:52 +0200 Subject: [PATCH 03/10] show the exact errors --- .github/workflows/check-random-test-order.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check-random-test-order.yaml b/.github/workflows/check-random-test-order.yaml index fede8548dd..b4fb3dd504 100644 --- a/.github/workflows/check-random-test-order.yaml +++ b/.github/workflows/check-random-test-order.yaml @@ -33,7 +33,6 @@ jobs: withr::local_envvar(TESTTHAT_PARALLEL = "FALSE") library(cli) library(glue) - library(testthat) pkgload::load_all(".") test_script_paths <- testthat::find_test_scripts("tests/testthat") seed <- sample.int(1e6, 1L) @@ -49,10 +48,14 @@ jobs: if (has_test_failures) { cli_alert_danger(glue("Tests in `{path}` are failing.")) any_test_failures <<- TRUE + failed_tests <- tibble::as_tibble(subset(report, failed == 1L)[, c("test", "result")]) + print(glue::glue_data(failed_tests, "Test `{test}` is failing:\n{purrr::pluck(result, 1L, 1L)}")) } if (has_test_errors) { cli_alert_danger(glue("There was error while running tests in `{path}`.")) any_test_errors <<- TRUE + errored_tests <- tibble::as_tibble(subset(report, error == 1L)[, c("test", "result")]) + print(glue::glue_data(errored_tests, "Test `{test}` has error:\n{purrr::pluck(result, 1L, 1L)}")) } if (!has_test_failures && !has_test_errors) { cli_alert_success(glue("All tests passing in `{path}`.")) @@ -64,10 +67,10 @@ jobs: purrr::walk(randomized_test_script_paths, test_path) cli_rule() if (any_test_failures) { - cli_abort("Tests in some files are failing. Check the log.") + cli_abort("Tests in some files are failing.") } if (any_test_errors) { - cli_abort("There was error while running tests in some files. Check the log.") + cli_abort("There was error while running tests in some files.") } if (!any_test_failures && !any_test_errors) { cli_alert_success("Tests from all files are passing!") From d462116d71fc1382927fb5aafa351417859d60b5 Mon Sep 17 00:00:00 2001 From: Indrajeet Patil Date: Thu, 20 Apr 2023 10:43:04 +0200 Subject: [PATCH 04/10] extract code to a script --- .dev/run-test-files-in-random-order.R | 54 +++++++++++++++++++ .../workflows/check-random-test-order.yaml | 48 +---------------- 2 files changed, 56 insertions(+), 46 deletions(-) create mode 100644 .dev/run-test-files-in-random-order.R diff --git a/.dev/run-test-files-in-random-order.R b/.dev/run-test-files-in-random-order.R new file mode 100644 index 0000000000..7c769bb059 --- /dev/null +++ b/.dev/run-test-files-in-random-order.R @@ -0,0 +1,54 @@ +library(cli) +library(glue) +withr::local_envvar(TESTTHAT_PARALLEL = "FALSE") +pkgload::load_all(".") + +test_script_paths <- testthat::find_test_scripts("tests/testthat") +seed <- sample.int(1e6, 1L) +cli_inform("Chosen seed for the current test run: {seed}") +set.seed(seed) +randomized_test_script_paths <- sample(test_script_paths) + +any_test_failures <- FALSE +any_test_errors <- FALSE + +test_path <- function(path) { + report <- as.data.frame(testthat::test_file(path, reporter = "silent")) + has_test_failures <- any(report$failed == 1L) + has_test_errors <- any(report$error == 1L) + if (has_test_failures) { + cli_alert_danger(glue("Tests in `{path}` are failing.")) + any_test_failures <<- TRUE + failed_tests <- tibble::as_tibble(subset(report, failed == 1L)[, c("test", "result")]) + print(glue::glue_data(failed_tests, "Test `{test}` is failing:\n{purrr::pluck(result, 1L, 1L)}")) + } + if (has_test_errors) { + cli_alert_danger(glue("There was error while running tests in `{path}`.")) + any_test_errors <<- TRUE + errored_tests <- tibble::as_tibble(subset(report, error == 1L)[, c("test", "result")]) + print(glue::glue_data(errored_tests, "Test `{test}` has error:\n{purrr::pluck(result, 1L, 1L)}")) + } + if (!has_test_failures && !has_test_errors) { + cli_alert_success(glue("All tests passing in `{path}`.")) + } +} + +cli_rule() +cli_inform("Running tests in random order:") +cli_rule() + +purrr::walk(randomized_test_script_paths, test_path) + +cli_rule() +if (any_test_failures) { + cli_abort("Tests in some files are failing.") +} + +if (any_test_errors) { + cli_abort("There was error while running tests in some files.") +} + +if (!any_test_failures && !any_test_errors) { + cli_alert_success("Tests from all files are passing!") +} +cli_rule() diff --git a/.github/workflows/check-random-test-order.yaml b/.github/workflows/check-random-test-order.yaml index b4fb3dd504..af30b7eef1 100644 --- a/.github/workflows/check-random-test-order.yaml +++ b/.github/workflows/check-random-test-order.yaml @@ -30,50 +30,6 @@ jobs: - name: Run Tests in Random Order run: | options(crayon.enabled = TRUE) - withr::local_envvar(TESTTHAT_PARALLEL = "FALSE") - library(cli) - library(glue) - pkgload::load_all(".") - test_script_paths <- testthat::find_test_scripts("tests/testthat") - seed <- sample.int(1e6, 1L) - cli_inform("Chosen seed for the current test run: {seed}") - set.seed(seed) - randomized_test_script_paths <- sample(test_script_paths) - any_test_failures <- FALSE - any_test_errors <- FALSE - test_path <- function(path) { - report <- as.data.frame(testthat::test_file(path, reporter = "silent")) - has_test_failures <- any(report$failed == 1L) - has_test_errors <- any(report$error == 1L) - if (has_test_failures) { - cli_alert_danger(glue("Tests in `{path}` are failing.")) - any_test_failures <<- TRUE - failed_tests <- tibble::as_tibble(subset(report, failed == 1L)[, c("test", "result")]) - print(glue::glue_data(failed_tests, "Test `{test}` is failing:\n{purrr::pluck(result, 1L, 1L)}")) - } - if (has_test_errors) { - cli_alert_danger(glue("There was error while running tests in `{path}`.")) - any_test_errors <<- TRUE - errored_tests <- tibble::as_tibble(subset(report, error == 1L)[, c("test", "result")]) - print(glue::glue_data(errored_tests, "Test `{test}` has error:\n{purrr::pluck(result, 1L, 1L)}")) - } - if (!has_test_failures && !has_test_errors) { - cli_alert_success(glue("All tests passing in `{path}`.")) - } - } - cli_rule() - cli_inform("Running tests in random order:") - cli_rule() - purrr::walk(randomized_test_script_paths, test_path) - cli_rule() - if (any_test_failures) { - cli_abort("Tests in some files are failing.") - } - if (any_test_errors) { - cli_abort("There was error while running tests in some files.") - } - if (!any_test_failures && !any_test_errors) { - cli_alert_success("Tests from all files are passing!") - } - cli_rule() + callr::rscript("run-test-files-in-random-order.R") + shell: Rscript {0} From b084cac331d8d39351948568c1fbd93a42e731d2 Mon Sep 17 00:00:00 2001 From: Indrajeet Patil Date: Thu, 20 Apr 2023 10:48:08 +0200 Subject: [PATCH 05/10] Update check-random-test-order.yaml --- .github/workflows/check-random-test-order.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-random-test-order.yaml b/.github/workflows/check-random-test-order.yaml index af30b7eef1..66c2dca012 100644 --- a/.github/workflows/check-random-test-order.yaml +++ b/.github/workflows/check-random-test-order.yaml @@ -30,6 +30,6 @@ jobs: - name: Run Tests in Random Order run: | options(crayon.enabled = TRUE) - callr::rscript("run-test-files-in-random-order.R") + callr::rscript(".dev/run-test-files-in-random-order.R") shell: Rscript {0} From f6e3b2f31cdf3bdd21c97a0e3e3f46cd2eba4fd0 Mon Sep 17 00:00:00 2001 From: Indrajeet Patil Date: Tue, 19 Sep 2023 08:42:55 +0200 Subject: [PATCH 06/10] run on release version --- .github/workflows/check-random-test-order.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-random-test-order.yaml b/.github/workflows/check-random-test-order.yaml index 66c2dca012..77dea2b53c 100644 --- a/.github/workflows/check-random-test-order.yaml +++ b/.github/workflows/check-random-test-order.yaml @@ -18,7 +18,7 @@ jobs: - uses: r-lib/actions/setup-r@v2 with: - r-version: "devel" + r-version: "release" use-public-rspm: true - uses: r-lib/actions/setup-r-dependencies@v2 From d4a7edc4bf285ee5ef7a0e0881ac14a6ce105e58 Mon Sep 17 00:00:00 2001 From: Indrajeet Patil Date: Sat, 15 Nov 2025 21:37:48 +0530 Subject: [PATCH 07/10] use latest testthat shuffle for randomized testing --- .dev/run-test-files-in-random-order.R | 54 ------------------- .../workflows/check-random-test-order.yaml | 3 +- .gitignore | 1 + DESCRIPTION | 2 +- 4 files changed, 3 insertions(+), 57 deletions(-) delete mode 100644 .dev/run-test-files-in-random-order.R diff --git a/.dev/run-test-files-in-random-order.R b/.dev/run-test-files-in-random-order.R deleted file mode 100644 index 7c769bb059..0000000000 --- a/.dev/run-test-files-in-random-order.R +++ /dev/null @@ -1,54 +0,0 @@ -library(cli) -library(glue) -withr::local_envvar(TESTTHAT_PARALLEL = "FALSE") -pkgload::load_all(".") - -test_script_paths <- testthat::find_test_scripts("tests/testthat") -seed <- sample.int(1e6, 1L) -cli_inform("Chosen seed for the current test run: {seed}") -set.seed(seed) -randomized_test_script_paths <- sample(test_script_paths) - -any_test_failures <- FALSE -any_test_errors <- FALSE - -test_path <- function(path) { - report <- as.data.frame(testthat::test_file(path, reporter = "silent")) - has_test_failures <- any(report$failed == 1L) - has_test_errors <- any(report$error == 1L) - if (has_test_failures) { - cli_alert_danger(glue("Tests in `{path}` are failing.")) - any_test_failures <<- TRUE - failed_tests <- tibble::as_tibble(subset(report, failed == 1L)[, c("test", "result")]) - print(glue::glue_data(failed_tests, "Test `{test}` is failing:\n{purrr::pluck(result, 1L, 1L)}")) - } - if (has_test_errors) { - cli_alert_danger(glue("There was error while running tests in `{path}`.")) - any_test_errors <<- TRUE - errored_tests <- tibble::as_tibble(subset(report, error == 1L)[, c("test", "result")]) - print(glue::glue_data(errored_tests, "Test `{test}` has error:\n{purrr::pluck(result, 1L, 1L)}")) - } - if (!has_test_failures && !has_test_errors) { - cli_alert_success(glue("All tests passing in `{path}`.")) - } -} - -cli_rule() -cli_inform("Running tests in random order:") -cli_rule() - -purrr::walk(randomized_test_script_paths, test_path) - -cli_rule() -if (any_test_failures) { - cli_abort("Tests in some files are failing.") -} - -if (any_test_errors) { - cli_abort("There was error while running tests in some files.") -} - -if (!any_test_failures && !any_test_errors) { - cli_alert_success("Tests from all files are passing!") -} -cli_rule() diff --git a/.github/workflows/check-random-test-order.yaml b/.github/workflows/check-random-test-order.yaml index 77dea2b53c..394be86a54 100644 --- a/.github/workflows/check-random-test-order.yaml +++ b/.github/workflows/check-random-test-order.yaml @@ -30,6 +30,5 @@ jobs: - name: Run Tests in Random Order run: | options(crayon.enabled = TRUE) - callr::rscript(".dev/run-test-files-in-random-order.R") - + testthat::test_dir("tests", shuffle = TRUE) shell: Rscript {0} diff --git a/.gitignore b/.gitignore index c3cf3b5460..4612431e3c 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ script.R *.Rcheck lintr_*.tar.gz testthat-problems.rds +tests/testthat/_problems/ docs inst/doc diff --git a/DESCRIPTION b/DESCRIPTION index 915e03d88b..51691bb611 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -46,7 +46,7 @@ Suggests: rlang, rmarkdown, rstudioapi (>= 0.2), - testthat (>= 3.2.1), + testthat (>= 3.3.0), tibble, tufte, withr (>= 2.5.0) From b1f9027f0dcb4f094d3cb8cbcdd9312e221d94cb Mon Sep 17 00:00:00 2001 From: Indrajeet Patil Date: Sat, 15 Nov 2025 21:57:34 +0530 Subject: [PATCH 08/10] no parallel --- .github/workflows/check-random-test-order.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/check-random-test-order.yaml b/.github/workflows/check-random-test-order.yaml index 394be86a54..bc43d820cd 100644 --- a/.github/workflows/check-random-test-order.yaml +++ b/.github/workflows/check-random-test-order.yaml @@ -30,5 +30,6 @@ jobs: - name: Run Tests in Random Order run: | options(crayon.enabled = TRUE) + withr::local_envvar(TESTTHAT_PARALLEL = "FALSE") testthat::test_dir("tests", shuffle = TRUE) shell: Rscript {0} From fe6e28375d1d1738f112a02f4c2816d2fa3ff0d4 Mon Sep 17 00:00:00 2001 From: Indrajeet Patil Date: Sun, 16 Nov 2025 19:04:38 +0530 Subject: [PATCH 09/10] show what seed was used --- .github/workflows/check-random-test-order.yaml | 12 ++++++------ .gitignore | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check-random-test-order.yaml b/.github/workflows/check-random-test-order.yaml index bc43d820cd..581b1bc56b 100644 --- a/.github/workflows/check-random-test-order.yaml +++ b/.github/workflows/check-random-test-order.yaml @@ -1,4 +1,3 @@ -# Run tests in random order on: push: branches: [main, master] @@ -14,22 +13,23 @@ jobs: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v5 - uses: r-lib/actions/setup-r@v2 with: - r-version: "release" use-public-rspm: true - uses: r-lib/actions/setup-r-dependencies@v2 with: - pak-version: devel - extra-packages: | - local::. + install-quarto: false + extra-packages: local::. - name: Run Tests in Random Order run: | options(crayon.enabled = TRUE) withr::local_envvar(TESTTHAT_PARALLEL = "FALSE") + seed <- sample.int(1e6, 1L) + cli::cli_inform("Chosen seed for the current test run: {seed}") + set.seed(seed) testthat::test_dir("tests", shuffle = TRUE) shell: Rscript {0} diff --git a/.gitignore b/.gitignore index 4612431e3c..be03102178 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ tests/testthat/_problems/ docs inst/doc .DS_Store +.claude/settings.local.json From e6b2dce8bf234f34c0661c1844fb5c0ae78a7787 Mon Sep 17 00:00:00 2001 From: Indrajeet Patil Date: Sun, 16 Nov 2025 21:33:47 +0530 Subject: [PATCH 10/10] only run randomized tests in release branches --- .github/workflows/check-random-test-order.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check-random-test-order.yaml b/.github/workflows/check-random-test-order.yaml index 581b1bc56b..c559073ef2 100644 --- a/.github/workflows/check-random-test-order.yaml +++ b/.github/workflows/check-random-test-order.yaml @@ -1,8 +1,7 @@ on: push: - branches: [main, master] - pull_request: - branches: [main, master] + branches: + - 'rc-*' name: check-random-test-order @@ -27,7 +26,6 @@ jobs: - name: Run Tests in Random Order run: | options(crayon.enabled = TRUE) - withr::local_envvar(TESTTHAT_PARALLEL = "FALSE") seed <- sample.int(1e6, 1L) cli::cli_inform("Chosen seed for the current test run: {seed}") set.seed(seed)