diff --git a/.Rbuildignore b/.Rbuildignore index 10f59c29..97348234 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -1,4 +1,5 @@ .+~ +BUGS DEVEL-README DEVEL-README.md \.Rhistory$ @@ -25,14 +26,15 @@ DEVEL-README.md ^TUNE.* ^\.git$ ^\.github$ +^\.pre-commit-config\.yaml$ ^check-reverse ^codecov\.yml$ ^cran-comments.md$ +^devel-examples ^devel-tests ^dist\.sh$ ^doc$ ^docs$ -^devel-examples ^git_version$ ^hooks/hook-evaluate ^hooks/hook-run @@ -47,10 +49,10 @@ DEVEL-README.md ^revdep$ ^scenario\.txt$ ^scripts -^src/iracebin/irace$ -^src/iracebin/irace.exe$ ^src/iracebin/ablation$ ^src/iracebin/ablation.exe$ +^src/iracebin/irace$ +^src/iracebin/irace.exe$ ^src/iracebin/irace2pyimp$ ^src/iracebin/irace2pyimp.exe$ ^svn_version$ @@ -73,5 +75,5 @@ DEVEL-README.md ^vignettes/figure ^vignettes/optbib ^web -_snaps _pkgdown\.yml$ +_snaps diff --git a/.Rinstignore b/.Rinstignore index 627fd2da..f8df1d02 100644 --- a/.Rinstignore +++ b/.Rinstignore @@ -1,4 +1,4 @@ -vignettes/section/irace-options.tex -^GenericWrapper4AC \.github +^GenericWrapper4AC _snaps +vignettes/section/irace-options.tex diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 8ca56034..e3b18047 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -11,7 +11,7 @@ name: R-CMD-check concurrency: group: ${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: true - + env: R_REMOTES_NO_ERRORS_FROM_WARNINGS: true _R_CHECK_FORCE_SUGGESTS_: false @@ -29,13 +29,12 @@ jobs: strategy: fail-fast: false matrix: - os: [ ubuntu-22.04, windows-latest, macos-13, macos-14 ] - r: [ release ] + os: [ windows-latest, macos-13, macos-14 ] + r: [ release, 4.0 ] include: - # Use 4.0 to check with rtools40's older compiler - - { os: windows-latest, r: '4.0' } - # Minimum supported version. - - {os: ubuntu-20.04, r: '4.0'} + # 4.0 is the minimum supported version. + - { os: ubuntu-20.04, r: '4.0'} + - { os: ubuntu-22.04, r: 'release' } # Use latest ubuntu to make it easier to install dependencies - { os: ubuntu-24.04, r: 'devel', http-user-agent: 'release' } @@ -53,7 +52,7 @@ jobs: - name: "[Linux] Install openmpi" if: runner.os == 'Linux' run: sudo apt-get update -y && sudo apt install -y libopenmpi-dev - + - uses: r-lib/actions/setup-r@v2 with: r-version: ${{ matrix.r }} @@ -77,7 +76,7 @@ jobs: needs: check cache-version: 1 install-pandoc: true - + - uses: r-lib/actions/setup-r-dependencies@v2 if: runner.os == 'macOS' with: @@ -88,7 +87,7 @@ jobs: needs: check cache-version: 1 install-pandoc: true - + - name: Check (as CRAN) env: NOT_CRAN: false @@ -96,14 +95,14 @@ jobs: with: args: 'c("--no-manual", "--as-cran")' build_args: 'c("--compact-vignettes=gs+qpdf")' - + - name: Check env: NOT_CRAN: true uses: r-lib/actions/check-r-package@v2 with: args: 'c("--no-manual", "--run-donttest", "--timings")' - + - name: Show testthat output if: failure() run: find check -name 'testthat.Rout*' -exec cat '{}' \; || true @@ -112,7 +111,7 @@ jobs: coverage: needs: R-CMD-check name: Coverage - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 continue-on-error: true timeout-minutes: 30 env: @@ -123,16 +122,17 @@ jobs: - uses: r-lib/actions/setup-r@v2 with: + r-version: release use-public-rspm: true - uses: r-lib/actions/setup-tinytex@v2-branch - + - name: Install additional LaTeX packages run: tlmgr install ${{ env.LATEX_PACKAGES }} - uses: r-lib/actions/setup-r-dependencies@v2 with: - extra-packages: any::covr, mlr=?ignore, Rmpi=?ignore, local::. + extra-packages: any::covr, any::xml2, mlr=?ignore, Rmpi=?ignore, local::. needs: coverage install-pandoc: true @@ -143,21 +143,29 @@ jobs: run: | options(covr.fix_parallel_mcexit = TRUE) coverage <- covr::package_coverage(type = "all", - quiet = FALSE, clean = FALSE, - install_path = file.path(Sys.getenv("RUNNER_TEMP"), "package"), - commentDonttest = FALSE) + quiet = FALSE, clean = FALSE, + install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package"), + commentDonttest = FALSE) print(coverage) - covr::codecov(coverage = coverage) + covr::to_cobertura(coverage) + #covr::codecov(coverage = coverage) shell: Rscript {0} - + - uses: codecov/codecov-action@v4 + with: + fail_ci_if_error: ${{ github.event_name != 'pull_request' && true || false }} + file: ./cobertura.xml + plugin: noop + disable_search: true + token: ${{ secrets.CODECOV_TOKEN }} + - name: Show testthat output if: always() run: | ## -------------------------------------------------------------------- find ${{ runner.temp }}/package -name 'testthat.Rout*' -exec cat '{}' \; || true shell: bash - + - name: Show failures if: failure() run: | @@ -167,7 +175,7 @@ jobs: - name: Upload test results if: failure() - uses: actions/upload-artifact@main + uses: actions/upload-artifact@v4 with: name: coverage-test-failures path: ${{ runner.temp }}/package @@ -193,7 +201,7 @@ jobs: - uses: r-lib/actions/setup-r@v2 with: use-public-rspm: true - + - uses: r-lib/actions/setup-tinytex@v2-branch - name: Install additional LaTeX packages @@ -204,7 +212,7 @@ jobs: extra-packages: any::pkgdown, mlr=?ignore, Rmpi=?ignore, any::remotes needs: website install-pandoc: true - + - name: Install package with vignettes run: | remotes::install_local(".", build = TRUE, upgrade = "never", force = TRUE, build_opts = c("--no-resave-data"), build_manual = TRUE, build_vignettes = TRUE) diff --git a/.github/workflows/revdepcheck.yml b/.github/workflows/revdepcheck.yml new file mode 100644 index 00000000..01854fd4 --- /dev/null +++ b/.github/workflows/revdepcheck.yml @@ -0,0 +1,21 @@ +on: + workflow_dispatch: + inputs: + which: + type: choice + description: Which dependents to check + options: + - strong + - most + +name: Reverse dependency check + +jobs: + revdep_check: + name: Reverse check ${{ inputs.which }} dependents + uses: r-devel/recheck/.github/workflows/recheck.yml@v1 + with: + which: ${{ inputs.which }} + subdirectory: '' # set if your R package is in a subdir of the git repo + repository: '' # set to recheck an R package from another git repo + ref: '' # set to recheck a custom tag/branch from another repo diff --git a/.gitignore b/.gitignore index 83f2e20d..aa3b49cb 100644 --- a/.gitignore +++ b/.gitignore @@ -63,7 +63,7 @@ docs/ /vignettes/*.bbl /vignettes/*.fls /vignettes/*.synctex* -/vignettes/irace-options.tex +/vignettes/section/irace-options.tex /vignettes/irace-package.tex /vignettes/irace-package.pdf /vignettes/irace-package.R diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..b1d27112 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,22 @@ +repos: + - repo: https://github.com/lorenzwalthert/precommit + rev: v0.4.3.9003 + hooks: + - id: parsable-R + - id: no-browser-statement + - id: no-debug-statement + - id: deps-in-desc + exclude: '\.Rnw$' + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + - id: check-yaml + - id: check-merge-conflict + args: [ --assume-in-merge ] + - id: check-added-large-files + args: ['--maxkb=200'] + - id: file-contents-sorter + files: '^(\.Rbuildignore|\.Rinstignore)$' + - id: end-of-file-fixer + exclude: '(\.Rd|python/doc/source/reference/.*|test-doctest-.*)' diff --git a/DESCRIPTION b/DESCRIPTION index 0037ebbf..584d5fef 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -7,7 +7,7 @@ Description: Iterated race is an extension of the Iterated F-race method for settings given a set of instances of an optimization problem. M. López-Ibáñez, J. Dubois-Lacoste, L. Pérez Cáceres, T. Stützle, and M. Birattari (2016) . -Version: 3.9.0.9000 +Version: 4.2.0.9000 Authors@R: c(person("Manuel", "López-Ibáñez", role = c("aut", "cre"), email = "manuel.lopez-ibanez@manchester.ac.uk", comment = c(ORCID = "0000-0001-9974-1295")), @@ -20,17 +20,17 @@ Authors@R: c(person("Manuel", "López-Ibáñez", role = c("aut", "cre"), person("Nguyen", "Dang", role = "ctb", email = "nttd@st-andrews.ac.uk")) Depends: R (>= 4.0.0) Imports: - stats, - utils, + R6, + codetools, compiler, - fs, data.table (>= 1.15.0), - matrixStats, - R6, + fs, + matrixStats (>= 1.4.1), spacefillr, + stats, + utils, withr Suggests: - ParamHelpers, Rmpi (>= 0.6.0), highr, knitr, diff --git a/Makefile b/Makefile index e8eb958c..be2c5bc8 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,8 @@ help: @echo "install install the package" @echo "build build the package as a tar.gz file" @echo "check build the package and run 'R CMD check'" - @echo "check TEST=x run test called test-x.R" + @echo "test run all tests under testthat" + @echo "test TEST=x run test called test-x.R" @echo "rsync copy the package and install it on $(RNODE)" @echo "cran build the package and run 'R CMD check --as-cran'" @echo "winbuild submit the package to the WINDOWS builder service" @@ -47,12 +48,13 @@ install: build cd $(BINDIR) && R CMD INSTALL $(INSTALL_FLAGS) $(PACKAGE)_$(PACKAGEVERSION).tar.gz quick-install: - cd $(BINDIR) && R CMD build $(BUILD_FLAGS) $(NO_BUILD_VIGNETTES) $(PACKAGEDIR) && R CMD INSTALL $(INSTALL_FLAGS) $(PACKAGE)_$(PACKAGEVERSION).tar.gz + R CMD INSTALL $(INSTALL_FLAGS) --no-docs $(PACKAGEDIR) genoptions: R/irace-options.R vignettes/section/irace-options.tex R/irace-options.R vignettes/section/irace-options.tex: scripts/irace_options.json scripts/generate-options.R cd scripts && R --slave -f generate-options.R && cd .. + touch R/irace-options.R vignettes/section/irace-options.tex gendoc: $(Reval) 'devtools::document()' @@ -61,7 +63,7 @@ pkgdown: gendoc $(Reval) 'pkgdown::build_site()' @$(MAKE) clean -build: +build: @$(MAKE) genoptions $(MAKE) releasevignette @if grep -q @ $(PACKAGEDIR)/vignettes/$(PACKAGE)-package.bib; then true; \ @@ -107,18 +109,18 @@ releasecheck: cran $(MAKE) macbuild check: build -ifdef TEST - _R_CHECK_FORCE_SUGGESTS_=false NOT_CRAN=true $(Reval) 'devtools::test(filter="$(TEST)", stop_on_failure = TRUE)' -else test -d ./GenericWrapper4AC/build || (cd GenericWrapper4AC && python3 setup.py install --user) cd $(BINDIR) && (_R_CHECK_FORCE_SUGGESTS_=false NOT_CRAN=true R CMD check --run-donttest --timings --install-args="$(INSTALL_FLAGS)" $(PACKAGE)_$(PACKAGEVERSION).tar.gz; cat $(PACKAGE).Rcheck/$(PACKAGE)-Ex.timings) -endif -# Using testthat nicer output +# Using testthat nicer output. test: clean +ifdef TEST + _R_CHECK_FORCE_SUGGESTS_=false NOT_CRAN=true $(Reval) 'devtools::test(filter="$(TEST)", stop_on_failure = TRUE)' +else _R_CHECK_FORCE_SUGGESTS_=false NOT_CRAN=true $(Reval) 'testthat::test_local(stop_on_failure = TRUE)' +endif -clean: +clean: cd $(PACKAGEDIR) && (./cleanup; make -C src -f Makevars clean) ## FIXME: building the vignettes is a bit complicated and sometimes fails. @@ -178,8 +180,8 @@ macbuild: releasebuild $(Reval) "rhub::check(platform='macos-highsierra-release-cran', $(RHUB_COMMON_ARGS))" winbuild: releasebuild - $(Reval) "devtools::check_win_release()" - $(Reval) "devtools::check_win_devel()" + $(Reval) "devtools::check_win_release(args = '--compact-vignettes=both')" + $(Reval) "devtools::check_win_devel(args = '--compact-vignettes=both')" $(Reval) "rhub::check_on_windows($(RHUB_COMMON_ARGS))" examples: quick-install diff --git a/NAMESPACE b/NAMESPACE index f30ba5e9..9a4292f1 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -62,11 +62,9 @@ export(target_runner_default) export(testConfigurations) export(testing_fromfile) export(testing_fromlog) -import(compiler) import(data.table) import(matrixStats) import(stats) -import(utils) import(withr) importFrom(R6,R6Class) importFrom(graphics,abline) @@ -79,3 +77,4 @@ importFrom(graphics,plot) importFrom(graphics,points) importFrom(graphics,strwidth) importFrom(spacefillr,generate_sobol_set) +importFrom(utils,str) diff --git a/NEWS.md b/NEWS.md index 2d8dbedf..5848ec0d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,12 +2,111 @@ # irace (development version) +## Major breaking changes +## New features and improvements +## Fixes + +# irace 4.2 + +## Major breaking changes + + * Recovery (`--recover-file`) has been completely re-implemented. This + version of irace cannot recover files generated by previous versions and + vice versa. + + * Now `blockSize` multiplies `elitistNewInstances`, which should ensure that + irace evaluates instances in blocks. + + * Several improvements in `ablation()` may result in different ablation + results using this version of irace. Also, the log generated by `ablation()` + has some important changes to record more detailed information and it is + more consistent with the log generated by `irace()`. + +## New features and improvements + + * When `targetRunner` or `targetEvaluator` are implemented in R, using random + numbers within those functions does not affect the sequence of random + numbers within irace. + + * `psRace()` gains a `psrace_logFile` argument to avoid overwriting `scenario$logFile`. + + * `psRace()` makes better use of the budget remaining. + + * New scenario option `cappingAfterFirstTest` + (`--capping-after-first-test`). If set to 1, elimination due to capping only + happens after `firstTest` instances are seen (issue #78, suggested by Nguyen Dang). + + * `getConfigurationById()` now returns configurations in the same order + (including repeated values) of the IDs passed as argument. + + * `ablation(type="full")` will evaluate all intermediate configurations on all + instances at once. This makes better use of parallel resources but will not + report any information about progress after each instance. + + * `ablation()` now works better with dependent-domain parameters and with + complicated conditions. Ablation will not generate intermediate + configurations that are forbidden. + + * The ACOTSP example in the user-guide now uses the ACOTSPQAP code and provides more guidance in case of errors. + +## Fixes + + * Fix #79: irace.exe did not encode the `libPath` correctly in R >= 4.0 on Windows, producing 'unrecognized escape in character string'. Fixed by replacing backslashes with forward slashes and quoting the arguments that may contain spaces. + + * Fix #76: Recovery (`--recover-file`) is working again with a completely new implementation. + + * Fixed documentation of `psRace()`. + + * Fix `psRace()` not saving `psrace_log` in `scenario$logFile`. + + * Fix bug in `psRace()` using more budget than the maximum given. + + * Handle the case when all elites are rejected before starting the race and + `sum(alive) <= minSurvival`. + + * Fix bug in `sampleSobol()` with parameters whose domain depends on a conditional parameter. + + +# irace 4.1 + +## Major breaking changes + + * Recovery (`--recover-file`) is currently broken (issue #76). + + * The `experiment` list passed to the `targetRunner` R function (see + `target_runner_default()`) does not contain an element `switches`. + This element can be obtained from `scenario$parameters$switches`. + + +## New features and improvements + + * Conditions that are always true because they depend on fixed parameters are + replaced by a `TRUE` value to speed up their evaluation. This speed-up is only + measurable with very large and complicated parameter spaces. + + * The options `src` and `target` of `ablation()` (or `--src` and `--target` of + the command-line `ablation` tool) now accept a character string that points + to a file containing a configuration that will be read with + `readConfigurationsFile()`. This allows performing ablation between + configurations not explored before. + +## Fixes + + * Fixed some typos in the user guide. (@sbomsdorf) + + * Fixed several bugs in post-selection including `Error in if (any(left >= 0L)) { : missing value where TRUE/FALSE needed`. + + * Fix bug in `ablation()` not calling `stop_parallel()`. + + +# irace 4.0 + ## Major breaking changes * Requires R version >= 4.0 * Logfiles `*.Rdata` use format version 3, which can only be read by R version >= 3.5. - + * The scenario options `forbiddenFile` and `forbiddenExps` have been removed and will give an error if present. Forbidden configurations are now specified in the parameter space description. See the example in @@ -20,13 +119,13 @@ * The default value of the scenario option `trainInstancesDir` is now `""`. The previous default value of `"./Instances"` often caused confusion to users not using files as training instances. - + * The `scenario` object now includes the `parameters` object. Thus functions such as `irace()`, which previously took as arguments both `scenario` and `parameters`, now only take `scenario`. This also means that the log file `irace.Rdata` does not contain a separate `parameters` element since this element can now be found within `scenario`. - + * The default value of the scenario option `softRestartThreshold` is now 0.0001 and does not depend on `digits`. @@ -38,13 +137,13 @@ different locations and each executable will use its corresponding version. The correct location can be verified by looking at the line `"installed at:"` printed in the output. - - * Adaptive capping is now enabled by default if `maxTime > 0` and `maxBound > 0`. + + * Adaptive capping is now enabled by default if `maxTime > 0` and `boundMax > 0`. It can be disabled with `--capping 0` in the command-line options or `capping=0` in the scenario options. See [Pérez-Cáceres et al. (2017)](https://iridia-ulb.github.io/references/#PerLopHooStu2017:lion) for details. (Leslie Pérez Cáceres, Manuel López-Ibáñez) * The scenario option `targetRunnerLauncherArgs`, introduced in version 3.5, - was removed and replaced by `targetCmdline`, which is more flexible (fixes #38). + was removed and replaced by `targetCmdline`, which is more flexible (fixes #38). Please see the user-guide for details. * Command-line options in joined form, given as `"--log-file= --check"`, without @@ -54,54 +153,54 @@ * irace will now give an error if you try to recover from a file generated by a different version of irace, since such attempts typically end up in errors that are difficult to understand. - + * irace warns about using `'&&'` and `'||'` instead of `'&'` and `'|'` in parameter conditions and forbidden expressions. A future version of irace will reject those uses as errors. - + * The internal function `irace.reload.debug()` has been removed. Use `devtools::reload()` instead. - + * The column `"instance"` of the `instancesList` data frame stored in the logFile has been renamed to `"instanceID"`. This data frame should not be accessed directly. Instead use the new function `get_instanceID_seed_pairs()`. - + * Using `maxTime > 0` with `elitist=0` now gives a clear error rather than fail later. (fix #65, reported by @DE0CH) - + * `path_rel2abs()` will not expand symlinks to avoid problems with Python's venv. (fix #64, reported by @DE0CH) - + * Expansion of `'~'` in Windows now follows the definition of `fs::path_expand()` rather than `base::path.expand()`. - + * irace is now more strict in enforcing runtime bounds given with `scenario$boundMax` and will stop with an error if the `target-runner` reports a runtime larger than the given bound. * All functions that contained a period (`'.'`) in the name have been renamed to use `'_'` instead. - + * The periods (`'.'`) in the arguments of `scenario$targetRunnerParallel` and `scenario$targetEvaluator` have also been replaced by `'_'`. - + * The environment `.irace` that was available in the log file under `iraceResults$state$.irace` is replaced directly by `iraceResults$state`. It contains similar information but some entries have been renamed. For example, the `experimentLog` data frame is now called `experiment_log` and it is a [`data.table`](https://r-datatable.com). - + * The interface of `psRace()` has been simplified. - + * `irace` will automatically execute a post-selection race (`psRace()`) using any remaining budget (currently only when `maxTime == 0`). To disable this behavior, set the scenario option `postselection` (`--postselection`) to `0`. - + ## New features and improvements * `sampleUniform()` and `sampleModel()` are significantly faster thanks to using [`data.table`](https://r-datatable.com). * Initial configurations are sampled using Sobol low-discrepancy sequences using `spacefillr::generate_sobol_set()`. This should provide a better initial distribution of parameter values. - + * Parameter spaces can be constructed programmatically using `parametersNew()`. See the documentation for details. * Ablation will report configurations that produced the same results, which @@ -115,7 +214,7 @@ instance used in `"full"` ablation. It replaces the previous parameter `n_instances`, whose definition was more difficult to use correctly. - * Matrix operations are faster thanks to `matrixStats`. + * Matrix operations are faster thanks to `matrixStats`. * New scenario option `blockSize` for defining blocks of instances. Configurations are only eliminated after evaluating a complete block and @@ -128,7 +227,7 @@ * New scenario option `targetRunnerTimeout`: Timeout in seconds of any `targetRunner` call (only applies to `target-runner` executables not to R functions). - + * `plotAblation()` has several new options: - `type='rank'` to plot ranks per instance instead of raw cost values. - `n` to limit the number of parameters shown in the plot. @@ -141,7 +240,7 @@ (Deyao Chen) * New functions `read_ablogfile()`, `has_testing_data()`, `irace_summarise()`. - + * New functions `get_random_seed()`, `set_random_seed()`, `restore_random_seed()` useful for writing `targetRunner` functions in R. @@ -155,26 +254,26 @@ * `readScenario()` (and command-line irace) do not require a `scenario.txt` file. (Contributed by @DE0CH) - + * `read_pcs_file()` now supports forbidden configurations. - + * When testing, `irace` now prints the random seed used for each instance as an additional column. - + * The package provides a new executable `target-runner-dummy` (or `target-runner-dummy.exe` in Windows) for the purposes of testing. It may also be useful for understanding the typical setup of `irace`. - + * New scenario option `minExperiments` to set a minimum budget of runs. (proposed by @Saethox, fixes #58) * New function `multi_irace()` for executing multiple runs of irace with the - same or different scenarios and parameters, possibly in parallel. + same or different scenarios and parameters, possibly in parallel. (Contributed by @Saethox) ## Fixes * `ablation_cmdline()` and `plotAblation()` no longer create an empty `Rplots.pdf` file when specifying an output PDF file. - + * Fix #66: when using `maxTime > 0`, irace estimates the time per run by executing 2 configurations on `firstTest` instances and adjusts `boundMax` to not go over `budgetEstimation`. This may result in a smaller `boundMax` @@ -185,10 +284,10 @@ * Fix #55: Configurations provided may use `` in addition to `NA` to denote the missing value of a disabled parameter. (Manuel López-Ibáñez, reported by @TheIronBorn) - + * Fix #44: irace now will give an error if the domain of real-valued (r) parameters would change depending on the value of `'digits'`. The solution - is to increase the value of `'digits'` or adjust the domain. + is to increase the value of `'digits'` or adjust the domain. (Manuel López-Ibáñez, reported by @mb706) * If scenario option `targetRunnerParallel` is set, irace no longer tries to @@ -198,25 +297,25 @@ * irace no longer sets `option(error=utils::recover())` in debug mode to avoid issues when calling irace from Python. The user can set this if desired. - - * Fix bug failing to restart with parameters that have dependent domains. - + + * Fix bug failing to restart with parameters that have dependent domains. + * Fix bug with `sampleInstances=FALSE` that could re-evaluate the same (instance, seed) pair more than once. - + * Fix bug when using `targetRunnerLauncher` and `targetRunner` contain whitespace. * Fix bug in `ablation_cmdline()` about missing `scenario` object. - + * `ablation()` will now save and restore the previous random seed. - + * `ablation()` will detect if the logfile (e.g., `irace.Rdata`) is incomplete. - + * `readConfigurationsFile()` now handles parameters with dependent domains. - + * Fix #71: Ensure `".ID."` is the first column in `checkTargetFiles()` (Manuel López-Ibáñez, reported by @ivan1arriola) - - + + # irace 3.5 ## New features and improvements @@ -230,7 +329,7 @@ * The package now provides an `ablation` executable (`ablation.exe` in Windows) that makes easier to perform ablation analysis without having any R knowledge. - + * The interface to functions `ablation()` and `plotAblation()` has been simplified. The `ablation()` function now allows overriding scenario settings. The `plotAblation()` function will not create the plot if the @@ -240,18 +339,18 @@ * The argument `n.instances` of `ablation()` has been renamed to `n_instances` and it is now a factor that multiplies `scenario$firstTest`. (Manuel López-Ibáñez) - + * New command-line option `--quiet` to run without producing any output except errors (also available as a scenario option). (Manuel López-Ibáñez) * New command-line option `--init` to initialize a scenario. (Deyao Chen) - + * Added support for HTCondor cluster framework to `--batchmode`. (Filippo Bistaffa) * `--check` now also check the contents of `configurationsFile` and runs - configurations provided via `initConfigurations`. + configurations provided via `initConfigurations`. (Manuel López-Ibáñez, reported by Andreea Avramescu) * New scenario options `targetRunnerLauncher` and `targetRunnerLauncherArgs` @@ -261,12 +360,12 @@ * New scenario option `minMeasurableTime`. (Manuel López-Ibáñez) - + * An error is produced if a variable set in the scenario file is not known to irace. If your scenario file contains R code, then use variable names beginning with a dot `'.'`, which will be ignored by irace. (Manuel López-Ibáñez) - + * Plotting functions have been moved to the new package [iraceplot](https://auto-optimization.github.io/iraceplot/). In particular, `configurationsBoxplot()` is replaced by `iraceplot::boxplot_training()` and @@ -287,7 +386,7 @@ * New functions `testing_fromlog()` and `testing_fromfile()` for independently executing the testing phase. The function `testing.main()` was removed as it - is superseded by the new ones. + is superseded by the new ones. (Manuel López-Ibáñez) * New function `read_logfile()` to easily read the log file produced by irace. @@ -301,23 +400,23 @@ * Generating the file `irace.Rdata` may be disabled by setting `logFile=""`. (Manuel López-Ibáñez, reported by Johann Dreo) - + * `path_rel2abs()` and `checkParameters()` are now exported so that other packages may use them. (Manuel López-Ibáñez) - + * `path_rel2abs()` also searches in system paths. (Manuel López-Ibáñez) * `readConfigurationsFile()` will now detect duplicated configurations and error. (Manuel López-Ibáñez) - + * The interface to functions `getFinalElites()`, `getConfigurationById()` and `getConfigurationByIteration()` has been simplified. - + * The package provides a `irace.sindef` file that may be used for building a standalone container of irace using Singularity. See the `README.md` file for instructions. (Contributed by Johann Dreo) - + * New example `examples/target-runner-python/target-runner-python-win.bat` contributed by Levi Ribeiro. @@ -340,8 +439,8 @@ (Manuel López-Ibáñez, reported by Marcelo de Souza) * Fixes to the Matlab example. (Manuel López-Ibáñez) - - * The default of `testType` is now set to `t-test` when capping is enabled. + + * The default of `testType` is now set to `t-test` when capping is enabled. (Manuel López-Ibáñez, reported by Jovana Radjenovic) * Fix various issues in the user guide. @@ -349,14 +448,14 @@ * Remove duplicated elites. (Manuel López-Ibáñez, reported by Federico Naldini) - + * Fix (#7): warnings with partial matched parameters. (Manuel López-Ibáñez, reported by Marc Becker) - + * Fix (#10): wrong assert with `elitist=0`. (Manuel López-Ibáñez) * Fix (#12): irace can be run with [FastR](https://www.graalvm.org/22.1/docs/getting-started/#run-r). - + * Fix (#13): Maximum number configurations immediately rejected reached. (Manuel López-Ibáñez) @@ -368,13 +467,13 @@ * With `sampleInstances = FALSE`, elitist irace does not change the order of instances already seen. However, if you want to make sure that the order of the instances is enforced, you also need to set `elitistNewInstances=0`. - + * The function `irace.usage()` was removed. It was not really useful for R users as the same result can be obtained by calling `irace.cmdline("--help")`. (Manuel López-Ibáñez) - - + + # irace 3.4.1 (31/03/2020) * `NEWS` converted to markdown. diff --git a/R/aaa.R b/R/aaa.R index 06805958..9b4b4d53 100644 --- a/R/aaa.R +++ b/R/aaa.R @@ -8,7 +8,7 @@ update_package_version <- function() if (is.na(git_rev)) git_rev <- "unknown" git <- Sys.which("git") - if (nchar(git) > 0L && fs::file_exists(".git") + if (git != "" && fs::file_exists(".git") && grepl("[0-9a-z]+$", system2(git, "describe --first-parent --always", stdout = TRUE), perl=TRUE)) { git_rev <- system2(git, "describe --dirty --first-parent --always --exclude '*'", stdout = TRUE) } @@ -22,3 +22,8 @@ update_package_version <- function() update_package_version() # We define this tentatively to avoid: undefined exports: irace_version irace_version <- "unknown" + +.irace_tolerance <- sqrt(.Machine$double.eps) +.irace_minimum_saving_time <- 60 # seconds +# Prefix for printing messages to the user. +.irace_msg_prefix <- "== irace == " diff --git a/R/ablation.R b/R/ablation.R index 4e1da851..2f3e0b19 100644 --- a/R/ablation.R +++ b/R/ablation.R @@ -1,8 +1,8 @@ -.ablation.params.def <- read.table(header=TRUE, stringsAsFactors = FALSE, text=" +.ablation.params.def <- utils::read.table(header=TRUE, stringsAsFactors = FALSE, text=" name ab type short long default description iraceResults 0 p -l --log-file NA 'Path to the (.Rdata) file created by irace from which the \"iraceResults\" object will be loaded.' -src 1 i -S --src 1 'Source configuration ID.' -target 1 i -T --target NA 'Target configuration ID. By default the best configuration found by irace.' +src 1 i -S --src 1 'Source configuration ID or the path to a file containing the configuration.' +target 1 i -T --target NA 'Target configuration ID (by default the best configuration found by irace) or the path to a file containing the configuration.' ab_params 1 s -P --params '' 'Specific parameter names to be used for the ablation (separated with commas). By default use all' type 1 s -t --type 'full' 'Type of ablation to perform: \"full\" will execute each configuration on all \"--n-instances\" to determine the best-performing one; \"racing\" will apply racing to find the best configurations.' nrep 1 i -n --nrep 1 'Number of replications per instance used in \"full\" ablation.' @@ -18,13 +18,15 @@ scenarioFile 0 p -s --scenario NA 'Scenario fil parallel 0 i '' --parallel NA 'Number of calls to targetRunner to execute in parallel. Values 0 or 1 mean no parallelization.' ") +## __VERSION__ below will be replaced by the version defined in R/version.R +## This avoids constant conflicts within this file. cat_ablation_license <- function() { ablation_license <- '#------------------------------------------------------------------------------ # ablation: An implementation in R of Ablation Analysis # Version: __VERSION__ -# Copyright (C) 2020--2022 +# Copyright (C) 2020--2025 # Manuel Lopez-Ibanez # Leslie Perez Caceres # @@ -40,11 +42,11 @@ cat_ablation_license <- function() #' #' Launch [ablation()] with the same command-line options as the command-line #' executable (`ablation.exe` in Windows). -#' -#' @param argv `character()`\cr The arguments -#' provided on the R command line as a character vector, e.g., +#' +#' @param argv `character()`\cr The arguments +#' provided on the R command line as a character vector, e.g., #' `c("-i", "irace.Rdata", "--src", 1)`. -#' +#' #' @details The function reads the parameters given on the command line #' used to invoke R, launches [ablation()] and possibly [plotAblation()]. #' @@ -54,14 +56,15 @@ cat_ablation_license <- function() #' ``` #' @return A list containing the following elements: #' \describe{ -#' \item{configurations}{Configurations tested in the ablation.} -#' \item{instances}{A matrix with the instances used in the experiments. First column has the -#' instances IDs from \code{iraceResults$scenario$instances}, second column the seed assigned to the instance.} +#' \item{allConfigurations}{Configurations tested in the ablation.} +#' \item{state}{State of the ablation process.} #' \item{experiments}{A matrix with the results of the experiments (columns are configurations, rows are instances).} #' \item{scenario}{Scenario object with the settings used for the experiments.} #' \item{trajectory}{IDs of the best configurations at each step of the ablation.} #' \item{best}{Best configuration found in the experiments.} +#' \item{complete}{`TRUE` if the ablation process was completed.} #' } +#' #' @seealso [plotAblation()] [ablation()] #' @examples #' ablation_cmdline("--help") @@ -84,7 +87,7 @@ ablation_cmdline <- function(argv = commandArgs(trailingOnly = TRUE)) } if (!is.null(parser$readArg (short = "-v", long = "--version"))) { - print(citation(package="irace")) + print(utils::citation(package="irace")) return(invisible(NULL)) } @@ -95,12 +98,12 @@ ablation_cmdline <- function(argv = commandArgs(trailingOnly = TRUE)) stop("Unknown command-line options: ", paste(parser$argv, collapse = " ")) if (is.null(params$iraceResults)) { - irace.error("You must provide the path to the '.Rdata' file that contains the 'iraceResults' object generated by irace.") + irace_error("You must provide the path to the '.Rdata' file that contains the 'iraceResults' object generated by irace.") return(invisible(NULL)) } iraceResults <- read_logfile(params$iraceResults) if (is.null(params$old_path) != is.null(params$new_path)) { - irace.error("To update paths you must provide both --old-path and --new-path.") + irace_error("To update paths you must provide both --old-path and --new-path.") return(invisible(NULL)) } else if (!is.null(params$old_path)) { iraceResults$scenario <- scenario_update_paths(iraceResults$scenario, params$old_path, params$new_path) @@ -113,21 +116,22 @@ ablation_cmdline <- function(argv = commandArgs(trailingOnly = TRUE)) scenario$seed <- NULL } for (p in c("execDir", "parallel")) { - if (!is.null(params[[p]])) scenario[[p]] <- params[[p]] + if (!is.null(params[[p]])) + scenario[[p]] <- params[[p]] } - + if (is_null_or_empty_or_na(trim(params$ablationLogFile))) { params$ablationLogFile <- NULL } else { params$ablationLogFile <- path_rel2abs(params$ablationLogFile) } - + if (!is.null(params$ab_params)) params$ab_params <- trimws(strsplit(params$ab_params, ",", fixed=TRUE)[[1L]]) # The shell may introduce extra quotes, remove them. params$plot_type <- trimws(gsub("[\"']", "", params$plot_type)) - + # We want to select elements that actually appear in params, otherwise we get NA names. ablation_params <- intersect(.ablation.params.def[.ablation.params.def$ab == 1, "name", drop=TRUE], names(params)) @@ -136,62 +140,106 @@ ablation_cmdline <- function(argv = commandArgs(trailingOnly = TRUE)) params[ablation_params], scenario)) if (!is.null(params[["plot"]]) || base::interactive()) { - plotAblation(ablog, pdf_file = params[["plot"]], type = params$plot_type) + plotAblation(ablog, pdf_file = params[["plot"]], type = params$plot_type) } invisible(ablog) } -## This function fixes dependent parameters when a parameter value has been -## changed. -fixDependenciesWithReference <- function(configuration, ref_configuration, parameters) +fix_dependents <- function(parameters, dep_names, new_conf, final_conf, changed) { - # Search parameters that need a value - changed <- c() - for (pname in parameters$names_variable) { - # If dependent parameter has been activated, set the value of the reference. - if (is.na(configuration[[pname]]) && conditionsSatisfied(parameters$conditions[[pname]], configuration)) { - if (!is.null(ref_configuration)) { - configuration[[pname]] <- ref_configuration[[pname]] - } - changed <- c(changed, pname) - # MANUEL: Why do we need to recurse here? - aux <- fixDependenciesWithReference(configuration=configuration, ref_configuration=ref_configuration, parameters) - changed <- c(changed, aux$changed) - configuration <- aux$configuration + # ... check every dependent parameter... + for (dep in dep_names) { + # If the conditions are satisfied and it doesn't have a value but it + # should, then change it. If the conditions are not satisfied and it + # has a value but it shouldn't, then change it. + ok <- conditionsSatisfied(parameters$conditions[[dep]], new_conf) + change <- ok == is.na(new_conf[[dep]]) + dep_param <- parameters$get(dep) + if (change) { + # If it doesn't have a value but it should, take it from final. + if (ok) { + if (dep_param[["is_dependent"]]) { + # If the parameter is domain-dependent, then: + if (anyNA(new_conf[, .SD, .SDcols=all.vars(dep_param$domain)])) { + # If it depends on a parameter that is disabled, then this is disabled, so no need to change. + next + } else if (is.na(final_conf[[dep]]) # If the final value is NA, we cannot fix it, so skip it. + # If parameter X enables dep but dep has domain that depends on + # parameter Y, then Y in new_conf may have a value that makes + # final_conf[[dep]] invalid. We cannot fix this, so skip. + || !is_within_dependent_bound(dep_param, new_conf, value = final_conf[[dep]])) + return(NULL) + } + set(new_conf, j = dep, value = final_conf[[dep]]) + } else { + set(new_conf, j = dep, value = NA) + } + changed <- c(changed, dep) + } else if (dep_param[["is_dependent"]] && !is.na(new_conf[[dep]])) { + # The condition may not have changed but the domain may depend on a parameter that has been changed. + if (anyNA(new_conf[, .SD, .SDcols=all.vars(dep_param$domain)])) { + set(new_conf, j = dep, value = NA) + changed <- c(changed, dep) + } else if (!is_within_dependent_bound(dep_param, new_conf, value = new_conf[[dep]])) { + # If the current value is not within bound and the final value is NA, we cannot fix it, so skip it. + if (is.na(final_conf[[dep]]) + # If parameter X enables dep but dep has domain that depends on + # parameter Y, then Y in new_conf may have a value that makes + # final_conf[[dep]] invalid. We cannot fix this, so skip. + || !is_within_dependent_bound(dep_param, new_conf, value = final_conf[[dep]])) + return(NULL) + set(new_conf, j = dep, value = final_conf[[dep]]) + changed <- c(changed, dep) + } } } - list(configuration=configuration, changed=changed) + changed } -## Function that generates the configurations of the ablation path +## Function that generates the configurations of the ablation path ## between initial_configuration and final_configuration. ## parameters can be selected by specifying them in para.names. -generateAblation <- function(initial_configuration, final_configuration, - parameters, param_names = NULL) -{ +generate_ablation <- function(initial_configuration, final_configuration, + parameters, param_names) +{ # Only change variable parameters - if (is.null(param_names)) - param_names <- parameters$names_variable - else - param_names <- setdiff(param_names, parameters$names_fixed) - - configurations <- NULL + param_names <- setdiff(param_names, parameters$names_fixed) + # Follow the hierarchy order. + hierarchy <- sort(parameters$hierarchy) + param_names <- intersect(names(parameters$hierarchy), param_names) + configurations <- list() changed_params <- list() + # skip if the value is the same or NA (not active) + skip <- (initial_configuration[param_names] == final_configuration[param_names]) + skip <- skip[1L, ] # Drop to a vector with names + skip <- skip | is.na(skip) for (pname in param_names) { - # Check if parameter is active. - if (!conditionsSatisfied(parameters$conditions[[pname]], initial_configuration)) next - # Check value is different in the initial and final configuration and if - # so, change the value. - if (initial_configuration[[pname]] == final_configuration[[pname]]) next - new_configuration <- initial_configuration - new_configuration[[pname]] <- final_configuration[[pname]] - # Set newly activated parameters if needed. - aux <- fixDependenciesWithReference(new_configuration, final_configuration, parameters) - new_configuration <- aux[["configuration"]] - changed_params[[length(changed_params) + 1L]] <- c(pname, aux[["changed"]]) - new_configuration[[".PARENT."]] <- initial_configuration[[".ID."]] - configurations <- rbind.data.frame(configurations, new_configuration) + # If parameter is not active, it will only change if its condition becomes true. + if (skip[[pname]]) next + param <- parameters$get(pname) + # If the parameter is domain-dependent, then we can only change its + # value if the new value is within the domain. + if (param[["is_dependent"]] + && !is_within_dependent_bound(param, initial_configuration, value = final_configuration[[pname]])) + next + new_configuration <- as.data.table(initial_configuration) + set(new_configuration, j = pname, value = final_configuration[[pname]]) + changed <- pname + # If other parameters depend on this one then... + if (any(sapply(parameters$depends, function(x) any(pname %in% x)))) { + changed <- fix_dependents(parameters, dep_names = names(hierarchy)[hierarchy > hierarchy[[pname]]], + new_conf = new_configuration, final_conf = final_configuration, changed = changed) + if (is.null(changed)) + next + } + new_configuration <- filter_forbidden(new_configuration, parameters$forbidden) + if (nrow(new_configuration) == 0L) next + changed_params <- c(changed_params, list(changed)) + configurations <- c(configurations, list(new_configuration)) } + configurations <- rbindlist(configurations) + set(configurations, j = ".PARENT.", value = configurations[[".ID."]]) + setDF(configurations) rownames(configurations) <- NULL list(configurations=configurations, changed_params=changed_params) } @@ -223,14 +271,14 @@ ab_generate_instances <- function(race_state, scenario, nrep, type, instancesFil stop("'nrep' has no effect when type == 'racing'") if (nrep > 1L && scenario$deterministic) stop("'nrep > 1' does not make sense with a deterministic scenario") - + if (instancesFile == "test") { scenario$instances <- scenario$testInstances } else if (instancesFile != "train") { scenario$instances <- readInstances(instancesFile = path_rel2abs(instancesFile)) } n_inst <- length(scenario$instances) - if (type == "full" && n_inst * nrep == 1) + if (type == "full" && n_inst * nrep == 1L) stop("'nrep' must be larger than 1 when type == 'full' and a single instance") generateInstances(race_state, scenario, n_inst * nrep) @@ -251,19 +299,19 @@ ab_generate_instances <- function(race_state, scenario, nrep, type, instancesFil #' @description Ablation is a method for analyzing the differences between two configurations. #' #' @inheritParams has_testing_data -#' @param src,target (`integer(1)`) Source and target configuration IDs. By default, the first configuration ever evaluated (ID 1) is used as `src` and the best configuration found by irace is used as target. -#' @param ab_params Specific parameter names to be used for the ablation. They must be in `parameters$names`. By default, use all parameters. -#' @param type Type of ablation to perform: `"full"` will execute each configuration on all `n_instances` to determine the best-performing one; `"racing"` will apply racing to find the best configurations. -#' @param nrep (`integer(1)`) Number of replications per instance used in `"full"` ablation. When `nrep > 1`, each configuration will be executed `nrep` times on each instance with different random seeds. -#' @param seed (`integer(1)`) Integer value to use as seed for the random number generation. -#' @param ablationLogFile (`character(1)`) Log file to save the ablation log. If `NULL`, the results are not saved to a file. -#' @param instancesFile (`character(1)`) Instances file used for ablation: `'train'`, `'test'` or a filename containing the list of instances. +#' @param src,target `integer(1)|character(1)`\cr Source and target configuration IDs. By default, the first configuration ever evaluated (ID 1) is used as `src` and the best configuration found by irace is used as target. If the argument is a string, it is interpreted as the path to a file, with the format specified by [readConfigurationsFile()], that contains the configuration. +#' @param ab_params `character()`\cr Specific parameter names to be used for the ablation. They must be in `parameters$names`. By default, use all parameters. +#' @param type `"full"|"racing"`\cr Type of ablation to perform: `"full"` will execute each configuration on all `n_instances` to determine the best-performing one; `"racing"` will apply racing to find the best configurations. +#' @param nrep `integer(1)`\cr Number of replications per instance used in `"full"` ablation. When `nrep > 1`, each configuration will be executed `nrep` times on each instance with different random seeds. +#' @param seed `integer(1)`\cr Integer value to use as seed for the random number generation. +#' @param ablationLogFile `character(1)`\cr Log file to save the ablation log. If `NULL`, the results are not saved to a file. +#' @param instancesFile `character(1)`\cr Instances file used for ablation: `'train'`, `'test'` or a filename containing the list of instances. #' @param ... Further arguments to override scenario settings, e.g., `debugLevel`, `parallel`, etc. #' #' @references #' C. Fawcett and H. H. Hoos. Analysing differences between algorithm #' configurations through ablation. Journal of Heuristics, 22(4):431–458, 2016. -#' +#' #' @inherit ablation_cmdline return #' @seealso [plotAblation()] [ablation_cmdline()] #' @examples @@ -289,33 +337,36 @@ ablation <- function(iraceResults, src = 1L, target = NULL, instancesFile="train", ...) { # Input check - if (missing(iraceResults) || is.null(iraceResults)) + if (missing(iraceResults) || is.null(iraceResults)) stop("You must provide an 'iraceResults' object generated by irace or the path to the '.Rdata' file that contains this object.") type <- match.arg(type) - if (!is.null(ablationLogFile)) + if (!is.null(ablationLogFile)) { file.check(ablationLogFile, writeable = TRUE, text = 'ablationLogFile') - + } + save_ablog <- function(complete) { ablog <- list(changes = changes, - configurations = all_configurations, - experiments = results, - instances = race_state$instances_log, - scenario = scenario, - trajectory = trajectory, - best = best_configuration, - complete = complete) - if (!is.null(ablationLogFile)) save(ablog, file = ablationLogFile, version = 3L) - ablog + # This name must match the one used in the irace log. + allConfigurations = as.data.frame(all_configurations), + experiments = experiments, + state = race_state, + scenario = scenario, + trajectory = trajectory, + best = best_configuration, + complete = complete) + if (!is.null(ablationLogFile)) + save(ablog, file = ablationLogFile, version = 3L) + invisible(ablog) } # Load the data of the log file. iraceResults <- read_logfile(iraceResults) log_version <- get_log_clean_version(iraceResults) if (log_version < "3.9.0") - irace.error("The version of the logfile (", log_version, ") is too old for this version of ablation") - + irace_error("The version of the logfile (", log_version, ") is too old for this version of ablation") + if (is.null(iraceResults$state$completed) || length(iraceResults$state$completed) != 1L || iraceResults$state$completed == "Incomplete") stop("The 'iraceResults' logfile seems to belong to an incomplete run of irace.") @@ -329,17 +380,41 @@ ablation <- function(iraceResults, src = 1L, target = NULL, race_state <- RaceState$new(scenario) # Generate instances scenario$instances <- ab_generate_instances(race_state, scenario, nrep, type, instancesFile) + parameters <- scenario$parameters + + # Process src and target. + if (is.character(src) && is.na(suppressWarnings(as.integer(src)))) { + src_configuration <- readConfigurationsFile(src, parameters) + if (nrow(src_configuration) != 1L) { + stop("Argument of src=\"", src, "\" contains more than one configuration!") + } + src <- 1L + max(iraceResults$allConfigurations$.ID.) + src_configuration$.ID. <- src + src_configuration$.PARENT. <- NA_integer_ + iraceResults$allConfigurations <- rbind.data.frame(iraceResults$allConfigurations, src_configuration) + # FIXME: Check for duplicates + } else if (src %not_in% iraceResults$allConfigurations[[".ID."]]) + stop("Source configuration ID (", src, ") cannot be found!") if (is.null(target)) target <- iraceResults$iterationElites[length(iraceResults$iterationElites)] - if (src %not_in% iraceResults$allConfigurations[[".ID."]]) - stop("Source configuration ID (", src, ") cannot be found!") - if (target %not_in% iraceResults$allConfigurations[[".ID."]]) + else if (is.character(target) && is.na(suppressWarnings(as.integer(target)))) { + target_configuration <- readConfigurationsFile(target, parameters) + if (nrow(target_configuration) != 1L) { + stop("Argument of target=\"", target, "\" contains more than one configuration!") + } + target <- 1L + max(iraceResults$allConfigurations$.ID.) + target_configuration$.ID. <- target + target_configuration$.PARENT. <- NA_integer_ + iraceResults$allConfigurations <- rbind.data.frame(iraceResults$allConfigurations, target_configuration) + # FIXME: Check for duplicates + } else if (target %not_in% iraceResults$allConfigurations[[".ID."]]) stop("Target configuration ID (", target, ") cannot be found!") + if (src == target) stop("Source and target configuration IDs must be different!") - - irace.note ("Starting ablation from ", src, " to ", target, "\n# Seed: ", race_state$seed, "\n") + + irace_note ("Starting ablation from ", src, " to ", target, "\n# Seed: ", race_state$seed, "\n") cat("# Source configuration (row number is ID):\n") src_configuration <- iraceResults$allConfigurations[src, , drop = FALSE] configurations_print(src_configuration) @@ -347,162 +422,119 @@ ablation <- function(iraceResults, src = 1L, target = NULL, target_configuration <- iraceResults$allConfigurations[target, , drop = FALSE] configurations_print(target_configuration) - parameters <- scenario$parameters # Select the parameters used for ablation if (is.null(ab_params)) { ab_params <- parameters$names } else if (!all(ab_params %in% parameters$names)) { - irace.error("Some of the parameters provided (", paste0(setdiff(ab_params, parameters$names), collapse=", "), + irace_error("Some of the parameters provided (", paste0(setdiff(ab_params, parameters$names), collapse=", "), ") are not defined in the parameter space.") } # Select parameters that are different in both configurations neq_params <- which(src_configuration[,ab_params] != target_configuration[,ab_params]) - if (length(neq_params) == 0L) - irace.error("src and target configurations are equal considering the parameters selected.\n") + if (length(neq_params) == 0L) + irace_error("src and target configurations are equal considering the parameters selected.\n") param_names <- colnames(src_configuration[,ab_params])[neq_params] - + # FIXME: Do we really need to override the ID? - src_configuration$.ID. <- best_id <- 1L - best_configuration <- all_configurations <- src_configuration - + src_configuration[[".ID."]] <- 1L + target_configuration[[".ID."]] <- 2L + all_configurations <- rbindlist(list(src_configuration, target_configuration), use.names=TRUE) + + race_state$start_parallel(scenario) + on.exit(race_state$stop_parallel(), add = TRUE) + # Execute source and target configurations. ## FIXME: We may already have these experiments in the logFile! - experiments <- createExperimentList(configurations = rbind(src_configuration, target_configuration), - parameters = parameters, - instances = scenario$instances, - instances_ID = race_state$instances_log[["instanceID"]], - seeds = race_state$instances_log[["seed"]], - bounds = scenario$boundMax) - irace.note("Executing source and target configurations on the given instances * nrep (", nrow(race_state$instances_log), ")...\n") - - race_state$start_parallel(scenario) - on.exit(race_state$start_parallel(scenario), add = TRUE) - target.output <- execute_experiments(race_state, experiments, scenario) - if (!is.null(scenario$targetEvaluator)) - target.output <- execute_evaluator (race_state$target_evaluator, experiments, scenario, target.output, - src_configuration) - # Save results - output <- sapply(target.output, getElement, "cost") - results <- matrix(NA, ncol = 1L, nrow = nrow(race_state$instances_log), - dimnames = list(seq_nrow(race_state$instances_log), 1L)) - results[,1L] <- output[seq_nrow(race_state$instances_log)] - lastres <- output[(nrow(race_state$instances_log)+1L):(2L * nrow(race_state$instances_log))] + irace_note("Executing source and target configurations on the given instances * nrep (", nrow(race_state$instances_log), ")...\n") + experiments <- do_experiments(race_state, + configurations = rbind.data.frame(src_configuration, target_configuration), + ninstances = nrow(race_state$instances_log), scenario = scenario, iteration = 0L) + irace_assert(ncol(experiments) == 2L) step <- 1L # Define variables needed trajectory <- 1L names(trajectory) <- "source" # FIXME: changes should only store the changed parameters. changes <- list() - ablog <- save_ablog(complete = FALSE) + best_configuration <- src_configuration + save_ablog(complete = FALSE) while (length(param_names) > 1L) { # Generate ablation configurations - cat("# Generating configurations (row number is ID):", param_names,"\n") - ab_aux <- generateAblation(best_configuration, target_configuration, parameters, - param_names) + cat("# Generating configurations (row number is ID):", param_names, "\n") + ab_aux <- generate_ablation(best_configuration, target_configuration, parameters, + param_names) aconfigurations <- ab_aux$configurations - if (is.null(aconfigurations)) { + if (nrow(aconfigurations) == 0L) { cat("# Stopping ablation, no parameter change possible.\n") break } ## FIXME: We may already have these configurations in the logFile! - # New configurations ids - ## FIXME: These should be generated with respect to the logFile to make - ## sure we don't have duplicate IDs. - aconfigurations[[".ID."]] <- max(0L, all_configurations[[".ID."]]) + seq_nrow(aconfigurations) + # New configurations IDs + aconfigurations[[".ID."]] <- max(all_configurations[[".ID."]]) + seq_nrow(aconfigurations) configurations_print(aconfigurations, metadata = FALSE) - all_configurations <- rbind(all_configurations, aconfigurations) - - # Set variables for the racing procedure - if (scenario$capping) { - # For using capping we must set elite data - elite_data <- list(experiments = results[,best_configuration[[".ID."]], drop=FALSE]) - race_conf <- rbind(best_configuration, aconfigurations) - race_state$next_instance <- nrow(race_state$instances_log) + 1L + all_configurations <- rbindlist(list(all_configurations, aconfigurations), use.names=TRUE) + + irace_note("Ablation (", type, ") of ", nrow(aconfigurations), + " configurations on ", nrow(race_state$instances_log), " instances (this may take a while ...).\n") + if (type == "full") { + step_experiments <- do_experiments(race_state, + configurations = aconfigurations, + ninstances = nrow(race_state$instances_log), + scenario = scenario, iteration = step) } else { - #LESLIE: for now we apply the non-elitis irace when type=="racing" - # we should define what is the standard - elite_data <- NULL - race_conf <- aconfigurations - scenario$elitist <- FALSE + # Set variables for the racing procedure + # FIXME: what about blockSize? race_state$next_instance <- 1L + race_output <- elitist_race(race_state, + maxExp = nrow(aconfigurations) * nrow(race_state$instances_log), + minSurvival = 1L, + elite_data = NULL, + configurations = aconfigurations, + scenario = scenario, + elitist_new_instances = 0L) + set(race_output$experiment_log, j = "iteration", value = step) + race_state$experiment_log <- + rbindlist(list(race_state$experiment_log, race_output$experiment_log), + use.names=TRUE) + step_experiments <- race_output$experiments } - - irace.note("Ablation (", type, ") of ", nrow(aconfigurations), - " configurations on ", nrow(race_state$instances_log), " instances.\n") - # Force the race to see all instances in "full" mode - # FIXME: Full mode should simply evaluate all instances in parallel. - if (type == "full") scenario$firstTest <- nrow(race_state$instances_log) - # FIXME: what about blockSize? - race_output <- elitist_race(race_state, - maxExp = nrow(aconfigurations) * nrow(race_state$instances_log), - minSurvival = 1L, - elite.data = elite_data, - configurations = race_conf, - scenario = scenario, - elitist_new_instances = 0L) - results <- merge_matrix(results, race_output$experiments) - - # Save log - ablog <- save_ablog(complete = FALSE) - + experiments <- merge_matrix(experiments, step_experiments) + + save_ablog(complete = FALSE) # Get the best configuration based on the criterion of irace - # MANUEL: Doesn't race_output already give you all this info??? - cranks <- overall_ranks(results[,aconfigurations[[".ID."]],drop=FALSE], test = scenario$testType) + cranks <- overall_ranks(experiments[, aconfigurations[[".ID."]], drop=FALSE], test = scenario$testType) best_id <- which.min(cranks)[1L] - # cand.mean <- colMeans2(results[,aconfigurations$.ID.,drop=FALSE], na.rm=TRUE) changes[[step]] <- ab_aux$changed_params best_change <- changes[[step]][[best_id]] trajectory <- c(trajectory, aconfigurations[[".ID."]][best_id]) - - # Report best - # FIXME: This ID does not actually match the configuration ID - # The race already reports the best. + + # Report best. cat("# Best changed parameters:\n") for (i in seq_along(best_change)) { cat("#", best_change[i], ":", best_configuration[,best_change[i]], "->", aconfigurations[best_id, best_change[i]], "\n") } - best_configuration <- aconfigurations[best_id,,drop=FALSE] best_id <- best_configuration[[".ID."]] param_names <- param_names[param_names %not_in% best_change] step <- step + 1L } - - # Add last configuration and its results - # FIXME: This may be overriding the ID of an existing configuration!!! - target_configuration[[".ID."]] <- max(all_configurations[[".ID."]]) + 1L - all_configurations <- rbind(all_configurations, target_configuration) - results <- cbind(results, matrix(lastres, ncol = 1L, - dimnames=list(seq_nrow(race_state$instances_log), - target_configuration[[".ID."]]))) + trajectory <- c(trajectory, target_configuration[[".ID."]]) - # Get the overall best - cranks <- overall_ranks(results[,trajectory, drop=FALSE], test = scenario$testType) + cranks <- overall_ranks(experiments[,trajectory, drop=FALSE], test = scenario$testType) best_id <- which.min(cranks)[1L] - ## FIXME: At this point, the rownames of all_configurations does not match - ## all_configurations[[".ID."]] That is confusing and a potential source of - ## bugs. Instead of fixing it here, we should not generate the discrepancy - ## ever. - best_configuration <- all_configurations[trajectory[best_id],,drop=FALSE] - irace.note("Final best configuration:\n") + setDF(all_configurations, rownames = all_configurations[[".ID."]]) + best_configuration <- all_configurations[trajectory[best_id], , drop=FALSE] + irace_note("Final best configuration:\n") configurations_print(best_configuration) # Check for duplicated results: - report_duplicated_results(results, all_configurations) + report_duplicated_results(experiments, all_configurations) - # LESLIE: If we use racing we can have a matrix of results that is not - # complete, how should we do the plots? - # MANUEL: Do not plot anything that was discarded - save_ablog(complete = TRUE) } -substrRight <- function(x, n){ - substr(x, nchar(x)-n+1, nchar(x)) -} - ablation_labels <- function(trajectory, configurations) { configurations <- removeConfigurationsMetaData(configurations[trajectory, , drop = FALSE]) @@ -513,8 +545,8 @@ ablation_labels <- function(trajectory, configurations) current <- configurations[i, , drop = FALSE] # Select everything that is NOT NA now and was different or NA before. select <- !is.na(current) & (is.na(last) | (current != last)) - irace.assert(!anyNA(select)) - labels[i] <- paste0(substrRight(param_names[select], 10), "=", current[, select], collapse = "\n") + irace_assert(!anyNA(select)) + labels[i] <- paste0(param_names[select], "=", current[, select], collapse = "\n") last <- current } labels @@ -557,25 +589,30 @@ plotAblation <- function (ablog, pdf_file = NULL, width = 20, type <- trimws(unlist(strsplit(type, ",", fixed=TRUE))) type <- match.arg(type, several.ok = TRUE) if (missing(ylab) && ("rank" %in% type)) ylab <- "Rank per instance" - + if (missing(ablog) || is.null(ablog)) { - irace.error("You must provide an 'ablog' object generated by ablation() or the path to the '.Rdata' file that contains this object.") + irace_error("You must provide an 'ablog' object generated by ablation() or the path to the '.Rdata' file that contains this object.") } ablog <- read_ablogfile(ablog) if (!ablog$complete) stop("The ablog shows that the ablation procedure did not complete cleanly and only contains partial information") - - + + if (!is.null(pdf_file)) { if (!is.file.extension(pdf_file, ".pdf")) pdf_file <- paste0(pdf_file, ".pdf") cat("Creating PDF file '", pdf_file, "'\n", sep="") local_cairo_pdf(pdf_file, width = width, height = height, onefile= TRUE) } - - configurations <- ablog$configurations + + configurations <- ablog$allConfigurations + # Support irace < 4.2 + if (is.null(configurations)) + configurations <- ablog$configurations + trajectory <- ablog$trajectory - if (n > 0L) trajectory <- trajectory[seq_len(n+1L)] + if (n > 0L) + trajectory <- trajectory[seq_len(n+1L)] # Generate labels labels <- ablation_labels(trajectory, configurations) if (!is.null(rename_labels)) @@ -601,16 +638,16 @@ plotAblation <- function (ablog, pdf_file = NULL, width = 20, bx <- boxplot(experiments[, trajectory], plot=FALSE) if (is.null(ylim)) { ylim <- range(bx$stats[is.finite(bx$stats)], - bx$out[is.finite(bx$out)], + bx$out[is.finite(bx$out)], bx$conf[is.finite(bx$conf)]) } } - + plot(costs_avg, xaxt = "n", xlab = NA, ylab = ylab, ylim = ylim, type = "b", pch = 19, ..., panel.first = { grid(nx = NA, ny = NULL, lwd = 2); - abline(h = c(costs_avg[1], tail(costs_avg, n = 1)), + abline(h = c(costs_avg[1], utils::tail(costs_avg, n = 1L)), col = "lightgray", lty = "dotted", lwd = 2) }) axis(1, at = seq_along(costs_avg), labels = labels, las = 3) if ("boxplot" %in% type) { @@ -622,9 +659,9 @@ plotAblation <- function (ablog, pdf_file = NULL, width = 20, #' Read the log file (`log-ablation.Rdata`) produced by [irace::ablation()]. #' -#' @param filename Filename that contains the log file saved by [ablation()]. Example: `log-ablation.Rdata`. -#' -#' @return (`list()`) +#' @param filename `character(1)`\cr Filename that contains the log file saved by [ablation()]. Example: `log-ablation.Rdata`. +#' +#' @return `list()` #' @concept ablation #' @export read_ablogfile <- function(filename) read_logfile(filename, name = "ablog") diff --git a/R/cluster.R b/R/cluster.R index df8f6a92..c9367254 100644 --- a/R/cluster.R +++ b/R/cluster.R @@ -89,7 +89,7 @@ cluster_lapply <- function(X, scenario, poll.time = 2) torque = torque.job.finished, slurm = slurm.job.finished, htcondor = htcondor.job.finished, - irace.error ("Invalid value of scenario$batchmode = ", scenario$batchmode)) + irace_error ("Invalid value of scenario$batchmode = ", scenario$batchmode)) # Parallel controls how many jobs we send at once. Some clusters have low # limits. @@ -101,25 +101,25 @@ cluster_lapply <- function(X, scenario, poll.time = 2) chunks <- split(X, ceiling(seq_along(X) / chunksize)) for (chunk in chunks) { if (debugLevel >= 1) { - irace.note ("Sending ", length(chunk), " / ", length(X), " jobs\n") + irace_note ("Sending ", length(chunk), " / ", length(X), " jobs\n") } output <- lapply(chunk, exec_target_runner, scenario = scenario, target_runner = target_runner_qsub) jobIDs <- sapply(output, "[[", "jobID") - + ## Wait for cluster jobs to finish. if (length(jobIDs) > 0L && debugLevel >= 1L) { - irace.note("Waiting for jobs ('.' == ", poll.time, " s) ") + irace_note("Waiting for jobs ('.' == ", poll.time, " s): ") } for (jobID in jobIDs) { while (!cluster.job.finished(jobID)) { if (debugLevel >= 1) { cat(".") } Sys.sleep(poll.time) } - if (debugLevel >= 1) { - cat("\n") - irace.note ("DONE (", jobID, ")\n") - } + if (debugLevel >= 1) + cat(" DONE(", jobID, ") ") } + if (debugLevel >= 1) + cat("\n") } output } diff --git a/R/configurations.R b/R/configurations.R index 469d42d6..ccef9213 100644 --- a/R/configurations.R +++ b/R/configurations.R @@ -1,22 +1,16 @@ +.param_na_value_type <- list(i = NA_integer_, r = NA_real_, c = NA_character_, o = NA_character_) + # Returns a data.table configurations_alloc <- function(colnames, nrow, parameters) { - parameter_type <- function(type) { - switch(type, - i = NA_integer_, - r = NA_real_, - c = NA_character_, - o = NA_character_, - irace.internal.error("Unknown type '", type, "'")) + column_type <- function(x, n, types) { + what <- switch(x, + .ID. = NA_integer_, + .PARENT. = NA_integer_, + .WEIGHT. = NA_real_, + .param_na_value_type[[ types[x] ]]) + rep_len(what, n) } - - column_type <- function(x, n, types) - rep(switch(x, - .ID. = NA_integer_, - .PARENT. = NA_integer_, - .WEIGHT. = NA_real_, - parameter_type(types[x])), n) - x <- sapply(colnames, column_type, n=nrow, types = parameters[["types"]], simplify=FALSE, USE.NAMES=TRUE) setDT(x) @@ -78,66 +72,66 @@ which_satisfied <- function(configurations, condition) #' with a period. This function should be used before printing the #' configurations to output only the values for the parameters of the #' configuration without metadata possibly useless to the user. -#' +#' #' @param configurations `data.frame`\cr Parameter configurations of the #' target algorithm (one per row). -#' +#' #' @return The same data frame without "metadata". -#' -#' @seealso +#' +#' @seealso #' [configurations_print_command()] to print the configurations as command lines. #' [configurations_print()] to print the configurations as a data frame. -#' +#' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export removeConfigurationsMetaData <- function(configurations) configurations[, !startsWith(colnames(configurations), "."), drop = FALSE] #' Print configurations as a data frame -#' +#' #' @inheritParams removeConfigurationsMetaData -#' +#' #' @param metadata `logical(1)`\cr whether to print the metadata or #' not. The metadata are data for the configurations (additionally to the #' value of each parameter) used by \pkg{irace}. -#' +#' #' @return None. #' #' @seealso #' [configurations_print_command()] to print the configurations as command-line strings. -#' +#' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export configurations_print <- function(configurations, metadata = FALSE) { - if (!is.data.frame(configurations)) + if (is.data.table(configurations) || !is.data.frame(configurations)) configurations <- as.data.frame(configurations, stringsAsFactors = FALSE) rownames(configurations) <- configurations[[".ID."]] if (!metadata) configurations <- removeConfigurationsMetaData(configurations) - + print.data.frame(configurations, digits = 15L) } #' Print configurations as command-line strings. -#' +#' #' Prints configurations after converting them into a representation for the #' command-line. -#' +#' #' @inheritParams removeConfigurationsMetaData #' @inheritParams printParameters -#' +#' #' @return None. #' #' @seealso #' [configurations_print()] to print the configurations as a data frame. -#' +#' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export configurations_print_command <- function(configurations, parameters) { - if (nrow(configurations) <= 0) return(invisible()) + if (nrow(configurations) <= 0L) return(invisible()) ids <- as.numeric(configurations$.ID.) configurations <- removeConfigurationsMetaData(configurations) # Re-sort the columns diff --git a/R/generation.R b/R/generation.R index 9c587a36..f6b53832 100644 --- a/R/generation.R +++ b/R/generation.R @@ -27,6 +27,15 @@ repairConfigurations <- function(x, parameters, repair) x } +is_within_dependent_bound <- function(param, configuration, value) +{ + domain <- unlist(lapply(param[["domain"]], eval, configuration)) + # Value gets truncated (defined from robotics initial requirements) + if (param[["type"]] == "i") + domain <- as.integer(domain) + domain[[1L]] <= value && value <= domain[[2L]] +} + ## Calculates the parameter bounds when parameters domain is dependent getDependentBound <- function(param, configuration) { @@ -36,11 +45,11 @@ getDependentBound <- function(param, configuration) # If it depends on a parameter that is disabled, then this is disabled. if (anyNA(configuration[deps])) return(NA) domain <- sapply(domain, eval, configuration) - irace.assert(all(is.finite(domain))) + irace_assert(all(is.finite(domain))) # Value gets truncated (defined from robotics initial requirements) if (param[["type"]] == "i") domain <- as.integer(domain) if (domain[[1L]] > domain[[2L]]) { - irace.error ("Invalid domain (", paste0(domain, collapse=", "), + irace_error ("Invalid domain (", paste0(domain, collapse=", "), ") generated for parameter '", param[["name"]], "' that depends on parameters (", paste0(deps, collapse=", "), "). This is NOT a bug in irace. Check the definition of these parameters.") @@ -58,16 +67,18 @@ get_dependent_domain <- function(param, configuration) deps <- all.vars(domain) # FIXME: This function should not be called if the parent is disabled. # If it depends on a parameter that is disabled, then this is disabled. - if (anyNA(configuration[deps])) return(NA) + if (anyNA(configuration[deps])) + return(NA) - irace.assert(is.expression(domain)) + irace_assert(is.expression(domain)) domain <- sapply(domain, eval, configuration, USE.NAMES=FALSE) - irace.assert(all(is.finite(domain))) + irace_assert(all(is.finite(domain))) # Value gets truncated (defined from robotics initial requirements) - if (param[["type"]] == "i") domain <- as.integer(domain) + if (param[["type"]] == "i") + domain <- as.integer(domain) if (domain[[1L]] > domain[[2L]]) { # FIXME: Add test for this error. - irace.error ("Invalid domain (", paste0(domain, collapse=", "), + irace_error ("Invalid domain (", paste0(domain, collapse=", "), ") generated for parameter '", param[["name"]], "' that depends on parameters (", paste0(deps, collapse=", "), "). This is NOT a bug in irace. Check the definition of these parameters.") @@ -87,12 +98,11 @@ generate_sobol <- function(parameters, n, repair = NULL) # FIXME: How to do this faster using data.table? for (x in nodep_names) set(confs, j = x, value = param_quantile(parameters$get(x), confs[[x]])) - + for (x in parameters$names_fixed) set(confs, j = x, value = parameters$domains[[x]]) - + setcolorder(confs, parameters$names) - max_level <- max(hierarchy) if (max_level > 1L) { .NEWVALUE <- .DOMAIN <- NULL # To silence CRAN warnings. @@ -101,29 +111,24 @@ generate_sobol <- function(parameters, n, repair = NULL) dep_names <- names(hierarchy)[hierarchy == k+1L] for (p in dep_names) { param <- parameters$get(p) - idx_satisfied <- which_satisfied(confs, param[["condition"]]) if (param$isFixed) { - # If somehow this fixed parameter was not satisfied sometimes, just set its value to NA. - confs[!idx_satisfied, (p):= NA ] - } else if (length(idx_satisfied)) { + if (!isTRUE(param[["condition"]])) + # If somehow this fixed parameter was not satisfied sometimes, just set its value to NA. + set(confs, which(!eval(param[["condition"]], confs)), j = p, value = NA_character_) + next + } + idx_satisfied <- which_satisfied(confs, param[["condition"]]) + if (length(idx_satisfied)) { if (param[["is_dependent"]]) { confs[idx_satisfied, let(.DOMAIN = list(get_dependent_domain(param, .SD))), by=.I, .SDcols=prev_names] confs[idx_satisfied, .NEWVALUE := param_quantile(param, .SD, domain = unlist(.DOMAIN)), by=.I, .SDcols=p] - confs[, (p):=.NEWVALUE] - confs[, let(.NEWVALUE=NULL, .DOMAIN=NULL)] + set(confs, j = ".DOMAIN", value = NULL) } else { confs[idx_satisfied, .NEWVALUE := param_quantile(param, unlist(.SD)), .SDcols=p] - confs[, (p):=.NEWVALUE] - confs[, let(.NEWVALUE=NULL)] } + set(confs, j = c(p, ".NEWVALUE"), value = list(confs[[".NEWVALUE"]], NULL)) } else { - na_value <- switch(param[["type"]], - i = NA_integer_, - r = NA_real_, - c = NA_character_, - o = NA_character_, - irace.internal.error("Unknown type '", param[["type"]], "'")) - set(confs, j = p, value = na_value) + set(confs, j = p, value = .param_na_value_type[[ param[["type"]] ]]) } } } @@ -136,7 +141,7 @@ sampleSobol <- function(parameters, n, repair = NULL) { newConfigurations <- generate_sobol(parameters, n, repair) newConfigurations <- unique(newConfigurations) - forbidden <- parameters$forbidden + forbidden <- parameters$forbidden newConfigurations <- filter_forbidden(newConfigurations, forbidden) have <- nrow(newConfigurations) if (have < n) { @@ -145,7 +150,7 @@ sampleSobol <- function(parameters, n, repair = NULL) newConfigurations <- unique(newConfigurations) newConfigurations <- filter_forbidden(newConfigurations, forbidden) if (nrow(newConfigurations) == 0L) { - irace.error("irace tried to sample a configuration not forbidden without success, perhaps your constraints are too strict?") + irace_error("irace tried to sample a configuration not forbidden without success, perhaps your constraints are too strict?") } newConfigurations <- truncate_rows(newConfigurations, n) } @@ -190,7 +195,7 @@ generate_uniform <- function(parameters, nbConfigurations, repair = NULL) sampleUniform <- function(parameters, nbConfigurations, repair = NULL) { newConfigurations <- generate_uniform(parameters, nbConfigurations, repair) - forbidden <- parameters$forbidden + forbidden <- parameters$forbidden if (!is.null(forbidden)) { retries <- 100L repeat { @@ -202,7 +207,7 @@ sampleUniform <- function(parameters, nbConfigurations, repair = NULL) generate_uniform(parameters, needed, repair = repair))) retries <- retries - 1L if (retries == 0L) { - irace.error("irace tried 100 times to sample uniformly a configuration not forbidden without success, perhaps your constraints are too strict?") + irace_error("irace tried 100 times to sample uniformly a configuration not forbidden without success, perhaps your constraints are too strict?") } } } @@ -215,11 +220,11 @@ sample_from_model <- function(parameters, eliteConfigurations, model, { # FIXME: We only need .WEIGHT. from eliteConfigurations. ids_elites <- names(model[[1L]]) - irace.assert(identical(as.integer(ids_elites), as.integer(eliteConfigurations[[".ID."]])), { - print(str(ids_elites)) - print(str(eliteConfigurations[[".ID."]])) + irace_assert(identical(as.integer(ids_elites), as.integer(eliteConfigurations[[".ID."]])), { + print(utils::str(ids_elites)) + print(utils::str(eliteConfigurations[[".ID."]])) }) - + newConfigurations <- configurations_alloc(parameters$names, nrow = nbNewConfigurations, parameters) idx_elites <- sample.int(n = length(ids_elites), size = nbNewConfigurations, prob = eliteConfigurations[[".WEIGHT."]], replace = TRUE) @@ -256,7 +261,7 @@ sample_from_model <- function(parameters, eliteConfigurations, model, c(pname) := list(sample_model(param, .N, this_model[[ .BY[[1L]] ]])), by = .PARENT.] } - newConfigurations[, .PARENT. := as.integer(.PARENT.)] + set(newConfigurations, j = ".PARENT.", value = as.integer(newConfigurations[[".PARENT."]])) repairConfigurations(newConfigurations, parameters, repair) newConfigurations } @@ -265,7 +270,7 @@ sampleModel <- function(parameters, eliteConfigurations, model, nbNewConfigurations, repair = NULL) { if (nbNewConfigurations <= 0) - irace.error ("The number of configurations to generate appears to be negative or zero.") + irace_error ("The number of configurations to generate appears to be negative or zero.") newConfigurations <- sample_from_model(parameters, eliteConfigurations, model, nbNewConfigurations, repair) forbidden <- parameters$forbidden @@ -280,17 +285,9 @@ sampleModel <- function(parameters, eliteConfigurations, model, newConfigurations <- rbindlist(list(newConfigurations, tmp)) retries <- retries - 1L if (retries == 0L) { - irace.error("irace tried 100 times to sample from the model a configuration not forbidden without success, perhaps your constraints are too strict?") + irace_error("irace tried 100 times to sample from the model a configuration not forbidden without success, perhaps your constraints are too strict?") } } } newConfigurations } - - - - - - - - diff --git a/R/irace-options.R b/R/irace-options.R index f3477b1c..efdb9966 100644 --- a/R/irace-options.R +++ b/R/irace-options.R @@ -4,151 +4,152 @@ # Types are b(oolean), i(nteger), s(tring), r(eal), p(ath), x (R object or no value) # FIXME: Add special type for R functions. # FIXME: For i and r add their domain. -.irace.params.def <- structure(list(name = c(".help", ".version", ".check", ".init", -".onlytest", "scenarioFile", "execDir", "parameterFile", "parameters", -"initConfigurations", "configurationsFile", "logFile", "recoveryFile", -"instances", "trainInstancesDir", "trainInstancesFile", "sampleInstances", -"testInstancesDir", "testInstancesFile", "testInstances", "testNbElites", -"testIterationElites", "testType", "firstTest", "blockSize", -"eachTest", "targetRunner", "targetRunnerLauncher", "targetCmdline", -"targetRunnerRetries", "targetRunnerTimeout", "targetRunnerData", -"targetRunnerParallel", "targetEvaluator", "deterministic", "maxExperiments", -"minExperiments", "maxTime", "budgetEstimation", "minMeasurableTime", -"parallel", "loadBalancing", "mpi", "batchmode", "quiet", "debugLevel", -"seed", "softRestart", "softRestartThreshold", "elitist", "elitistNewInstances", -"elitistLimit", "repairConfiguration", "capping", "cappingType", -"boundType", "boundMax", "boundDigits", "boundPar", "boundAsTimeout", -"postselection", "aclib", "nbIterations", "nbExperimentsPerIteration", -"minNbSurvival", "nbConfigurations", "mu", "confidence"), type = c("x", -"x", "x", "x", "p", "p", "p", "p", "x", "x", "p", "p", "p", "s", -"p", "p", "b", "p", "p", "x", "i", "b", "s", "i", "i", "i", "p", -"p", "s", "i", "i", "x", "x", "p", "b", "i", "i", "i", "r", "r", -"i", "b", "b", "s", "b", "i", "i", "b", "r", "b", "i", "i", "x", -"b", "s", "s", "i", "i", "i", "b", "b", "b", "i", "i", "i", "i", -"i", "r"), short = c("-h", "-v", "-c", "-i", "", "-s", "", "-p", -"", "", "", "-l", "", "", "", "", "", "", "", "", "", "", "", -"", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", -"", "", "", "", "", "-q", "", "", "", "", "-e", "", "", "", "", -"", "", "", "", "", "", "", "", "", "", "", "", "", ""), long = c("--help", -"--version", "--check", "--init", "--only-test", "--scenario", -"--exec-dir", "--parameter-file", "", "", "--configurations-file", -"--log-file", "--recovery-file", "", "--train-instances-dir", -"--train-instances-file", "--sample-instances", "--test-instances-dir", -"--test-instances-file", "", "--test-num-elites", "--test-iteration-elites", -"--test-type", "--first-test", "--block-size", "--each-test", -"--target-runner", "--target-runner-launcher", "--target-cmdline", -"--target-runner-retries", "--target-runner-timeout", "", "", -"--target-evaluator", "--deterministic", "--max-experiments", -"--min-experiments", "--max-time", "--budget-estimation", "--min-measurable-time", -"--parallel", "--load-balancing", "--mpi", "--batchmode", "--quiet", -"--debug-level", "--seed", "--soft-restart", "--soft-restart-threshold", -"--elitist", "--elitist-new-instances", "--elitist-limit", "", -"--capping", "--capping-type", "--bound-type", "--bound-max", -"--bound-digits", "--bound-par", "--bound-as-timeout", "--postselection", -"--aclib", "--iterations", "--experiments-per-iteration", "--min-survival", -"--num-configurations", "--mu", "--confidence"), default = c(NA, -NA, NA, "", "", "./scenario.txt", "./", "./parameters.txt", "", -"", "", "./irace.Rdata", "", "", "", "", "1", "", "", "", "1", -"0", "", "5", "1", "1", "./target-runner", "", "{configurationID} {instanceID} {seed} {instance} {bound} {targetRunnerArgs}", -"0", "0", "", "", "", "0", "0", NA, "0", "0.05", "0.01", "0", -"1", "0", "0", "0", "0", NA, "1", "1e-04", "1", "1", "2", "", -NA, "median", "candidate", "0", "0", "1", "1", "1", "0", "0", -"0", "0", "0", "5", "0.95"), domain = c(NA, NA, NA, NA, NA, NA, -NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, -"F-test,t-test,t-test-holm,t-test-bonferroni", NA, NA, NA, NA, -NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, -"sge,pbs,torque,slurm,htcondor", NA, NA, NA, NA, NA, NA, NA, -NA, NA, NA, "median,mean,worst,best", "instance,candidate", NA, -NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA), description = c("Show this help.", -"Show irace package version.", "Check scenario.", "Initialize the working directory with template config files.", -"Only test the configurations given in the file passed as argument.", -"File that describes the configuration scenario setup and other irace settings.", -"Directory where the programs will be run.", "File that contains the description of the parameters of the target algorithm.", -"", "", "File that contains a table of initial configurations. If empty or `NULL`, all initial configurations are randomly generated.", -"File to save tuning results as an R dataset, either absolute path or relative to execDir.", -"Previously saved log file to recover the execution of `irace`, either absolute path or relative to the current directory. If empty or `NULL`, recovery is not performed.", -"", "Directory where training instances are located; either absolute path or relative to current directory. If no `trainInstancesFiles` is provided, all the files in `trainInstancesDir` will be listed as instances.", -"File that contains a list of training instances and optionally additional parameters for them. If `trainInstancesDir` is provided, `irace` will search for the files in this folder.", -"Randomly sample the training instances or use them in the order given.", -"Directory where testing instances are located, either absolute or relative to current directory.", -"File containing a list of test instances and optionally additional parameters for them.", -"", "Number of elite configurations returned by irace that will be tested if test instances are provided.", -"Enable/disable testing the elite configurations found at each iteration.", -"Statistical test used for elimination. The default value selects `t-test` if `capping` is enabled or `F-test`, otherwise. Valid values are: F-test (Friedman test), t-test (pairwise t-tests with no correction), t-test-bonferroni (t-test with Bonferroni's correction for multiple comparisons), t-test-holm (t-test with Holm's correction for multiple comparisons).", -"Number of instances evaluated before the first elimination test. It must be a multiple of `eachTest`.", -"Number of training instances, that make up a `block' in `trainInstancesFile`. Elimination of configurations will only be performed after evaluating a complete block and never in the middle of a block. Each block typically contains one instance from each instance class (type or family) and the block size is the number of classes.", -"Number of instances evaluated between elimination tests.", "Executable called for each configuration that executes the target algorithm to be tuned. See the templates and examples provided.", -"Executable that will be used to launch the target runner, when `targetRunner` cannot be executed directly (e.g., a Python script in Windows).", -"Command-line arguments provided to `targetRunner` (or `targetRunnerLauncher` if defined). The substrings `\\{configurationID\\}`, `\\{instanceID\\}`, `\\{seed\\}`, `\\{instance\\}`, and `\\{bound\\}` will be replaced by their corresponding values. The substring `\\{targetRunnerArgs\\}` will be replaced by the concatenation of the switch and value of all active parameters of the particular configuration being evaluated. The substring `\\{targetRunner\\}`, if present, will be replaced by the value of `targetRunner` (useful when using `targetRunnerLauncher`).", -"Number of times to retry a call to `targetRunner` if the call failed.", -"Timeout in seconds of any `targetRunner` call (only applies to `target-runner` executables not to R functions), ignored if 0.", -"Optional data passed to `targetRunner`. This is ignored by the default `targetRunner` function, but it may be used by custom `targetRunner` functions to pass persistent data around.", -"Optional R function to provide custom parallelization of `targetRunner`.", -"Optional script or R function that provides a numeric value for each configuration. See templates/target-evaluator.tmpl", -"If the target algorithm is deterministic, configurations will be evaluated only once per instance.", -"Maximum number of runs (invocations of `targetRunner`) that will be performed. It determines the maximum budget of experiments for the tuning.", -"Minimum number of runs (invocations of `targetRunner`) that will be performed. It determines the minimum budget of experiments for the tuning. The actual budget depends on the number of parameters and `minSurvival`.", -"Maximum total execution time for the executions of `targetRunner`. `targetRunner` must return two values: cost and time. This value and the one returned by `targetRunner` must use the same units (seconds, minutes, iterations, evaluations, ...).", -"Fraction (smaller than 1) of the budget used to estimate the mean computation time of a configuration. Only used when `maxTime` > 0", -"Minimum time unit that is still (significantly) measureable.", -"Number of calls to `targetRunner` to execute in parallel. Values `0` or `1` mean no parallelization.", -"Enable/disable load-balancing when executing experiments in parallel. Load-balancing makes better use of computing resources, but increases communication overhead. If this overhead is large, disabling load-balancing may be faster.", -"Enable/disable MPI. Use `Rmpi` to execute `targetRunner` in parallel (parameter `parallel` is the number of slaves).", -"Specify how irace waits for jobs to finish when `targetRunner` submits jobs to a batch cluster: sge, pbs, torque, slurm or htcondor. `targetRunner` must submit jobs to the cluster using, for example, `qsub`.", -"Reduce the output generated by irace to a minimum.", "Debug level of the output of `irace`. Set this to 0 to silence all debug messages. Higher values provide more verbose debug messages.", -"Seed of the random number generator (by default, generate a random seed).", -"Enable/disable the soft restart strategy that avoids premature convergence of the probabilistic model.", -"Soft restart threshold value for numerical parameters.", "Enable/disable elitist irace.", -"Number of instances added to the execution list before previous instances in elitist irace.", -"In elitist irace, maximum number per race of elimination tests that do not eliminate a configuration. Use 0 for no limit.", -"User-defined R function that takes a configuration generated by irace and repairs it.", -"Enable the use of adaptive capping, a technique designed for minimizing the computation time of configurations. Capping is enabled by default if `elitist` is active, `maxTime > 0` and `boundMax > 0`.", -"Measure used to obtain the execution bound from the performance of the elite configurations: median, mean, worst, best.", -"Method to calculate the mean performance of elite configurations: candidate or instance.", -"Maximum execution bound for `targetRunner`. It must be specified when capping is enabled.", -"Precision used for calculating the execution time. It must be specified when capping is enabled.", -"Penalization constant for timed out executions (executions that reach `boundMax` execution time).", -"Replace the configuration cost of bounded executions with `boundMax`.", -"Perform a postselection race after the execution of irace to consume all remaining budget. Value 0 disables the postselection race.", -"Enable/disable AClib mode. This option enables compatibility with GenericWrapper4AC as targetRunner script.", -"Maximum number of iterations.", "Number of runs of the target algorithm per iteration.", -"Minimum number of configurations needed to continue the execution of each race (iteration).", -"Number of configurations to be sampled and evaluated at each iteration.", -"Parameter used to define the number of configurations sampled and evaluated at each iteration.", -"Confidence level for the elimination test.")), row.names = c(".help", -".version", ".check", ".init", ".onlytest", "scenarioFile", "execDir", -"parameterFile", "parameters", "initConfigurations", "configurationsFile", -"logFile", "recoveryFile", "instances", "trainInstancesDir", -"trainInstancesFile", "sampleInstances", "testInstancesDir", -"testInstancesFile", "testInstances", "testNbElites", "testIterationElites", -"testType", "firstTest", "blockSize", "eachTest", "targetRunner", -"targetRunnerLauncher", "targetCmdline", "targetRunnerRetries", -"targetRunnerTimeout", "targetRunnerData", "targetRunnerParallel", -"targetEvaluator", "deterministic", "maxExperiments", "minExperiments", -"maxTime", "budgetEstimation", "minMeasurableTime", "parallel", -"loadBalancing", "mpi", "batchmode", "quiet", "debugLevel", "seed", -"softRestart", "softRestartThreshold", "elitist", "elitistNewInstances", -"elitistLimit", "repairConfiguration", "capping", "cappingType", -"boundType", "boundMax", "boundDigits", "boundPar", "boundAsTimeout", -"postselection", "aclib", "nbIterations", "nbExperimentsPerIteration", +.irace.params.def <- structure(list(name = c(".help", ".version", ".check", ".init", +".onlytest", "scenarioFile", "execDir", "parameterFile", "parameters", +"initConfigurations", "configurationsFile", "logFile", "recoveryFile", +"instances", "trainInstancesDir", "trainInstancesFile", "sampleInstances", +"testInstancesDir", "testInstancesFile", "testInstances", "testNbElites", +"testIterationElites", "testType", "firstTest", "blockSize", +"eachTest", "targetRunner", "targetRunnerLauncher", "targetCmdline", +"targetRunnerRetries", "targetRunnerTimeout", "targetRunnerData", +"targetRunnerParallel", "targetEvaluator", "deterministic", "maxExperiments", +"minExperiments", "maxTime", "budgetEstimation", "minMeasurableTime", +"parallel", "loadBalancing", "mpi", "batchmode", "quiet", "debugLevel", +"seed", "softRestart", "softRestartThreshold", "elitist", "elitistNewInstances", +"elitistLimit", "repairConfiguration", "capping", "cappingAfterFirstTest", +"cappingType", "boundType", "boundMax", "boundDigits", "boundPar", +"boundAsTimeout", "postselection", "aclib", "nbIterations", "nbExperimentsPerIteration", +"minNbSurvival", "nbConfigurations", "mu", "confidence"), type = c("x", +"x", "x", "x", "p", "p", "p", "p", "x", "x", "p", "p", "p", "s", +"p", "p", "b", "p", "p", "x", "i", "b", "s", "i", "i", "i", "p", +"p", "s", "i", "i", "x", "x", "p", "b", "i", "i", "i", "r", "r", +"i", "b", "b", "s", "b", "i", "i", "b", "r", "b", "i", "i", "x", +"b", "b", "s", "s", "i", "i", "i", "b", "b", "b", "i", "i", "i", +"i", "i", "r"), short = c("-h", "-v", "-c", "-i", "", "-s", "", +"-p", "", "", "", "-l", "", "", "", "", "", "", "", "", "", "", +"", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", +"", "", "", "", "", "", "-q", "", "", "", "", "-e", "", "", "", +"", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" +), long = c("--help", "--version", "--check", "--init", "--only-test", +"--scenario", "--exec-dir", "--parameter-file", "", "", "--configurations-file", +"--log-file", "--recovery-file", "", "--train-instances-dir", +"--train-instances-file", "--sample-instances", "--test-instances-dir", +"--test-instances-file", "", "--test-num-elites", "--test-iteration-elites", +"--test-type", "--first-test", "--block-size", "--each-test", +"--target-runner", "--target-runner-launcher", "--target-cmdline", +"--target-runner-retries", "--target-runner-timeout", "", "", +"--target-evaluator", "--deterministic", "--max-experiments", +"--min-experiments", "--max-time", "--budget-estimation", "--min-measurable-time", +"--parallel", "--load-balancing", "--mpi", "--batchmode", "--quiet", +"--debug-level", "--seed", "--soft-restart", "--soft-restart-threshold", +"--elitist", "--elitist-new-instances", "--elitist-limit", "", +"--capping", "--capping-after-first-test", "--capping-type", +"--bound-type", "--bound-max", "--bound-digits", "--bound-par", +"--bound-as-timeout", "--postselection", "--aclib", "--iterations", +"--experiments-per-iteration", "--min-survival", "--num-configurations", +"--mu", "--confidence"), default = c(NA, NA, NA, "", "", "./scenario.txt", +"./", "./parameters.txt", "", "", "", "./irace.Rdata", "", "", +"", "", "1", "", "", "", "1", "0", "", "5", "1", "1", "./target-runner", +"", "{configurationID} {instanceID} {seed} {instance} {bound} {targetRunnerArgs}", +"0", "0", "", "", "", "0", "0", NA, "0", "0.05", "0.01", "0", +"1", "0", "0", "0", "0", NA, "1", "1e-04", "1", "1", "2", "", +NA, "0", "median", "candidate", "0", "0", "1", "1", "1", "0", +"0", "0", "0", "0", "5", "0.95"), domain = c(NA, NA, NA, NA, +NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, +NA, NA, "F-test,t-test,t-test-holm,t-test-bonferroni", NA, NA, +NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, +NA, NA, "sge,pbs,torque,slurm,htcondor", NA, NA, NA, NA, NA, +NA, NA, NA, NA, NA, NA, "median,mean,worst,best", "instance,candidate", +NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA), description = c("Show this help.", +"Show irace package version.", "Check scenario.", "Initialize the working directory with template config files.", +"Only test the configurations given in the file passed as argument.", +"File that describes the configuration scenario setup and other irace settings.", +"Directory where the programs will be run.", "File that contains the description of the parameters of the target algorithm.", +"", "", "File that contains a table of initial configurations. If empty or `NULL`, all initial configurations are randomly generated.", +"File to save tuning results as an R dataset, either absolute path or relative to execDir.", +"Previously saved log file to recover the execution of `irace`, either absolute path or relative to the current directory. If empty or `NULL`, recovery is not performed.", +"", "Directory where training instances are located; either absolute path or relative to current directory. If no `trainInstancesFiles` is provided, all the files in `trainInstancesDir` will be listed as instances.", +"File that contains a list of training instances and optionally additional parameters for them. If `trainInstancesDir` is provided, `irace` will search for the files in this folder.", +"Randomly sample the training instances or use them in the order given.", +"Directory where testing instances are located, either absolute or relative to current directory.", +"File containing a list of test instances and optionally additional parameters for them.", +"", "Number of elite configurations returned by irace that will be tested if test instances are provided.", +"Enable/disable testing the elite configurations found at each iteration.", +"Statistical test used for elimination. The default value selects `t-test` if `capping` is enabled or `F-test`, otherwise. Valid values are: F-test (Friedman test), t-test (pairwise t-tests with no correction), t-test-bonferroni (t-test with Bonferroni's correction for multiple comparisons), t-test-holm (t-test with Holm's correction for multiple comparisons).", +"Number of instances evaluated before the first elimination test. It must be a multiple of `eachTest`.", +"Number of training instances, that make up a 'block' in `trainInstancesFile`. Elimination of configurations will only be performed after evaluating a complete block and never in the middle of a block. Each block typically contains one instance from each instance class (type or family) and the block size is the number of classes. The value of `blockSize` will multiply `firstTest`, `eachTest` and `elitistNewInstances`.", +"Number of instances evaluated between elimination tests.", "Executable called for each configuration that executes the target algorithm to be tuned. See the templates and examples provided.", +"Executable that will be used to launch the target runner, when `targetRunner` cannot be executed directly (e.g., a Python script in Windows).", +"Command-line arguments provided to `targetRunner` (or `targetRunnerLauncher` if defined). The substrings `\\{configurationID\\}`, `\\{instanceID\\}`, `\\{seed\\}`, `\\{instance\\}`, and `\\{bound\\}` will be replaced by their corresponding values. The substring `\\{targetRunnerArgs\\}` will be replaced by the concatenation of the switch and value of all active parameters of the particular configuration being evaluated. The substring `\\{targetRunner\\}`, if present, will be replaced by the value of `targetRunner` (useful when using `targetRunnerLauncher`).", +"Number of times to retry a call to `targetRunner` if the call failed.", +"Timeout in seconds of any `targetRunner` call (only applies to `target-runner` executables not to R functions), ignored if 0.", +"Optional data passed to `targetRunner`. This is ignored by the default `targetRunner` function, but it may be used by custom `targetRunner` functions to pass persistent data around.", +"Optional R function to provide custom parallelization of `targetRunner`.", +"Optional script or R function that provides a numeric value for each configuration. See templates/target-evaluator.tmpl", +"If the target algorithm is deterministic, configurations will be evaluated only once per instance.", +"Maximum number of runs (invocations of `targetRunner`) that will be performed. It determines the maximum budget of experiments for the tuning.", +"Minimum number of runs (invocations of `targetRunner`) that will be performed. It determines the minimum budget of experiments for the tuning. The actual budget depends on the number of parameters and `minSurvival`.", +"Maximum total execution time for the executions of `targetRunner`. `targetRunner` must return two values: cost and time. This value and the one returned by `targetRunner` must use the same units (seconds, minutes, iterations, evaluations, ...).", +"Fraction (smaller than 1) of the budget used to estimate the mean computation time of a configuration. Only used when `maxTime` > 0", +"Minimum time unit that is still (significantly) measureable.", +"Number of calls to `targetRunner` to execute in parallel. Values `0` or `1` mean no parallelization.", +"Enable/disable load-balancing when executing experiments in parallel. Load-balancing makes better use of computing resources, but increases communication overhead. If this overhead is large, disabling load-balancing may be faster.", +"Enable/disable MPI. Use `Rmpi` to execute `targetRunner` in parallel (parameter `parallel` is the number of slaves).", +"Specify how irace waits for jobs to finish when `targetRunner` submits jobs to a batch cluster: sge, pbs, torque, slurm or htcondor. `targetRunner` must submit jobs to the cluster using, for example, `qsub`.", +"Reduce the output generated by irace to a minimum.", "Debug level of the output of `irace`. Set this to 0 to silence all debug messages. Higher values (1, 2 or 3) provide more verbose debug messages.", +"Seed of the random number generator (by default, generate a random seed).", +"Enable/disable the soft restart strategy that avoids premature convergence of the probabilistic model.", +"Soft restart threshold value for numerical parameters.", "Enable/disable elitist irace.", +"Number of instances added to the execution list before previous instances in elitist irace.", +"In elitist irace, maximum number per race of elimination tests that do not eliminate a configuration. Use 0 for no limit.", +"User-defined R function that takes a configuration generated by irace and repairs it.", +"Enable the use of adaptive capping, a technique designed for minimizing the computation time of configurations. Capping is enabled by default if `elitist` is active, `maxTime > 0` and `boundMax > 0`.", +"If set to 1, elimination due to capping only happens after `firstTest` instances are seen.", +"Measure used to obtain the execution bound from the performance of the elite configurations: median, mean, worst, best.", +"Method to calculate the mean performance of elite configurations: candidate or instance.", +"Maximum execution bound for `targetRunner`. It must be specified when capping is enabled.", +"Precision used for calculating the execution time. It must be specified when capping is enabled.", +"Penalization constant for timed out executions (executions that reach `boundMax` execution time).", +"Replace the configuration cost of bounded executions with `boundMax`.", +"Perform a postselection race after the execution of irace to consume all remaining budget. Value 0 disables the postselection race.", +"Enable/disable AClib mode. This option enables compatibility with GenericWrapper4AC as targetRunner script.", +"Maximum number of iterations.", "Number of runs of the target algorithm per iteration.", +"Minimum number of configurations needed to continue the execution of each race (iteration).", +"Number of configurations to be sampled and evaluated at each iteration.", +"Parameter used to define the number of configurations sampled and evaluated at each iteration.", +"Confidence level for the elimination test.")), row.names = c(".help", +".version", ".check", ".init", ".onlytest", "scenarioFile", "execDir", +"parameterFile", "parameters", "initConfigurations", "configurationsFile", +"logFile", "recoveryFile", "instances", "trainInstancesDir", +"trainInstancesFile", "sampleInstances", "testInstancesDir", +"testInstancesFile", "testInstances", "testNbElites", "testIterationElites", +"testType", "firstTest", "blockSize", "eachTest", "targetRunner", +"targetRunnerLauncher", "targetCmdline", "targetRunnerRetries", +"targetRunnerTimeout", "targetRunnerData", "targetRunnerParallel", +"targetEvaluator", "deterministic", "maxExperiments", "minExperiments", +"maxTime", "budgetEstimation", "minMeasurableTime", "parallel", +"loadBalancing", "mpi", "batchmode", "quiet", "debugLevel", "seed", +"softRestart", "softRestartThreshold", "elitist", "elitistNewInstances", +"elitistLimit", "repairConfiguration", "capping", "cappingAfterFirstTest", +"cappingType", "boundType", "boundMax", "boundDigits", "boundPar", +"boundAsTimeout", "postselection", "aclib", "nbIterations", "nbExperimentsPerIteration", "minNbSurvival", "nbConfigurations", "mu", "confidence"), class = "data.frame") -.irace.params.names <- c("scenarioFile", "execDir", "parameterFile", "parameters", "initConfigurations", -"configurationsFile", "logFile", "recoveryFile", "instances", -"trainInstancesDir", "trainInstancesFile", "sampleInstances", -"testInstancesDir", "testInstancesFile", "testInstances", "testNbElites", -"testIterationElites", "testType", "firstTest", "blockSize", -"eachTest", "targetRunner", "targetRunnerLauncher", "targetCmdline", -"targetRunnerRetries", "targetRunnerTimeout", "targetRunnerData", -"targetRunnerParallel", "targetEvaluator", "deterministic", "maxExperiments", -"minExperiments", "maxTime", "budgetEstimation", "minMeasurableTime", -"parallel", "loadBalancing", "mpi", "batchmode", "quiet", "debugLevel", -"seed", "softRestart", "softRestartThreshold", "elitist", "elitistNewInstances", -"elitistLimit", "repairConfiguration", "capping", "cappingType", -"boundType", "boundMax", "boundDigits", "boundPar", "boundAsTimeout", -"postselection", "aclib", "nbIterations", "nbExperimentsPerIteration", +.irace.params.names <- c("scenarioFile", "execDir", "parameterFile", "parameters", "initConfigurations", +"configurationsFile", "logFile", "recoveryFile", "instances", +"trainInstancesDir", "trainInstancesFile", "sampleInstances", +"testInstancesDir", "testInstancesFile", "testInstances", "testNbElites", +"testIterationElites", "testType", "firstTest", "blockSize", +"eachTest", "targetRunner", "targetRunnerLauncher", "targetCmdline", +"targetRunnerRetries", "targetRunnerTimeout", "targetRunnerData", +"targetRunnerParallel", "targetEvaluator", "deterministic", "maxExperiments", +"minExperiments", "maxTime", "budgetEstimation", "minMeasurableTime", +"parallel", "loadBalancing", "mpi", "batchmode", "quiet", "debugLevel", +"seed", "softRestart", "softRestartThreshold", "elitist", "elitistNewInstances", +"elitistLimit", "repairConfiguration", "capping", "cappingAfterFirstTest", +"cappingType", "boundType", "boundMax", "boundDigits", "boundPar", +"boundAsTimeout", "postselection", "aclib", "nbIterations", "nbExperimentsPerIteration", "minNbSurvival", "nbConfigurations", "mu", "confidence") ## FIXME: If these values are special perhaps they should be saved in $state ? .irace.params.recover <- c("instances", "seed", "testInstances", # We need this because this data may mutate "targetRunnerData", "elitist", "deterministic") - diff --git a/R/irace-package.R b/R/irace-package.R index 8f7cfd3c..34df1f59 100644 --- a/R/irace-package.R +++ b/R/irace-package.R @@ -3,13 +3,14 @@ #' \packageDescription{irace} #' #' @name irace-package -#' @import stats utils compiler matrixStats withr data.table +#' @import stats matrixStats withr data.table +#' @importFrom utils str #' @importFrom R6 R6Class #' @importFrom graphics abline axis boxplot par plot points strwidth bxp grid #' @importFrom spacefillr generate_sobol_set -#' +#' #' @details License: GPL (>= 2) -#' +#' #' @author Maintainers: Manuel López-Ibáñez and Leslie Pérez Cáceres #' \email{irace-package@googlegroups.com} #' @@ -20,12 +21,12 @@ #' Thomas Stützle, and Mauro Birattari. The irace package: Iterated #' Racing for Automatic Algorithm Configuration. \emph{Operations Research #' Perspectives}, 2016. \doi{10.1016/j.orp.2016.09.002} -#' +#' #' Manuel López-Ibáñez, Jérémie Dubois-Lacoste, Thomas Stützle, and Mauro #' Birattari. \emph{The irace package, Iterated Race for Automatic #' Algorithm Configuration}. Technical Report TR/IRIDIA/2011-004, IRIDIA, #' Université Libre de Bruxelles, Belgium, 2011. -#' +#' #' Manuel López-Ibáñez and Thomas Stützle. The Automatic Design of #' Multi-Objective Ant Colony Optimization Algorithms. \emph{IEEE Transactions #' on Evolutionary Computation}, 2012. @@ -33,5 +34,10 @@ #' [irace()] for examples and `vignette(package = "irace")` for the user-guide. "_PACKAGE" -# Prefix for printing messages to the user. -.irace_msg_prefix <- "== irace == " + +# This silences CRAN NOTE: +# Namespace in Imports field not imported from: 'codetools' +# All declared Imports should be used. +.ignore_unused_imports <- function() { # nocov start + codetools::findGlobals +} # nocov end diff --git a/R/irace.R b/R/irace.R index 9c1e0a6c..f608cacb 100644 --- a/R/irace.R +++ b/R/irace.R @@ -1,6 +1,3 @@ -# Sets irace variables from a recovery file. It is executed in the -# parent environment which must be irace(). -# # FIXME: Restoring occurs after reading the command-line/scenario file. At # least for the irace command-line parameters (scenario), it should occur # before. We would need to: @@ -16,31 +13,23 @@ # # A work-around is to modify the recovery file (you can load it in R, # modify scenario then save it again). -recoverFromFile <- function(filename) +recoverFromFile <- function(filename, scenario = list()) { - # substitute() is needed to evaluate filename here. - eval.parent(substitute({ - iraceResults <- read_logfile(filename) - if (iraceResults$irace_version != irace::irace_version) - irace.error("Recovery file '", filename, "' was generated by a version of irace (", - iraceResults$irace_version, ") different from this version of irace (", - irace::irace_version, ").") - # These variables are not state, but they are used directly by irace. - for (name in c("allConfigurations")) - assign(name, iraceResults[[name]]) - # Restore part of scenario but not all. - for (name in .irace.params.recover) - scenario[[name]] <- iraceResults$scenario[[name]] - # We call checkScenario again to fix any inconsistencies in the recovered - # data. - # FIXME: Do not call checkScenario earlier and instead do the minimum to check recoveryFile. - scenario <- checkScenario(scenario) - race_state <- iraceResults$state$clone() - race_state$recover() - firstRace <- FALSE - options(.race.debug.level = scenario$debugLevel) - options(.irace.debug.level = scenario$debugLevel) - })) + iraceResults <- read_logfile(filename) + if (iraceResults$irace_version != irace::irace_version) + irace_error("Recovery file '", filename, "' was generated by a version of irace (", + iraceResults$irace_version, ") different from this version of irace (", + irace::irace_version, ").") + + # Restore part of scenario but not all. + for (name in .irace.params.recover) + scenario[[name]] <- iraceResults$scenario[[name]] + # We call checkScenario() again to fix any inconsistencies in the recovered data. + # FIXME: Do not call checkScenario earlier and instead do the minimum to check recoveryFile. + scenario <- checkScenario(scenario) + race_state <- iraceResults$state$clone() + race_state$initialize(scenario, recover = TRUE) + race_state } ## @@ -49,17 +38,17 @@ recoverFromFile <- function(filename) # FIXME: This function is too slow and it shows up in profiles. numeric.configurations.equal <- function(x, configurations, parameters, threshold, param.names) { - d <- rep(0.0, nrow(configurations)) + d <- numeric(nrow(configurations)) isSimilar <- matrix(TRUE, nrow = nrow(configurations), ncol = length(param.names)) selected <- seq_nrow(configurations) for (i in seq_along(param.names)) { pname <- param.names[i] param <- parameters$get(pname) - + is_dep <- param[["is_dependent"]] x_domain <- if (is_dep) getDependentBound(param, x) else param[["domain"]] x_range <- x_domain[[2L]] - x_domain[[1L]] - + X <- x[[pname]] # FIXME: Since at the end we select a subset of configurations, we could use selected here. y <- configurations[[pname]] @@ -71,7 +60,7 @@ numeric.configurations.equal <- function(x, configurations, parameters, threshol if (is.na (X) && is.na(Y)) { # Both NA, just ignore this pname next } else if (xor(is.na (X), is.na(Y))) { # Distance is 1.0, so not equal - isSimilar[j,i] <- FALSE + isSimilar[j,i] <- FALSE } else { # FIXME: Why is this updating d[j]? It seems that if the difference is # large for one configuration, then it will be assumed to be large for @@ -98,7 +87,7 @@ numeric.configurations.equal <- function(x, configurations, parameters, threshol selected <- selected[index] if (nrow(isSimilar) == 0L) break } - + if (length(selected) == 0L) return(NULL) c(x[[".ID."]], configurations[selected,".ID."]) @@ -112,16 +101,16 @@ similarConfigurations <- function(configurations, parameters, threshold) { # FIXME: Use data.table debug.level <- getOption(".irace.debug.level", 0) - if (debug.level >= 1) irace.note ("Computing similarity of configurations .") + if (debug.level >= 1) irace_note ("Computing similarity of configurations .") # Create vectors of categorical and numerical p <- parameters$types %in% c("c","o") vecCat <- parameters$names[p & !parameters$isFixed] vecNum <- parameters$names[!p & !parameters$isFixed] - irace.assert(all(parameters$types[vecCat] %in% c("c","o"))) - irace.assert(all(parameters$types[vecNum] %not_in% c("c","o"))) - irace.assert(length(intersect(vecCat, vecNum)) == 0) + irace_assert(all(parameters$types[vecCat] %in% c("c","o"))) + irace_assert(all(parameters$types[vecNum] %not_in% c("c","o"))) + irace_assert(length(intersect(vecCat, vecNum)) == 0) nbCater <- length(vecCat) nbNumer <- length(vecNum) @@ -130,7 +119,7 @@ similarConfigurations <- function(configurations, parameters, threshold) if (nbCater > 0) { ## Build a vector with the categorical appended together in a string strings <- do.call(paste, c(configurations[, vecCat, drop=FALSE], sep = " ; ")) - + if (nbNumer != 0) configurations <- configurations[, c(".ID.", vecNum)] ord.strings <- order(strings) configurations <- configurations[ord.strings, ] @@ -138,24 +127,24 @@ similarConfigurations <- function(configurations, parameters, threshold) ## keep similar (index i == true means is the same as i + 1) similarIdx <- strings[-length(strings)] == strings[-1] - + ## Now let's get just a FALSE if we remove it, TRUE otherwise: keepIdx <- c(similarIdx[1], (similarIdx[-1] | similarIdx[-length(similarIdx)]), similarIdx[length(similarIdx)]) - + ## filtering them out: configurations <- configurations [keepIdx, , drop=FALSE] ## filtering their strings out (to use them to define blocks): strings <- strings[keepIdx] - + ## if everything is already filtered out, return if (nrow(configurations) == 0) { if (debug.level >= 1) cat(" DONE\n") return(NULL) } } - + ### Numerical parameters within blocks of the same string ### if (nbNumer > 0) { similar <- c() @@ -169,7 +158,7 @@ similarConfigurations <- function(configurations, parameters, threshold) blockIds <- which(strings == strings[beginBlock]) endBlock <- blockIds[length(blockIds)] - irace.assert (endBlock > beginBlock) + irace_assert (endBlock > beginBlock) ## Loop inside blocks: for (i in seq(beginBlock, endBlock - 1L)) { ## Compare configuration i with all the ones that are after in the block @@ -200,7 +189,7 @@ similarConfigurations <- function(configurations, parameters, threshold) similar <- unique(similar) configurations <- configurations[configurations[[".ID."]] %in% similar, ] } - + if (debug.level >= 1) cat(" DONE\n") if (nrow(configurations) == 0L) return(NULL) configurations[[".ID."]] @@ -218,15 +207,16 @@ computeComputationalBudget <- function(remainingBudget, indexIteration, ## The number of configurations computeNbConfigurations <- function(currentBudget, indexIteration, mu, eachTest, blockSize, - nElites = 0L, nOldInstances = 0L, newInstances = 0L, + nElites = 0L, nOldInstances = 0L, newInstances = 0L, maxConfigurations = 1024L) { # FIXME: This is slightly incorrect, because we may have elites that have not # been executed on all nOldInstances. Thus, we need to pass explicitly the # budget that we save (that is, number of entries that are not NA). savedBudget <- nElites * nOldInstances - n <- max (mu + blockSize * eachTest * min(5L, indexIteration), - round_to_next_multiple(nOldInstances + newInstances, blockSize * eachTest)) + eachTest <- eachTest * blockSize + n <- max (mu + eachTest * min(5L, indexIteration), + round_to_next_multiple(nOldInstances + newInstances, eachTest)) min (floor ((currentBudget + savedBudget) / n), maxConfigurations) } @@ -240,9 +230,9 @@ computeMinimumBudget <- function(scenario, minSurvival, nbIterations, elitist_ne { blockSize <- scenario$blockSize eachTest <- blockSize * scenario$eachTest - Tnew <- elitist_new_instances + Tnew <- elitist_new_instances # This is already multiplied by blockSize. mu <- scenario$mu - + # This is computed from the default formulas as follows: # B_1 = B / I # B_2 = B - (B/I) / (I - 1) = B / I @@ -262,7 +252,7 @@ computeMinimumBudget <- function(scenario, minSurvival, nbIterations, elitist_ne # with a smaller budget. However, the exact formula requires computing B_i # taking into account the floor() function, which is not obvious. - minimumBudget <- (minSurvival + 1L) * nbIterations + minimumBudget <- (minSurvival + 1L) * nbIterations # We need to compute T_i: if (scenario$elitist) { @@ -315,11 +305,11 @@ checkMinimumBudget <- function(scenario, remainingBudget, minSurvival, nbIterati if (remainingBudget >= minimumBudget) return(TRUE) if (scenario$maxTime == 0) { - irace.error("Insufficient budget: ", + irace_error("Insufficient budget: ", "With the current settings, irace will require a value of ", "'maxExperiments' of at least '", minimumBudget, "'.") } else if (nbIterations == 1L) { - irace.error("Insufficient budget: ", + irace_error("Insufficient budget: ", "With the current settings and estimated time per run (", boundEstimate, ") irace will require a value of 'maxTime' of at least '", ceiling((minimumBudget * boundEstimate) + timeUsed), "'.") @@ -348,31 +338,31 @@ generateInstances <- function(race_state, scenario, n, update = FALSE) blockSize <- scenario$blockSize n_blocks <- length(instances) / blockSize # Sample instances index in groups (ntimes) - selected_blocks <- unlist(lapply(rep(n_blocks, ntimes), sample.int, replace = FALSE)) + selected_blocks <- unlist(lapply(rep.int(n_blocks, ntimes), sample.int, replace = FALSE)) sindex <- c(outer(seq_len(blockSize), (selected_blocks - 1L) * blockSize, "+")) } else { - sindex <- rep(seq_along(instances), ntimes) + sindex <- rep.int(seq_along(instances), ntimes) } # Sample seeds. - race_state$instances_log <- rbind(race_state$instances_log, - data.table(instanceID = sindex, seed = runif_integer(length(sindex)))) + race_state$instances_log <- rbindlist(use.names = TRUE, list( + race_state$instances_log, + data.table(instanceID = sindex, seed = runif_integer(length(sindex))))) race_state$instances_log } do_experiments <- function(race_state, configurations, ninstances, scenario, iteration) { instances <- seq_len(ninstances) - # FIXME: modify race_wrapper to take instances as an argument. - output <- lapply(instances, race_wrapper, race_state = race_state, - configurations = configurations, + output <- race_wrapper_helper(race_state = race_state, + configurations = configurations, + instance_idx = instances, bounds = rep(scenario$boundMax, nrow(configurations)), - which_alive = seq_nrow(configurations), which_exe = seq_nrow(configurations), - scenario = scenario) - + is_exe = rep_len(TRUE, nrow(configurations) * ninstances), scenario = scenario) + + set(output, j = "iteration", value = iteration) Results <- race_state$update_experiment_log(output, instances = instances, - configurations_id = configurations[[".ID."]], - scenario = scenario, iteration = iteration) - + scenario = scenario) + rejected_ids <- configurations[[".ID."]][colAnys(is.infinite(Results))] scenario$parameters$forbid_configurations( race_state$update_rejected(rejected_ids, configurations) @@ -384,38 +374,38 @@ do_experiments <- function(race_state, configurations, ninstances, scenario, ite allConfigurationsInit <- function(scenario) { initConfigurations <- scenario$initConfigurations - + confs_from_file <- NULL if (!is.null.or.empty(scenario$configurationsFile)) { confs_from_file <- readConfigurationsFile(scenario$configurationsFile, scenario$parameters, scenario$debugLevel) } - if (!is.null.or.empty(initConfigurations)) { + if (is.null.or.empty(initConfigurations)) { + initConfigurations <- confs_from_file + } else { if (!is.null.or.empty(scenario$configurationsFile) && !identical(initConfigurations, confs_from_file)) - irace.warning("'initConfigurations' provided in 'scenario',", + irace_warning("'initConfigurations' provided in 'scenario',", " thus ignoring configurations from file '", scenario$configurationsFile, "'.") cat("# Adding", nrow(initConfigurations), "initial configuration(s)\n") initConfigurations <- fix_configurations(initConfigurations, scenario$parameters, debugLevel = scenario$debugLevel) - } else { - initConfigurations <- confs_from_file } - - if (!is.null.or.empty(initConfigurations)) { + + if (is.null.or.empty(initConfigurations)) { + allConfigurations <- configurations_alloc(c(".ID.", scenario$parameters$names, ".PARENT."), + nrow = 0L, parameters = scenario$parameters) + setDF(allConfigurations) + } else { allConfigurations <- cbind(.ID. = seq_nrow(initConfigurations), initConfigurations, .PARENT. = NA_integer_) rownames(allConfigurations) <- allConfigurations[[".ID."]] num <- nrow(allConfigurations) allConfigurations <- checkForbidden(allConfigurations, scenario$parameters$forbidden) if (nrow(allConfigurations) < num) { - irace.warning(num - nrow(allConfigurations), " of the ", num, + irace_warning(num - nrow(allConfigurations), " of the ", num, " initial configurations were forbidden", " and, thus, discarded.") } - } else { - allConfigurations <- configurations_alloc(c(".ID.", scenario$parameters$names, ".PARENT."), - nrow = 0L, parameters = scenario$parameters) - setDF(allConfigurations) } allConfigurations } @@ -426,7 +416,7 @@ allConfigurationsInit <- function(scenario) # Output: nbElites elites, sorted by ranks, with the weights assigned. extractElites <- function(configurations, nbElites, debugLevel) { - irace.assert(nbElites > 0L) + irace_assert(nbElites > 0L) # Keep only alive configurations. elites <- as.data.table(configurations) before <- nrow(elites) @@ -436,11 +426,10 @@ extractElites <- function(configurations, nbElites, debugLevel) elites <- unique(elites, by=which(!startsWith(colnames(elites), "."))) after <- nrow(elites) if (debugLevel >= 2L && after < before) - irace.note("Dropped ", before - after, " duplicated elites.\n") + irace_note("Dropped ", before - after, " duplicated elites.\n") after <- min(after, nbElites) - .RANK. <- NULL - setorder(elites, .RANK.) + setorderv(elites, cols=".RANK.") selected <- seq_len(after) elites <- elites[selected, ] set(elites, j = ".WEIGHT.", value = ((after + 1L) - selected) / (after * (after + 1L) / 2)) @@ -462,14 +451,14 @@ extractElites <- function(configurations, nbElites, debugLevel) #' The execution of this function is reproducible under some conditions. See #' the FAQ section in the [User #' Guide](https://cran.r-project.org/package=irace/vignettes/irace-package.pdf). -#' +#' #' @inheritParams defaultScenario #' #' @template return_irace #' #' @examples #' \dontrun{ -#' # In general, there are three steps: +#' # In general, there are three steps: #' scenario <- readScenario(filename = "scenario.txt") #' irace(scenario = scenario) #' } @@ -496,10 +485,10 @@ extractElites <- function(configurations, nbElites, debugLevel) #' f_rastrigin <- function (x) { #' sum(x * x - 10 * cos(2 * pi * x) + 10) #' } -#' +#' #' ## We generate 20 instances (in this case, weights): #' weights <- rnorm(20, mean = 0.9, sd = 0.02) -#' +#' #' ## On this set of instances, we are interested in optimizing two #' ## parameters of the SANN algorithm: tmax and temp. We setup the #' ## parameter space as follows: @@ -509,7 +498,7 @@ extractElites <- function(configurations, nbElites, debugLevel) #' ' #' ## We use the irace function readParameters to read this table: #' parameters <- readParameters(text = parameters_table) -#' +#' #' ## Next, we define the function that will evaluate each candidate #' ## configuration on a single instance. For simplicity, we restrict to #' ## three-dimensional functions and we set the maximum number of @@ -518,7 +507,7 @@ extractElites <- function(configurations, nbElites, debugLevel) #' { #' instance <- experiment$instance #' configuration <- experiment$configuration -#' +#' #' D <- 3 #' par <- runif(D, min=-1, max=1) #' fn <- function(x) { @@ -542,7 +531,7 @@ extractElites <- function(configurations, nbElites, debugLevel) #' ## external program or function. #' return(list(cost = res$value)) #' } -#' +#' #' ## We define a configuration scenario by setting targetRunner to the #' ## function define above, instances to the first 10 random weights, and #' ## a maximum budget of 'maxExperiments' calls to targetRunner. @@ -552,20 +541,20 @@ extractElites <- function(configurations, nbElites, debugLevel) #' # Do not create a logFile #' logFile = "", #' parameters = parameters) -#' +#' #' ## We check that the scenario is valid. This will also try to execute #' ## target_runner. #' checkIraceScenario(scenario) -#' +#' #' \donttest{ #' ## We are now ready to launch irace. We do it by means of the irace #' ## function. The function will print information about its #' ## progress. This may require a few minutes, so it is not run by default. #' tuned_confs <- irace(scenario = scenario) -#' +#' #' ## We can print the best configurations found by irace as follows: #' configurations_print(tuned_confs) -#' +#' #' ## We can evaluate the quality of the best configuration found by #' ## irace versus the default configuration of the SANN algorithm on #' ## the other 10 instances previously generated. @@ -584,11 +573,11 @@ extractElites <- function(configurations, nbElites, debugLevel) #' ## To do so, first we apply the default configuration of the SANN #' ## algorithm to these instances: #' default <- test(data.frame(tmax=10, temp=10)) -#' +#' #' ## We extract and apply the winning configuration found by irace #' ## to these instances: #' tuned <- test(removeConfigurationsMetaData(tuned_confs[1,])) -#' +#' #' ## Finally, we can compare using a boxplot the quality obtained with the #' ## default parametrization of SANN and the quality obtained with the #' ## best configuration found by irace. @@ -612,19 +601,19 @@ irace <- function(scenario) irace_common <- function(scenario, simple, output.width = 9999L) { if (!simple) { - op <- options(width = output.width) # Do not wrap the output. - on.exit(options(op), add = TRUE) + withr::local_options(width = output.width) # Do not wrap the output. } scenario <- checkScenario(scenario) debugLevel <- scenario$debugLevel - if (debugLevel >= 1) { - op.debug <- options(warning.length = 8170) - if (!base::interactive()) options(error = irace.dump.frames) - on.exit(options(op.debug), add = TRUE) + if (debugLevel >= 1L) { + op <- list(warning.length = 8170L) + if (!base::interactive()) + op <- c(op, list(error = irace_dump_frames)) + withr::local_options(op) printScenario (scenario) } - + elite_configurations <- irace_run(scenario = scenario) if (simple) return(elite_configurations) @@ -633,7 +622,7 @@ irace_common <- function(scenario, simple, output.width = 9999L) cat("# Best configurations (first number is the configuration ID;", " listed from best to worst according to the ", order_str, "):\n", sep = "") configurations_print(elite_configurations) - + cat("# Best configurations as commandlines (first number is the configuration ID;", " listed from best to worst according to the ", order_str, "):\n", sep = "") configurations_print_command (elite_configurations, scenario$parameters) } @@ -643,13 +632,21 @@ irace_common <- function(scenario, simple, output.width = 9999L) irace_run <- function(scenario) { + # Recover state from file? + if (is.null.or.empty(scenario$recoveryFile)) { + race_state <- RaceState$new(scenario) + } else { + irace_note ("Recovering from file: '", scenario$recoveryFile,"'\n") + race_state <- recoverFromFile(scenario$recoveryFile, scenario = scenario) + } + quiet <- scenario$quiet catInfo <- if (quiet) do_nothing else function(..., verbose = TRUE) { - irace.note (..., "\n") + irace_note (..., "\n") if (verbose) { cat ("# Iteration: ", indexIteration, "\n", "# nbIterations: ", nbIterations, "\n", - "# experimentsUsedSoFar: ", nrow(race_state$experiment_log), "\n", + "# experimentsUsed: ", experimentsUsed, "\n", "# timeUsed: ", timeUsed, "\n", "# remainingBudget: ", remainingBudget, "\n", "# currentBudget: ", currentBudget, "\n", @@ -667,204 +664,199 @@ irace_run <- function(scenario) # FIXME: Do we need to clone? race_state$completed <- reason iraceResults$state <- race_state - irace_save_logfile(iraceResults, scenario) + save_irace_logfile(iraceResults, logfile = scenario$logFile) # FIXME: Handle scenario$maxTime > 0 - if (scenario$postselection && scenario$maxTime == 0 && floor(remainingBudget / scenario$blockSize) > 1L) - psRace(iraceResults, max_experiments = remainingBudget, iteration_elites=TRUE) + if (scenario$postselection && scenario$maxTime == 0 && floor(remainingBudget / max(scenario$blockSize, scenario$eachTest)) > 1L) + psRace(iraceResults, max_experiments = remainingBudget, iteration_elites = TRUE) else - race_state$elite_configurations + elite_configurations } debugLevel <- scenario$debugLevel + # Set options controlling debug level. + # FIXME: This should be the other way around, the options set the debugLevel. + options(.race.debug.level = debugLevel) + options(.irace.debug.level = debugLevel) - # Recover state from file? - if (!is.null.or.empty(scenario$recoveryFile)) { - irace.note ("Resuming from file: '", scenario$recoveryFile,"'\n") - recoverFromFile(scenario$recoveryFile) - race_state$start_parallel(scenario) - on.exit(race_state$stop_parallel(), add = TRUE) - } else { # Do not recover - firstRace <- TRUE - race_state <- RaceState$new(scenario) - # Set options controlling debug level. - # FIXME: This should be the other way around, the options set the debugLevel. - options(.race.debug.level = debugLevel) - options(.irace.debug.level = debugLevel) - # Create a data frame of all configurations ever generated. - allConfigurations <- allConfigurationsInit(scenario) - irace.assert(is.integer(allConfigurations[[".ID."]])) - nbUserConfigurations <- nrow(allConfigurations) - - # To save the logs - iraceResults <- list( - scenario = scenario, - irace_version = irace_version, - iterationElites = c(), - allElites = list(), - experiments = matrix(nrow = 0L, ncol = 0L)) - blockSize <- scenario$blockSize - model <- NULL - nbConfigurations <- 0L - elite_configurations <- data.frame(stringsAsFactors=FALSE) - - nbIterations <- if (scenario$nbIterations == 0) - computeNbIterations(scenario$parameters$nbVariable) - else scenario$nbIterations - nbIterations <- floor(nbIterations) - - minSurvival <- if (scenario$minNbSurvival == 0) - computeTerminationOfRace(scenario$parameters$nbVariable) - else scenario$minNbSurvival - minSurvival <- floor(minSurvival) - - # Generate initial instance + seed list - generateInstances(race_state, scenario, - n = if (scenario$maxExperiments != 0) ceiling(scenario$maxExperiments / minSurvival) - else max(scenario$firstTest, length(scenario$instances))) - indexIteration <- 1L - timeUsed <- 0 - boundEstimate <- NA - - race_state$start_parallel(scenario) - on.exit(race_state$stop_parallel(), add = TRUE) - - if (scenario$maxTime == 0) { - if (is.na(scenario$minExperiments)) { - remainingBudget <- scenario$maxExperiments - } else { - remainingBudget <- max(scenario$minExperiments, - computeMinimumBudget(scenario, minSurvival, nbIterations, - race_state$elitist_new_instances)) - } - } else { ## Estimate time when maxTime is defined. - ## IMPORTANT: This is firstTest because these configurations will be - ## considered elite later, thus preserved up to firstTest, which is - ## fine. If a larger number of instances is used, it would prevent - ## discarding these configurations. - # Get the number of instances to be used. - ninstances <- round_to_next_multiple(scenario$firstTest, scenario$blockSize) - estimationTime <- ceiling(scenario$maxTime * scenario$budgetEstimation) - irace.note("Estimating execution time using ", 100 * scenario$budgetEstimation, - "% of ", scenario$maxTime, " = ", estimationTime, "\n") - - # Estimate the number of configurations to be used - nconfigurations <- max(2L, floor(scenario$parallel / ninstances)) - next_configuration <- 1L - nruns <- nconfigurations * ninstances - boundEstimate <- if (is.null(scenario$boundMax)) 1.0 else scenario$boundMax - if (estimationTime < boundEstimate * nruns) { - boundEstimate <- max(ceiling_digits(estimationTime / nruns, scenario$boundDigits), scenario$minMeasurableTime) - if (!is.null(scenario$boundMax)) { - irace.warning("boundMax = ", scenario$boundMax, " is too large, using ", boundEstimate, " instead.\n") + firstRace <- TRUE + # Create a data frame of all configurations ever generated. + allConfigurations <- allConfigurationsInit(scenario) + irace_assert(is.integer(allConfigurations[[".ID."]])) + nbUserConfigurations <- nrow(allConfigurations) + + # To save the logs + iraceResults <- list( + scenario = scenario, + irace_version = irace_version, + iterationElites = c(), + allElites = list(), + experiments = matrix(nrow = 0L, ncol = 0L)) + model <- NULL + nbConfigurations <- 0L + elite_configurations <- data.frame(stringsAsFactors=FALSE) + + nbIterations <- if (scenario$nbIterations == 0) + computeNbIterations(scenario$parameters$nbVariable) + else scenario$nbIterations + nbIterations <- floor(nbIterations) + + minSurvival <- if (scenario$minNbSurvival == 0) + computeTerminationOfRace(scenario$parameters$nbVariable) + else scenario$minNbSurvival + minSurvival <- floor(minSurvival) + # FIXME: Do this initialization within race_state. + race_state$minSurvival <- minSurvival + + # Generate initial instance + seed list + generateInstances(race_state, scenario, + n = if (scenario$maxExperiments != 0) ceiling(scenario$maxExperiments / minSurvival) + else max(scenario$firstTest, length(scenario$instances))) + indexIteration <- 1L + experimentsUsed <- 0L + timeUsed <- 0 + boundEstimate <- NA + + race_state$start_parallel(scenario) + on.exit(race_state$stop_parallel(), add = TRUE) + + if (scenario$maxTime == 0) { + if (is.na(scenario$minExperiments)) { + remainingBudget <- scenario$maxExperiments + } else { + remainingBudget <- max(scenario$minExperiments, + computeMinimumBudget(scenario, minSurvival, nbIterations, + race_state$elitist_new_instances)) + } + } else { ## Estimate time when maxTime is defined. + ## IMPORTANT: This is firstTest because these configurations will be + ## considered elite later, thus preserved up to firstTest, which is + ## fine. If a larger number of instances is used, it would prevent + ## discarding these configurations. + # Get the number of instances to be used. + ninstances <- scenario$firstTest * scenario$blockSize + estimationTime <- ceiling(scenario$maxTime * scenario$budgetEstimation) + irace_note("Estimating execution time using ", 100 * scenario$budgetEstimation, + "% of ", scenario$maxTime, " = ", estimationTime, "\n") + + # Estimate the number of configurations to be used + nconfigurations <- max(2L, floor(scenario$parallel / ninstances)) + next_configuration <- 1L + nruns <- nconfigurations * ninstances + boundEstimate <- if (is.null(scenario$boundMax)) 1.0 else scenario$boundMax + if (estimationTime < boundEstimate * nruns) { + boundEstimate <- max(ceiling_digits(estimationTime / nruns, scenario$boundDigits), scenario$minMeasurableTime) + if (!is.null(scenario$boundMax)) { + irace_warning("boundMax=", scenario$boundMax, " is too large, using ", boundEstimate, " instead.\n") # FIXME: We should not modify the scenario scenario$boundMax <- boundEstimate - } } - - repeat { - # Sample new configurations if needed + } + + repeat { + # Sample new configurations if needed if (nrow(allConfigurations) < nconfigurations) { newConfigurations <- sampleSobol(scenario$parameters, - nconfigurations - nrow(allConfigurations), - repair = scenario$repairConfiguration) + nconfigurations - nrow(allConfigurations), + repair = scenario$repairConfiguration) set(newConfigurations, j = ".ID.", value = max(0L, vlast(allConfigurations[[".ID."]])) + seq_nrow(newConfigurations)) setcolorder(newConfigurations, ".ID.", before=1L) setDF(newConfigurations) + # FIXME: use rbindlist(use.names=TRUE) allConfigurations <- rbind(allConfigurations, newConfigurations) rownames(allConfigurations) <- allConfigurations[[".ID."]] # We may have generated less than the number requested if there were duplicates. nconfigurations <- nrow(allConfigurations) } - # Estimate the mean execution time. - # FIXME: Shouldn't we pass the bounds? - experiments <- do_experiments(race_state, - configurations = allConfigurations[next_configuration:nconfigurations, ], - ninstances = ninstances, scenario = scenario, - # These experiments are assigned iteration 0. - iteration = 0L) - # FIXME: Here we should check if everything timed out and increase the bound dynamically. - iraceResults$experiments <- merge_matrix(iraceResults$experiments, experiments) - rownames(iraceResults$experiments) <- seq_nrow(iraceResults$experiments) - # For the used time, we count the time reported in all configurations - # including rejected ones. - timeUsed <- sum(race_state$experiment_log[["time"]], na.rm = TRUE) - # User should return time zero for rejected_ids. - boundEstimate <- timeUsed / nrow(race_state$experiment_log) - boundEstimate <- max(ceiling_digits(boundEstimate, scenario$boundDigits), scenario$minMeasurableTime) - next_configuration <- nconfigurations + 1L - - # Calculate how many new configurations: - # 1. We do not want to overrun estimationTime - new_conf <- floor(((estimationTime - timeUsed) / boundEstimate) / ninstances) - # 2. But there is no point in executing more configurations than those - # that we can execute in parallel. - new_conf <- min(new_conf, max(1L, floor(scenario$parallel / ninstances))) - if (timeUsed >= estimationTime || new_conf == 0L || nconfigurations == 1024L) - break - else - nconfigurations <- min(1024L, nconfigurations + new_conf) - } # end of repeat - - if (length(race_state$rejected_ids)) - irace.note ("Immediately rejected configurations: ", - paste0(race_state$rejected_ids, collapse = ", ") , "\n") - - # Update budget - remainingBudget <- round((scenario$maxTime - timeUsed) / boundEstimate) - elite_configurations <- allConfigurations[allConfigurations[[".ID."]] %not_in% race_state$rejected_ids, , drop = FALSE] - irace.assert(is.integer(elite_configurations[[".ID."]])) - # Without elitist, the racing does not re-use the results computed during - # the estimation. This means that the time used during estimation needs - # to be spent again during racing, thus leaving less time for racing. We - # want to avoid having less time for racing, and this is an - # implementation detail, thus we assume that the time was not actually - # wasted. - if (!scenario$elitist) timeUsed <- 0 - irace.note("Estimated execution time is ", boundEstimate, " based on ", - next_configuration - 1L, " configurations and ", - ninstances," instances. Used time: ", timeUsed, - ", remaining time: ", (scenario$maxTime - timeUsed), - ", remaining budget (experiments): ", remainingBudget, "\n") - if (!is.null(scenario$boundMax) && 2 * boundEstimate < scenario$boundMax) { - irace.warning("boundMax=", scenario$boundMax, " is much larger than estimated execution time, using ", - 2 * boundEstimate, " instead.\n") - scenario$boundMax <- 2 * boundEstimate - } - } # end of time estimation + # Estimate the mean execution time. + # FIXME: Shouldn't we pass the bounds? + experiments <- do_experiments(race_state, + configurations = allConfigurations[next_configuration:nconfigurations, ], + ninstances = ninstances, scenario = scenario, + # These experiments are assigned iteration 0. + iteration = 0L) + # FIXME: Here we should check if everything timed out and increase the bound dynamically. + iraceResults$experiments <- merge_matrix(iraceResults$experiments, experiments) + rownames(iraceResults$experiments) <- seq_nrow(iraceResults$experiments) + # For the used time, we count the time reported in all configurations + # including rejected ones. + timeUsed <- sum(race_state$experiment_log[["time"]], na.rm = TRUE) + experimentsUsed <- nrow(race_state$experiment_log) + # User should return time zero for rejected_ids. + boundEstimate <- timeUsed / experimentsUsed + boundEstimate <- max(ceiling_digits(boundEstimate, scenario$boundDigits), scenario$minMeasurableTime) + next_configuration <- nconfigurations + 1L - # Compute the total initial budget, that is, the maximum number of - # experiments that we can perform. - currentBudget <- if (scenario$nbExperimentsPerIteration == 0L) - computeComputationalBudget(remainingBudget, indexIteration, - nbIterations) - else scenario$nbExperimentsPerIteration + # Calculate how many new configurations: + # 1. We do not want to overrun estimationTime + new_conf <- floor(((estimationTime - timeUsed) / boundEstimate) / ninstances) + # 2. But there is no point in executing more configurations than those + # that we can execute in parallel. + new_conf <- min(new_conf, max(1L, floor(scenario$parallel / ninstances))) + if (timeUsed >= estimationTime || new_conf == 0L || nconfigurations == 1024L) + break + else + nconfigurations <- min(1024L, nconfigurations + new_conf) + } # end of repeat + + if (length(race_state$rejected_ids)) + irace_note ("Immediately rejected configurations: ", + paste0(race_state$rejected_ids, collapse = ", ") , "\n") - # Check that the budget is enough. For the time estimation case we reduce - # the number of iterations. - warn_msg <- NULL - while (!checkMinimumBudget(scenario, remainingBudget, minSurvival, nbIterations, - boundEstimate, timeUsed, race_state$elitist_new_instances)) - { - if (is.null(warn_msg)) - warn_msg <- - paste0("With the current settings and estimated time per run (", - boundEstimate, - ") irace will not have enough budget to execute the minimum", - " number of iterations (", nbIterations, "). ", - "Execution will continue by assuming that the estimated time", - " is too high and reducing the minimum number of iterations,", - " however, if the estimation was correct or too low,", - " results might not be better than random sampling.\n") - nbIterations <- nbIterations - 1L - scenario$nbConfigurations <- if (scenario$nbConfigurations > 0L) - min(minSurvival * 2L, scenario$nbConfigurations) - else minSurvival * 2L + # Update budget + remainingBudget <- round((scenario$maxTime - timeUsed) / boundEstimate) + elite_configurations <- allConfigurations[allConfigurations[[".ID."]] %not_in% race_state$rejected_ids, , drop = FALSE] + irace_assert(is.integer(elite_configurations[[".ID."]])) + # Without elitist, the racing does not re-use the results computed during + # the estimation. This means that the time used during estimation needs + # to be spent again during racing, thus leaving less time for racing. We + # want to avoid having less time for racing, and this is an + # implementation detail, thus we assume that the time was not actually + # wasted. + if (!scenario$elitist) timeUsed <- 0 + irace_note("Estimated execution time is ", boundEstimate, " based on ", + next_configuration - 1L, " configurations and ", + ninstances," instances. Used time: ", timeUsed, + ", remaining time: ", (scenario$maxTime - timeUsed), + ", remaining budget (experiments): ", remainingBudget, "\n") + if (!is.null(scenario$boundMax) && 2 * boundEstimate < scenario$boundMax) { + irace_warning("boundMax=", scenario$boundMax, " is much larger than estimated execution time, using ", + 2 * boundEstimate, " instead.\n") + scenario$boundMax <- 2 * boundEstimate } - if (!is.null(warn_msg)) irace.warning(warn_msg) - - } #end of do not recover - + } # end of time estimation + + # Compute the total initial budget, that is, the maximum number of + # experiments that we can perform. + currentBudget <- if (scenario$nbExperimentsPerIteration == 0L) + computeComputationalBudget(remainingBudget, indexIteration, + nbIterations) + else scenario$nbExperimentsPerIteration + + # Check that the budget is enough. For the time estimation case we reduce + # the number of iterations. + warn_msg <- NULL + while (!checkMinimumBudget(scenario, remainingBudget, minSurvival, nbIterations, + boundEstimate, timeUsed, race_state$elitist_new_instances)) + { + if (is.null(warn_msg)) + warn_msg <- + paste0("With the current settings and estimated time per run (", + boundEstimate, + ") irace will not have enough budget to execute the minimum", + " number of iterations (", nbIterations, "). ", + "Execution will continue by assuming that the estimated time", + " is too high and reducing the minimum number of iterations,", + " however, if the estimation was correct or too low,", + " results might not be better than random sampling.\n") + nbIterations <- nbIterations - 1L + scenario$nbConfigurations <- if (scenario$nbConfigurations > 0L) + min(minSurvival * 2L, scenario$nbConfigurations) + else minSurvival * 2L + } + if (!is.null(warn_msg)) + irace_warning(warn_msg) + catInfo("Initialization\n", if (scenario$elitist) paste0("# Elitist race\n", @@ -881,40 +873,34 @@ irace_run <- function(scenario) else paste0("# time budget: ", scenario$maxTime - timeUsed, "\n"), "# mu: ", scenario$mu, "\n", "# deterministic: ", scenario$deterministic, "\n", - - if (scenario$capping) + + if (scenario$capping) paste0("# capping: ", scenario$cappingType, "\n", - "# type bound: ", scenario$boundType, "\n", + "# type bound: ", scenario$boundType, "\n", "# boundMax: ", scenario$boundMax, "\n", - "# par bound: ", scenario$boundPar, "\n", + "# par bound: ", scenario$boundPar, "\n", "# bound digits: ", scenario$boundDigits, "\n") else if (!is.null(scenario$boundMax)) paste0("# boundMax: ", scenario$boundMax, "\n"), verbose = FALSE) - + blockSize <- scenario$blockSize + repeat { - # Save recovery info. - race_state$save_recovery(currentBudget = currentBudget, - elite_configurations = elite_configurations, - experimentsUsedSoFar = nrow(race_state$experiment_log), - indexIteration = indexIteration, - minSurvival = minSurvival, - model = model, - nbConfigurations = nbConfigurations, - nbIterations = nbIterations, - remainingBudget = remainingBudget, - timeUsed = timeUsed, - boundEstimate = boundEstimate) + # FIXME: We could directly use race_state$timeUsed everywhere. + race_state$timeUsed <- timeUsed ## Save to the log file. iraceResults$allConfigurations <- allConfigurations - iraceResults$state <- race_state - irace_save_logfile(iraceResults, scenario) - - # Consistency checks - # With elitist=TRUE we should never re-run the same configuration on the same (instance,seed) pair - if (scenario$elitist) - irace.assert(sum(!is.na(iraceResults$experiments)) == nrow(race_state$experiment_log)) + race_state$save_recovery(iraceResults, logfile = scenario$logFile) + + + # With elitist=TRUE and without targetEvaluator we should never re-run the + # same configuration on the same (instance,seed) pair. + if (scenario$elitist) { + irace_assert(sum(!is.na(iraceResults$experiments)) == experimentsUsed) + if (is.null(scenario$targetEvaluator)) + irace_assert(experimentsUsed == nrow(race_state$experiment_log)) + } if (remainingBudget <= 0) { catInfo("Stopped because budget is exhausted") @@ -929,9 +915,8 @@ irace_run <- function(scenario) if (scenario$nbIterations == 0L) { nbIterations <- indexIteration } else { - if (debugLevel >= 1L) { + if (debugLevel >= 1L) catInfo("Limit of iterations reached", verbose = FALSE) - } return(irace_finish(iraceResults, scenario, reason = "Limit of iterations reached")) } } @@ -940,7 +925,7 @@ irace_run <- function(scenario) currentBudget <- if (scenario$nbExperimentsPerIteration == 0L) computeComputationalBudget(remainingBudget, indexIteration, nbIterations) else scenario$nbExperimentsPerIteration - + # Compute the number of configurations for this race. if (scenario$elitist && !firstRace) { nbConfigurations <- @@ -964,8 +949,8 @@ irace_run <- function(scenario) } # If still not enough budget, then try to do at least one test. if (nbConfigurations <= minSurvival) { - nbConfigurations <- computeNbConfigurations(currentBudget, indexIteration = 1, - mu = 1, eachTest = scenario$eachTest, blockSize = blockSize, + nbConfigurations <- computeNbConfigurations(currentBudget, indexIteration = 1L, + mu = 1L, eachTest = scenario$eachTest, blockSize = blockSize, nElites = nrow(elite_configurations), nOldInstances = nrow(iraceResults$experiments), newInstances = 0L) @@ -979,7 +964,7 @@ irace_run <- function(scenario) nElites = 0L, nOldInstances = 0L, newInstances = 0L) } - + # If a value was given as a parameter, then this value limits the maximum, # but if we have budget only for less than this, then we have run out of # budget. @@ -997,7 +982,7 @@ irace_run <- function(scenario) return(irace_finish(iraceResults, scenario, reason = "Not enough budget to enforce the value of nbConfigurations")) } } - + # Stop if the number of configurations to test is NOT larger than the minimum. if (nbConfigurations <= minSurvival) { catInfo("Stopped because there is not enough budget left to race more than ", @@ -1020,10 +1005,10 @@ irace_run <- function(scenario) # the number of elites. catInfo("Stopped because ", "there is not enough budget left to race newly sampled configurations.") - #(number of elites + 1) * (mu + min(5, indexIteration)) > remainingBudget" + #(number of elites + 1) * (mu + min(5, indexIteration)) > remainingBudget" return(irace_finish(iraceResults, scenario, reason = "Not enough budget left to race newly sampled configurations")) } - + if (scenario$elitist) { # The non-elite have to run up to the first test. The elites consume # budget at most up to the new instances. @@ -1039,7 +1024,7 @@ irace_run <- function(scenario) } catInfo("Iteration ", indexIteration, " of ", nbIterations, "\n", - "# experimentsUsedSoFar: ", nrow(race_state$experiment_log), "\n", + "# experimentsUsed: ", experimentsUsed, "\n", if (scenario$maxTime == 0) "" else paste0("# timeUsed: ", timeUsed, "\n", "# boundEstimate: ", boundEstimate, "\n"), @@ -1047,7 +1032,7 @@ irace_run <- function(scenario) "# currentBudget: ", currentBudget, "\n", "# nbConfigurations: ", nbConfigurations, verbose = FALSE) - + iraceResults$softRestart[indexIteration] <- FALSE # Sample for the first time. if (firstRace) { @@ -1076,7 +1061,7 @@ irace_run <- function(scenario) catInfo("Only ", nbConfigurations, " from the initial configurations will be used", verbose = FALSE) } - + # This is made only in case that the number of configurations used in # the time estimation is more than needed. if (nrow(elite_configurations) == nbConfigurations) { @@ -1086,19 +1071,19 @@ irace_run <- function(scenario) raceConfigurations <- raceConfigurations[seq_len(nbConfigurations), , drop = FALSE] } } # end of indexIteration == 1 - + } else { # How many new configurations should be sampled? nbNewConfigurations <- nbConfigurations - nrow(elite_configurations) # Update the model based on elites configurations - if (debugLevel >= 1L) irace.note("Update model\n") + if (debugLevel >= 1L) irace_note("Update model\n") model <- updateModel(scenario$parameters, elite_configurations, model, indexIteration, nbIterations, nbNewConfigurations, elitist = scenario$elitist) if (debugLevel >= 2L) printModel (model) if (debugLevel >= 1L) - irace.note("Sample ", nbNewConfigurations, " configurations from model\n") - irace.assert(is.integer(elite_configurations[[".ID."]])) + irace_note("Sample ", nbNewConfigurations, " configurations from model\n") + irace_assert(is.integer(elite_configurations[[".ID."]])) newConfigurations <- sampleModel(scenario$parameters, elite_configurations, model, nbNewConfigurations, repair = scenario$repairConfiguration) @@ -1118,7 +1103,7 @@ irace_run <- function(scenario) # Rprof(NULL) if (!is.null(restart_ids)) { if (debugLevel >= 1L) - irace.note("Soft restart: ", paste(collapse = " ", restart_ids), " !\n") + irace_note("Soft restart: ", paste(collapse = " ", restart_ids), " !\n") model <- restartModel(model, raceConfigurations, restart_ids, scenario$parameters, nbNewConfigurations) iraceResults$softRestart[indexIteration] <- TRUE @@ -1146,7 +1131,7 @@ irace_run <- function(scenario) } if (debugLevel >= 2L) { - irace.note("Configurations for the race n ", indexIteration, + irace_note("Configurations for the race n ", indexIteration, " (elite configurations listed first, then new configurations):\n") configurations_print(raceConfigurations, metadata = TRUE) } @@ -1165,59 +1150,66 @@ irace_run <- function(scenario) update = TRUE) } - if (debugLevel >= 1L) irace.note("Launch race\n") + if (debugLevel >= 1L) irace_note("Launch race\n") raceResults <- elitist_race (race_state, scenario = scenario, configurations = raceConfigurations, maxExp = currentBudget, minSurvival = minSurvival, - elite.data = elite_data, + elite_data = elite_data, elitist_new_instances = if (firstRace) 0L - else race_state$elitist_new_instances) + else race_state$elitist_new_instances) # We add indexIteration as an additional column. set(raceResults$experiment_log, j = "iteration", value = indexIteration) - race_state$experiment_log <- rbind(race_state$experiment_log, raceResults$experiment_log) + # FIXME: There is a chance that the process stops after we remove + # race_experiment_log in elitist_race(), but before we update + # race_state$experiment_log here. Doing the two steps in a different order + # would be more robust but would need a smarter recovery routine that + # checks for duplicates. + race_state$experiment_log <- rbindlist(list(race_state$experiment_log, raceResults$experiment_log), + use.names=TRUE) # Merge new results. iraceResults$experiments <- merge_matrix(iraceResults$experiments, raceResults$experiments) - + # Update remaining budget. - if (scenario$maxTime > 0L) { + experimentsUsed <- experimentsUsed + raceResults$experimentsUsed + if (scenario$maxTime > 0L) { timeUsed <- timeUsed + sum(raceResults$experiment_log[["time"]], na.rm = TRUE) - boundEstimate <- timeUsed / nrow(race_state$experiment_log) + boundEstimate <- timeUsed / experimentsUsed remainingBudget <- round((scenario$maxTime - timeUsed) / boundEstimate) } else { - remainingBudget <- remainingBudget - nrow(raceResults$experiment_log) + remainingBudget <- remainingBudget - raceResults$experimentsUsed } if (debugLevel >= 3L) { - irace.note("Results for the race of iteration ", indexIteration, + irace_note("Results for the race of iteration ", indexIteration, " (from best to worst, according to the ", test.type.order.str(scenario$testType), "):\n") configurations_print(raceResults$configurations, metadata = TRUE) } - if (debugLevel >= 1L) irace.note("Extracting elites\n") + if (debugLevel >= 1L) irace_note("Extracting elites\n") elite_configurations <- extractElites(raceResults$configurations, nbElites = minSurvival, debugLevel = scenario$debugLevel) - irace.note("Elite configurations (first number is the configuration ID;", + irace_note("Elite configurations (first number is the configuration ID;", " listed from best to worst according to the ", test.type.order.str(scenario$testType), "):\n") if (!quiet) configurations_print(elite_configurations, metadata = debugLevel >= 1L) iraceResults$iterationElites[indexIteration] <- elite_configurations[[".ID."]][1L] iraceResults$allElites[[indexIteration]] <- elite_configurations[[".ID."]] - + if (firstRace) { - if (debugLevel >= 1L) irace.note("Initialise model\n") + if (debugLevel >= 1L) irace_note("Initialise model\n") model <- initialiseModel(scenario$parameters, elite_configurations) if (debugLevel >= 2L) printModel (model) firstRace <- FALSE } - + if (debugLevel >= 1L) { - irace.note("End of iteration ", indexIteration, "\n") + irace_note("End of iteration ", indexIteration, "\n") if (debugLevel >= 3L) { - irace.note("All configurations (sampling order):\n") + irace_note("All configurations (sampling order):\n") configurations_print(allConfigurations, metadata = TRUE) - irace.note("Memory used in irace():\n") + irace_note("Memory used in irace():\n") race_state$print_mem_used() } } diff --git a/R/irace_summarise.R b/R/irace_summarise.R index 0a9fa9ba..851800e1 100644 --- a/R/irace_summarise.R +++ b/R/irace_summarise.R @@ -37,6 +37,7 @@ irace_summarise <- function(iraceResults) if (is.null(version)) version <- iraceResults$irace.version + # Here to support older versions of irace. time_targetrunner <- iraceResults$state$recovery_info$timeUsed if (is.null(time_targetrunner)) time_targetrunner <- iraceResults$state$timeUsed @@ -51,7 +52,7 @@ irace_summarise <- function(iraceResults) n_configurations = nrow(iraceResults$allConfigurations), n_initial_configurations = if (is.null(iraceResults$scenario$initConfigurations)) 0L else nrow(iraceResults$scenario$initConfigurations), n_instances = nrow(iraceResults$experiments), - n_experiments = nrow(experiment_log), + n_experiments = if (is.null(iraceResults$scenario$targetEvaluator)) nrow(experiment_log) else sum(!is.na(iraceResults$experiments)), n_elites = length(iraceResults$allElites[[n_iterations]]), n_soft_restarts = sum(iraceResults$softRestart), n_rejected = length(rejected_ids), diff --git a/R/main.R b/R/main.R index ba28748c..15c37612 100644 --- a/R/main.R +++ b/R/main.R @@ -2,8 +2,8 @@ # irace: An implementation in R of Iterated Race. # ------------------------------------------------------------------------- # -# Copyright (C) 2010-2020 -# Manuel López-Ibáñez +# Copyright (C) 2010-2025 +# Manuel López-Ibáñez # Jérémie Dubois-Lacoste # Leslie Perez Caceres # @@ -12,12 +12,12 @@ # it and/or modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 of the # License, or (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. -# +# # You should have received a copy of the GNU General Public License # along with this program; if not, you can obtain a copy of the GNU # General Public License at: @@ -31,7 +31,7 @@ #' irace_license #' #' A character string containing the license information of \pkg{irace}. -#' +#' #' @export ## __VERSION__ below will be replaced by the version defined in R/version.R ## This avoids constant conflicts within this file. @@ -39,9 +39,9 @@ irace_license <- '#------------------------------------------------------------------------------ # irace: An implementation in R of (Elitist) Iterated Racing # Version: __VERSION__ -# Copyright (C) 2010-2020 +# Copyright (C) 2010-2025 # Manuel Lopez-Ibanez -# Jeremie Dubois-Lacoste +# Jeremie Dubois-Lacoste # Leslie Perez Caceres # # This is free software, and you are welcome to redistribute it under certain @@ -54,14 +54,12 @@ irace_license <- #------------------------------------------------------------------------------ ' cat_irace_license <- function() -{ cat(sub("__VERSION__", irace_version, irace_license, fixed=TRUE)) -} #' Higher-level interface to launch irace. #' #' @inheritParams defaultScenario -#' +#' #' @param output.width `integer(1)`\cr The width used for the screen #' output. #' @@ -87,17 +85,17 @@ cat_irace_license <- function() irace_main <- function(scenario, output.width = 9999L) irace_common(scenario = scenario, simple=FALSE, output.width = output.width) -#' Test configurations given in `.Rdata` file +#' Test configurations given in the logfile (typically `irace.Rdata`) produced by \pkg{irace}. #' #' `testing_fromlog` executes the testing of the target algorithm configurations #' found by an \pkg{irace} execution. -#' -#' @param logFile Path to the `.Rdata` file produced by \pkg{irace}. +#' +#' @param logFile `character(1)`\cr Path to the logfile (typically `irace.Rdata`) produced by \pkg{irace}. #' #' @param testNbElites Number of (final) elite configurations to test. Overrides #' the value found in `logFile`. -#' -#' @param testIterationElites (`logical(1)`) If `FALSE`, only the final +#' +#' @param testIterationElites `logical(1)`\cr If `FALSE`, only the final #' `testNbElites` configurations are tested; otherwise, also test the best #' configurations of each iteration. Overrides the value found in `logFile`. #' @@ -107,8 +105,8 @@ irace_main <- function(scenario, output.width = 9999L) #' #' @param testInstances Character vector of the instances to be used in the `targetRunner` when executing the testing. #' -#' @return Boolean. `TRUE` if the testing ended successfully otherwise, `FALSE`. -#' +#' @return `logical(1)`\cr `TRUE` if the testing ended successfully otherwise, `FALSE`. +#' #' @details The function `testing_fromlog` loads the `logFile` and obtains the #' testing setup and configurations to be tested. Within the `logFile`, the #' variable `scenario$testNbElites` specifies how many final elite @@ -119,7 +117,7 @@ irace_main <- function(scenario, output.width = 9999L) #' #' @seealso [defaultScenario()] to provide a default scenario for \pkg{irace}. #' [testing_fromfile()] provides a different interface for testing. -#' +#' #' @author Manuel López-Ibáñez and Leslie Pérez Cáceres #' @concept running #' @export @@ -127,13 +125,13 @@ testing_fromlog <- function(logFile, testNbElites, testIterationElites, testInstancesDir, testInstancesFile, testInstances) { if (is.null.or.empty(logFile)) { - irace.note("No logFile provided to perform the testing of configurations. Skipping testing.\n") + irace_note("No logFile provided to perform the testing of configurations. Skipping testing.\n") return(FALSE) } iraceResults <- read_logfile(logFile) scenario <- iraceResults[["scenario"]] instances_changed <- FALSE - + if (!missing(testNbElites)) scenario[["testNbElites"]] <- testNbElites if (!missing(testIterationElites)) @@ -141,7 +139,7 @@ testing_fromlog <- function(logFile, testNbElites, testIterationElites, if (!missing(testInstances)) scenario[["testInstances"]] <- testInstances - + if (!missing(testInstancesDir)) { scenario$testInstancesDir <- testInstancesDir instances_changed <- TRUE @@ -150,8 +148,8 @@ testing_fromlog <- function(logFile, testNbElites, testIterationElites, scenario$testInstancesFile <- testInstancesFile instances_changed <- TRUE } - - cat("\n\n# Testing of elite configurations:", scenario$testNbElites, + + cat("\n\n# Testing of elite configurations:", scenario$testNbElites, "\n# Testing iteration configurations:", scenario$testIterationElites,"\n") if (scenario$testNbElites <= 0) return (FALSE) @@ -160,11 +158,11 @@ testing_fromlog <- function(logFile, testNbElites, testIterationElites, if (instances_changed || is.null.or.empty(scenario[["testInstances"]])) { scenario <- setup_test_instances(scenario) if (is.null.or.empty(scenario[["testInstances"]])) { - irace.note("No test instances, skip testing\n") + irace_note("No test instances, skip testing\n") return(FALSE) } } - + # Get configurations that will be tested if (scenario$testIterationElites) testing_id <- sapply(iraceResults$allElites, @@ -176,7 +174,7 @@ testing_fromlog <- function(logFile, testNbElites, testIterationElites, testing_id <- unique.default(unlist(testing_id)) configurations <- iraceResults$allConfigurations[testing_id, , drop=FALSE] - irace.note ("Testing configurations (in no particular order): ", paste(testing_id, collapse=" "), "\n") + irace_note ("Testing configurations (in no particular order): ", paste(testing_id, collapse=" "), "\n") testing_common(configurations, scenario, iraceResults) return(TRUE) } @@ -186,7 +184,7 @@ testing_fromlog <- function(logFile, testNbElites, testIterationElites, #' Executes the testing of an explicit list of configurations given in #' `filename` (same format as in [readConfigurationsFile()]). A `logFile` is #' created unless disabled in `scenario`. This may overwrite an existing one! -#' +#' #' @param filename `character(1)`\cr Path to a file containing configurations: one configuration #' per line, one parameter per column, parameter names in header. #' @@ -195,13 +193,13 @@ testing_fromlog <- function(logFile, testNbElites, testIterationElites, #' @return iraceResults #' #' @seealso [testing_fromlog()] provides a different interface for testing. -#' +#' #' @author Manuel López-Ibáñez #' @concept running #' @export testing_fromfile <- function(filename, scenario) { - irace.note ("Checking scenario.\n") + irace_note ("Checking scenario.\n") scenario <- checkScenario(scenario) if (!scenario$quiet) printScenario(scenario) @@ -211,15 +209,15 @@ testing_fromfile <- function(filename, scenario) num <- nrow(configurations) configurations <- checkForbidden(configurations, scenario$parameters$forbidden) if (nrow(configurations) < num) { - irace.warning("Some of the configurations in the configurations file were forbidden", + irace_warning("Some of the configurations in the configurations file were forbidden", "and, thus, discarded.") } # To save the logs iraceResults <- list(scenario = scenario, irace_version = irace_version, allConfigurations = configurations) - - irace.note ("Testing configurations (in the order given as input): \n") + + irace_note ("Testing configurations (in the order given as input): \n") testing_common(configurations, scenario, iraceResults) } @@ -228,11 +226,11 @@ testing_common <- function(configurations, scenario, iraceResults) verbose <- !scenario$quiet if (verbose) configurations_print(configurations) iraceResults$testing <- testConfigurations(configurations, scenario) - irace_save_logfile(iraceResults, scenario) - irace.note ("Testing results (column number is configuration ID in no particular order):\n") + save_irace_logfile(iraceResults, logfile = scenario$logFile) + irace_note ("Testing results (column number is configuration ID in no particular order):\n") if (verbose) print(cbind(seeds = iraceResults$testing$seeds, as.data.frame(iraceResults$testing$experiments))) - irace.note ("Finished testing\n") + irace_note ("Finished testing\n") iraceResults } @@ -240,13 +238,13 @@ testing_common <- function(configurations, scenario, iraceResults) #' #' Test that the given irace scenario can be run by checking the scenario #' settings provided and trying to run the target-algorithm. -#' +#' #' @inheritParams defaultScenario -#' +#' #' @return returns `TRUE` if successful and gives an error and returns `FALSE` #' otherwise. -#' -#' @details If the `parameters` argument is missing, then the parameters +#' +#' @details If the `parameters` argument is missing, then the parameters #' will be read from the file `parameterFile` given by `scenario`. If #' `parameters` is provided, then `parameterFile` will not be read. This function will #' try to execute the target-algorithm. @@ -262,23 +260,23 @@ testing_common <- function(configurations, scenario, iraceResults) #' @export checkIraceScenario <- function(scenario) { - irace.note ("Checking scenario\n") + irace_note ("Checking scenario\n") scenario$debugLevel <- 2L scenario <- checkScenario(scenario) if (!scenario$quiet) printScenario(scenario) - - irace.note("Checking target runner.\n") + + irace_note("Checking target runner.\n") if (checkTargetFiles(scenario = scenario)) { - irace.note("Check successful.\n") + irace_note("Check successful.\n") return(TRUE) } - irace.error("Check unsuccessful.\n") + irace_error("Check unsuccessful.\n") return(FALSE) } -init <- function() +init <- function() { - irace.note("Initializing working directory...\n") + irace_note("Initializing working directory...\n") libPath <- system.file(package = "irace") tmplFiles <- list.files(file.path(libPath, "templates")) for (file in tmplFiles) { @@ -298,19 +296,19 @@ init <- function() #' #' Calls [irace_main()] using command-line options, maybe parsed from the #' command line used to invoke R. -#' -#' @param argv (\code{character()}) \cr The arguments -#' provided on the R command line as a character vector, e.g., -#' \code{c("--scenario", "scenario.txt", "-p", "parameters.txt")}. -#' Using the default value (not providing the parameter) is the -#' easiest way to call \code{irace_cmdline}. -#' +#' +#' @param argv `character()`\cr The arguments +#' provided on the R command line as a character vector, e.g., +#' `c("--scenario", "scenario.txt", "-p", "parameters.txt")`. +#' Using the default value (not providing the parameter) is the +#' easiest way to call [irace_cmdline()]. +#' #' @details The function reads the parameters given on the command line #' used to invoke R, finds the name of the scenario file, #' initializes the scenario from the file (with the function -#' \code{\link{readScenario}}) and possibly from parameters passed in +#' [readScenario()]) and possibly from parameters passed in #' the command line. It finally starts \pkg{irace} by calling -#' \code{\link{irace_main}}. +#' [irace_main()]. #' #' List of command-line options: #' ```{r echo=FALSE,comment=NA} @@ -319,7 +317,7 @@ init <- function() #' #' @templateVar return_invisible TRUE #' @template return_irace -#' +#' #' @seealso #' [irace_main()] to start \pkg{irace} with a given scenario. #' @examples @@ -330,7 +328,7 @@ init <- function() irace_cmdline <- function(argv = commandArgs(trailingOnly = TRUE)) { parser <- CommandArgsParser$new(argv = argv, argsdef = .irace.params.def) - quiet <- !is.null(parser$readArg (short = "-q", long = "--quiet")) + quiet <- !is.null(parser$readArg (short = "-q", long = "--quiet")) if (quiet) { op <- options(.irace.quiet = TRUE) on.exit(options(op)) @@ -344,15 +342,15 @@ irace_cmdline <- function(argv = commandArgs(trailingOnly = TRUE)) return(invisible(NULL)) } if (!is.null(parser$readArg(short = "-v", long = "--version"))) { - print(citation(package="irace")) + print(utils::citation(package="irace")) return(invisible(NULL)) } - + if (!is.null(parser$readArg(short = "-i", long = "--init"))) { init() return(invisible(NULL)) } - + # Read the scenario file and the command line scenarioFile <- parser$readCmdLineParameter ("scenarioFile", default = "") scenario <- readScenario(scenarioFile) @@ -361,7 +359,7 @@ irace_cmdline <- function(argv = commandArgs(trailingOnly = TRUE)) parser$readCmdLineParameter(paramName = param, default = scenario[[param]]) } if (quiet) scenario$quiet <- TRUE - + # Check scenario if (!is.null(parser$readArg (short = "-c", long = "--check"))) { checkIraceScenario(scenario) @@ -375,8 +373,8 @@ irace_cmdline <- function(argv = commandArgs(trailingOnly = TRUE)) } if (length(parser$argv)) - irace.error ("Unknown command-line options: ", paste(parser$argv, collapse = " ")) - + irace_error ("Unknown command-line options: ", paste(parser$argv, collapse = " ")) + irace_common(scenario = scenario, simple=FALSE) } @@ -396,12 +394,12 @@ checkTargetFiles <- function(scenario) repair = scenario$repairConfiguration) set(configurations, j = ".ID.", value = seq_nrow(configurations)) setcolorder(configurations, ".ID.", before = 1L) - + # Read initial configurations provided by the user. initConfigurations <- allConfigurationsInit(scenario) setDT(initConfigurations) if (nrow(initConfigurations) > 0L) { - irace.assert(all(colnames(configurations) == colnames(initConfigurations))) + irace_assert(all(colnames(configurations) == colnames(initConfigurations))) configurations <- rbindlist(list(initConfigurations, configurations)) set(configurations, j = ".ID.", value = seq_nrow(configurations)) } @@ -417,11 +415,12 @@ checkTargetFiles <- function(scenario) race_state <- RaceState$new(scenario) race_state$start_parallel(scenario) on.exit(race_state$stop_parallel(), add = TRUE) - # FIXME: Create a function try.call(err.msg,warn.msg, fun, ...) # Executing targetRunner - cat("# Executing targetRunner (", nrow(configurations), "times)...\n") + irace_note("Executing targetRunner (", nrow(configurations), " times)...\n") result <- TRUE + # We cannot let targetRunner or targetEvaluator modify our random seed, so we save it. + withr::local_preserve_seed() output <- withCallingHandlers( tryCatch(execute_experiments(race_state, experiments, scenario), error = function(e) { @@ -437,18 +436,17 @@ checkTargetFiles <- function(scenario) invokeRestart("muffleWarning")}) if (scenario$debugLevel >= 1L) { - cat ("# targetRunner returned:\n") + cat("# targetRunner returned:\n") print(output, digits = 15L) } - - irace.assert(is.null(scenario$targetEvaluator) == is.null(race_state$target_evaluator)) + + irace_assert(is.null(scenario$targetEvaluator) == is.null(race_state$target_evaluator)) if (!result) return(FALSE) - + if (!is.null(scenario$targetEvaluator)) { - cat("# Executing targetEvaluator...\n") + irace_note("Executing targetEvaluator...\n") output <- withCallingHandlers( - tryCatch(execute_evaluator(race_state$target_evaluator, - experiments, scenario, output, configurations[[".ID."]]), + tryCatch(execute_evaluator(race_state$target_evaluator, experiments, scenario, output), error = function(e) { cat(sep = "\n", "\n# Error ocurred while executing targetEvaluator:", @@ -461,7 +459,7 @@ checkTargetFiles <- function(scenario) paste0(conditionMessage(w), collapse="\n")) invokeRestart("muffleWarning")}) if (scenario$debugLevel >= 1L) { - cat ("# targetEvaluator returned:\n") + cat("# targetEvaluator returned:\n") print(output, digits = 15L) } } diff --git a/R/model.R b/R/model.R index 48d7b97c..e229d1bf 100644 --- a/R/model.R +++ b/R/model.R @@ -35,8 +35,8 @@ initialiseModel <- function (parameters, configurations) type <- param[["type"]] if (type == "c") { nbValues <- length(param[["domain"]]) - value <- rep(1. / nbValues, nbValues) - param <- rep(list(value), nbConfigurations) + value <- rep_len(1. / nbValues, nbValues) + param <- rep_len(list(value), nbConfigurations) } else { if (type == "r" || type == "i") { sd <- init_sd_numeric(param) @@ -46,7 +46,7 @@ initialiseModel <- function (parameters, configurations) sd <- (length(domain) - 1L) * 0.5 values <- match(configurations[[currentParameter]], domain) } else { - irace.internal.error("Unknown parameter type '", type, "'") + irace_internal_error("Unknown parameter type '", type, "'") } # Assign current parameter value to model. param <- mapply(c, sd, values, SIMPLIFY=FALSE, USE.NAMES=FALSE) @@ -98,7 +98,7 @@ updateModel <- function(parameters, eliteConfigurations, oldModel, for (currentParameter in param_names) { param <- parameters$get(currentParameter) - irace.assert(all(ids_in_model %in% names(oldModel[[currentParameter]]))) + irace_assert(all(ids_in_model %in% names(oldModel[[currentParameter]]))) this_model <- oldModel[[currentParameter]][ids_in_model] values <- eliteConfigurations[[currentParameter]] values_not_na <- !is.na(values) @@ -109,7 +109,7 @@ updateModel <- function(parameters, eliteConfigurations, oldModel, values <- match(values, param[["domain"]]) this_model[values_not_na] <- mapply(update_prob, this_model[values_not_na], values, SIMPLIFY=FALSE) } else { - irace.assert(type %in% c("i", "r", "o")) + irace_assert(type %in% c("i", "r", "o")) if (type == "o") values <- match(values, param[["domain"]]) this_model[values_not_na] <- mapply(function(p, value) c(p[[1L]] * num_factor, value), @@ -143,7 +143,7 @@ restartModel <- function(model, configurations, restart_ids, parameters, for (pname in parameters$names_variable) { model_param <- model[[pname]] - irace.assert (all(restart_ids %in% names(model_param)), { + irace_assert (all(restart_ids %in% names(model_param)), { cat("Param:", pname, "\n") print(restart_ids) print(model) @@ -161,7 +161,7 @@ restartModel <- function(model, configurations, restart_ids, parameters, if (type == "i" || type == "r") { value <- init_sd_numeric(param) } else { - irace.assert(type == "o") + irace_assert(type == "o") value <- (length(param[["domain"]]) - 1L) * 0.5 } # Bring back the value 2 iterations or to the second iteration value. diff --git a/R/multi_irace.R b/R/multi_irace.R index b79a5e97..4ba1942a 100644 --- a/R/multi_irace.R +++ b/R/multi_irace.R @@ -8,16 +8,16 @@ #' } #' Each of the `k` runs can be repeated `n` times by supplying a value for `n`. #' -#' @param scenarios (`list()`) \cr A list of scenarios. +#' @param scenarios `list()`\cr A list of scenarios. #' If only a single scenario is supplied, it is used for all parameters. -#' @param parameters (`list()`) \cr A list of parameter space definitions. +#' @param parameters `list()`\cr A list of parameter space definitions. #' If only a single definition is supplied, it is used for all scenarios. -#' @param n (`integer(1)`) \cr The number of repetitions. -#' @param parallel (`integer(1)`) \cr The number of workers to use. +#' @param n `integer(1)`\cr The number of repetitions. +#' @param parallel `integer(1)`\cr The number of workers to use. #' A value of `1` means sequential execution. Note that `parallel > 1` is not supported on Windows. -#' @param split_output (`logical(1)`) \cr If `TRUE`, the output of [irace()] is written to `{execDir}/run_{i}/irace.out` +#' @param split_output `logical(1)`\cr If `TRUE`, the output of [irace()] is written to `{execDir}/run_{i}/irace.out` #' instead of the standard output. -#' @param global_seed (`integer(1)`) \cr The global seed used to seed the individual runs. +#' @param global_seed `integer(1)`\cr The global seed used to seed the individual runs. #' #' @return A list of the outputs of [irace()]. #' @@ -28,11 +28,11 @@ #' #' @concept running #' @export -multi_irace <- function(scenarios, parameters, n = 1L, parallel = 1, split_output = parallel > 1, global_seed = NULL) +multi_irace <- function(scenarios, parameters, n = 1L, parallel = 1L, split_output = parallel > 1L, global_seed = NULL) { # Parallel execution is not available on Windows. - if (.Platform$OS.type == 'windows') { - irace.assert(parallel == 1L) + if (.Platform$OS.type == 'windows' && parallel > 1L) { + irace_error("multi_irace() does not yet support parallel > 1 on Windows") } # Allow either the same number of scenarios and parameters, or a single scenario or parameter space definition. @@ -42,7 +42,7 @@ multi_irace <- function(scenarios, parameters, n = 1L, parallel = 1, split_outpu } else if (length(parameters) == 1L) { parameters <- rep(parameters, each = length(scenarios)) } else { - irace.error("Invalid arguments: ", + irace_error("Invalid arguments: ", "Cannot execute 'irace' with", length(scenarios), "scenarios and", length(parameters), "parameters.", "Either supply the same number of scenarios and parameters, @@ -76,7 +76,7 @@ multi_irace <- function(scenarios, parameters, n = 1L, parallel = 1, split_outpu scenarios[[i]]$execDir <- execDir fs::dir_create(execDir) - if (nzchar(logFile_old)) { + if (logFile_old != "") { logFile <- if (is.sub.path(logFile_old, execDir_old)) { # 'logFile' is located in the old 'execDir', so move it into the new 'execDir'. # 'path/to/execDir/logFile.rdata' -> 'path/to/execDir/run_{i}/logFile.rdata'. @@ -88,7 +88,7 @@ multi_irace <- function(scenarios, parameters, n = 1L, parallel = 1, split_outpu # pathExt <- tools::file_ext(logFile_old) # pathWithoutExtWithIndex <- sprintf("%s_%02d", pathWithoutExt, i) # paste(pathWithoutExtWithIndex, pathExt, sep = ".") - irace.error("Invalid 'logFile' path (", logFile_old, "): ", + irace_error("Invalid 'logFile' path (", logFile_old, "): ", "The 'logFile' must be located inside the 'execDir' (", execDir_old, ").") } scenarios[[i]]$logFile <- logFile @@ -118,7 +118,7 @@ multi_irace <- function(scenarios, parameters, n = 1L, parallel = 1, split_outpu # each configuration and repetitions may occur. errors <- unique(unlist(runs[sapply(runs, inherits, "try-error")])) cat(errors, file = stderr()) - irace.error("A child process triggered a fatal error") + irace_error("A child process triggered a fatal error") } } else { runs <- mapply(irace_run, scenarios, SIMPLIFY = FALSE) diff --git a/R/parameterAnalysis.R b/R/parameterAnalysis.R index b79545e3..346b15eb 100644 --- a/R/parameterAnalysis.R +++ b/R/parameterAnalysis.R @@ -1,18 +1,18 @@ #' Return the elite configurations of the final iteration. -#' +#' #' @inheritParams has_testing_data -#' @param n Number of elite configurations to return, if \code{n} is larger than the +#' @param n `integer(1)`\cr Number of elite configurations to return, if \code{n} is larger than the #' number of configurations, then only the existing ones are returned. The default (\code{n=0}) returns all of them. #' @param drop.metadata `logical(1)`\cr Remove metadata, such as the #' configuration ID and the ID of the parent, from the returned #' configurations. See [removeConfigurationsMetaData()]. -#' +#' #' @return A data frame containing the elite configurations required. #' #' @examples #' log_file <- system.file("exdata/irace-acotsp.Rdata", package="irace", mustWork=TRUE) #' print(removeConfigurationsMetaData(getFinalElites(log_file, n=1))) -#' +#' #' @author Manuel López-Ibáñez and Leslie Pérez Cáceres #' @concept analysis #' @export @@ -20,20 +20,20 @@ getFinalElites <- function(iraceResults, n = 0L, drop.metadata = FALSE) { if (missing(iraceResults)) stop("argument 'iraceResults' is missing") iraceResults <- read_logfile(iraceResults) - - last.elites <- iraceResults$allElites[[length(iraceResults$allElites)]] - + + last_elites <- iraceResults$allElites[[length(iraceResults$allElites)]] + if (n == 0L) - n <- length(last.elites) - - if (length(last.elites) < n) { - cat("Only", length(last.elites), "configurations available, reducing n,") - n <- length(last.elites) + n <- length(last_elites) + + if (length(last_elites) < n) { + cat("Only", length(last_elites), "configurations available, reducing n,") + n <- length(last_elites) } - last.elites <- last.elites[seq_len(n)] - + last_elites <- last_elites[seq_len(n)] + configurations <- subset(iraceResults$allConfigurations, - get(".ID.") %in% as.character(last.elites), + get(".ID.") %in% as.character(last_elites), drop = FALSE) if (drop.metadata) configurations <- removeConfigurationsMetaData(configurations) @@ -41,14 +41,15 @@ getFinalElites <- function(iraceResults, n = 0L, drop.metadata = FALSE) } #' Returns the configurations selected by ID. -#' -#' @param ids (`integer()`)\cr The id or a vector of ids of the candidates configurations to obtain. +#' +#' @param ids `integer()`\cr The id or a vector of ids of the candidates configurations to obtain. #' @inheritParams getFinalElites -#' -#' @return A data frame containing the elite configurations required. +#' +#' @return A data frame containing the elite configurations required, in the +#' order and with the repetitions given by `ids`. #' @examples #' log_file <- system.file("exdata/irace-acotsp.Rdata", package="irace", mustWork=TRUE) -#' getConfigurationById(log_file, ids = c(1,2), drop.metadata = TRUE) +#' getConfigurationById(log_file, ids = c(2,1), drop.metadata = TRUE) #' #' @author Manuel López-Ibáñez and Leslie Pérez Cáceres #' @concept analysis @@ -57,7 +58,7 @@ getConfigurationById <- function(iraceResults, ids, drop.metadata = FALSE) { if (missing(iraceResults)) stop("argument 'iraceResults' is missing") iraceResults <- read_logfile(iraceResults) - + if (length(ids) < 1L) stop("You must provide at least one configuration id.") get_configuration_by_id_helper(iraceResults$allConfigurations, ids, drop_metadata = drop.metadata) @@ -65,16 +66,16 @@ getConfigurationById <- function(iraceResults, ids, drop.metadata = FALSE) #' Returns the configurations by the iteration in which they were executed. #' -#' @param iterations (`integer()`)\cr The iteration number or a vector of iteration numbers from where +#' @param iterations `integer()`\cr The iteration number or a vector of iteration numbers from where #' the configurations should be obtained. Negative values start counting from the last iteration. #' @inheritParams getFinalElites -#' +#' #' @return A data frame containing the elite configurations required. #' #' @examples #' log_file <- system.file("exdata/irace-acotsp.Rdata", package="irace", mustWork=TRUE) #' getConfigurationByIteration(log_file, iterations = c(-2, -1), drop.metadata = TRUE) -#' +#' #' @author Manuel López-Ibáñez and Leslie Pérez Cáceres #' @concept analysis #' @export @@ -94,7 +95,7 @@ getConfigurationByIteration <- function(iraceResults, iterations, drop.metadata if (is.null(iraceResults$state$experiment_log) || nrow(iraceResults$state$experiment_log) == 0L) stop("'iraceResults' does not contain experiment_log, maybe the wrong file, an incomplete run or the wrong version of irace?") - + ids <- unique(subset(as.data.frame(iraceResults$state$experiment_log), iteration %in% iterations, select="configuration", drop=TRUE)) @@ -103,11 +104,11 @@ getConfigurationByIteration <- function(iraceResults, iterations, drop.metadata get_configuration_by_id_helper <- function(allConfigurations, ids, drop_metadata) -{ - configurations <- allConfigurations[allConfigurations[[".ID."]] %in% ids, , drop=FALSE] +{ + configurations <- allConfigurations[match(ids, allConfigurations[[".ID."]]), , drop=FALSE] if (nrow(configurations) == 0L) stop("No configuration found with ID:", ids, ".") - + if (drop_metadata) configurations <- removeConfigurationsMetaData(configurations) configurations @@ -117,9 +118,9 @@ get_configuration_by_id_helper <- function(allConfigurations, ids, drop_metadata #' (and optionally the actual instances). #' #' @inheritParams getFinalElites -#' @param index (`integer()`)\cr Indexes of the (instanceID,seed) pairs to be returned. The default returns everything. -#' @param instances (`logical(1)`)\cr Whether to add the actual instances as an additional column (only if the instances are of atomic type). -#' +#' @param index `integer()`\cr Indexes of the (instanceID,seed) pairs to be returned. The default returns everything. +#' @param instances `logical(1)`\cr Whether to add the actual instances as an additional column (only if the instances are of atomic type). +#' #' @return `data.table()`\cr With default arguments, a `data.table` containing two columns #' `"instanceID"` and `"seed"`. With `instances=TRUE` and if the instances #' are of atomic type (see [is.atomic()]) type, another column `instance` is @@ -148,7 +149,7 @@ get_instanceID_seed_pairs <- function(iraceResults, index, instances = FALSE) warning("instances=TRUE requested, but instances are not of atomic type") return(instances_log) } - + instanceID <- instances_log[["instanceID"]] cbind(instances_log, instance = instances[instanceID]) } diff --git a/R/parameters.R b/R/parameters.R index 911bb7ae..e30ad967 100644 --- a/R/parameters.R +++ b/R/parameters.R @@ -10,7 +10,7 @@ check_domain_dependencies <- function (params, depends, types) vars <- depends[[p]] flag <- vars %in% allnames if (!all(flag)) { - irace.error ("Domain (", paste0(domain, collapse=", "), + irace_error ("Domain (", paste0(domain, collapse=", "), ") of parameter '", p, "' is not valid: '", paste0(vars[!flag], collapse=", "), "' cannot be found in the scenario parameters: ", @@ -18,7 +18,7 @@ check_domain_dependencies <- function (params, depends, types) } flag <- types[vars] %in% c("i", "r") if (!all(flag)) { - irace.error ("Domain of parameter '", p, "' depends on non-numerical", + irace_error ("Domain of parameter '", p, "' depends on non-numerical", " parameters: ", paste0(vars[!flag], collapse=", "), " .") } @@ -26,7 +26,7 @@ check_domain_dependencies <- function (params, depends, types) fx <- setdiff(all.names(domain, unique=TRUE), all.vars(domain, unique=TRUE)) flag <- fx %in% allowed_fx if (!all(flag)) { - irace.error ("Domain of parameter '", p, "' uses function(s) ", + irace_error ("Domain of parameter '", p, "' uses function(s) ", "not yet supported by irace: ", paste0(fx[!flag], collapse=", "), " .") } @@ -40,7 +40,7 @@ check_forbidden_params <- function(x, pnames, filename = NULL) if (any(unique(unlist(lapply(x, all.names))) %in% c("&&", "||"))) { for (ex in x) { if (any(all.names(ex) %in% c("&&", "||"))) - irace.error("Please use '&' and '|' instead of '&&' and '|' in: ", deparse(ex), " .\n") + irace_error("Please use '&' and '|' instead of '&&' and '|' in: ", deparse(ex), " .\n") } } if (all(all.vars(x) %in% pnames)) return(invisible()) @@ -49,11 +49,11 @@ check_forbidden_params <- function(x, pnames, filename = NULL) if (length(v)) { v <- paste0(v, collapse=", ") if (is.null(filename)) { - irace.error("Forbidden expression '", deparse(ex), "' contains unknown parameter(s): ", v) + irace_error("Forbidden expression '", deparse(ex), "' contains unknown parameter(s): ", v) } else if (is.na(filename)) { - irace.error("Expression '", deparse(ex), "' after [forbidden] contains unknown parameter(s): ", v) + irace_error("Expression '", deparse(ex), "' after [forbidden] contains unknown parameter(s): ", v) } else { - irace.error("Expression '", deparse(ex), "' after [forbidden] in '", filename, "' contains unknown parameter(s): ", v) + irace_error("Expression '", deparse(ex), "' after [forbidden] in '", filename, "' contains unknown parameter(s): ", v) } } } @@ -81,17 +81,17 @@ param_level <- function(paramName, varsTree, rootParam = paramName) for (child in vars) { # The following line detects cycles if (child == rootParam) - irace.error("Cycle detected in subordinate parameters! ", + irace_error("Cycle detected in subordinate parameters! ", "Check definition of conditions and/or dependent domains.\n", "One parameter of this cycle is '", rootParam, "'") - + # The following line detects a missing definition if (child %not_in% names(varsTree)) - irace.error("A parameter definition is missing! ", + irace_error("A parameter definition is missing! ", "Check definition of parameters.\n", "Parameter '", paramName, "' depends on '", child, "' which is not defined.") - + level <- param_level(child, varsTree, rootParam) if (level > maxChildLevel) maxChildLevel <- level @@ -115,13 +115,13 @@ transform_domain <- function(transf, domain, type) # Reject log if domain contains zero or negative values if (any(domain <= 0)) stop("Domain (", lower, ", ", upper, ") of parameter of type 'log' contains non-positive values") - + trLower <- log(lower) # +1 to adjust before floor() trUpper <- if (type == "i") log(upper + 1) else log(upper) - - irace.assert(is.finite(trLower)) - irace.assert(is.finite(trUpper)) + + irace_assert(is.finite(trLower)) + irace_assert(is.finite(trUpper)) attr(transf, "lower") <- trLower attr(transf, "upper") <- trUpper return(transf) @@ -175,7 +175,7 @@ Parameter <- function(name, type, domain, label, condition, transf, if (!isTRUE(condition)) { condition <- param_parse_condition(condition) } - + if (type %in% c("i", "r")) { exp_domain <- parse_exp_domain(domain) domain <- sapply(exp_domain, USE.NAMES=FALSE, function(x) if (is.numeric(x)) x else NA) @@ -184,7 +184,7 @@ Parameter <- function(name, type, domain, label, condition, transf, digits <- as.integer(digits) if (digits < 15L && (!valid_real_bound(domain[1L], digits) - || !valid_real_bound(domain[2L], digits))) { + || !valid_real_bound(domain[2L], digits))) { for (i in seq.int(digits+1L, 15L)) { if (valid_real_bound(domain[1L], i) && valid_real_bound(domain[2L], i)) break @@ -198,14 +198,14 @@ Parameter <- function(name, type, domain, label, condition, transf, } else { # type == "i" if (any(!is.wholenumber(domain[!is.na(domain)]))) named_stop("invalid_domain", "for parameter '", name, "' of type 'integer', values must be integers") - if (anyNA(domain)) + if (anyNA(domain)) domain <- as.expression(sapply(exp_domain, USE.NAMES=FALSE, function(x) if (is.numeric(x)) as.integer(x) else x)) else domain <- as.integer(domain) } if (!is.expression(domain) && domain[1L] >= domain[2L]) named_stop("invalid_range", "lower bound must be smaller than upper bound in numeric range") - + } else { # type %in% c("c", "o") if (anyDuplicated(domain)) { dups <- duplicated(domain) @@ -218,19 +218,16 @@ Parameter <- function(name, type, domain, label, condition, transf, paste0(domain, collapse = ', ')) } } - + if (transf != "") transf <- transform_domain(transf, domain, type) isFixed <- (type == "c" || type == "o") && (length(domain) == 1L) + param <- list(name = name, type = type, domain = domain, label = label, is_dependent = is.expression(domain), + isFixed = isFixed, transform = transf, condition = condition) + if (type == "r") + param$digits <- digits - if (type == "r") { - param <- list(name = name, type = type, domain = domain, label = label, is_dependent = is.expression(domain), - isFixed = isFixed, transform = transf, condition = condition, digits = digits) - } else { - param <- list(name = name, type = type, domain = domain, label = label, is_dependent = is.expression(domain), - isFixed = isFixed, transform = transf, condition = condition) - } class(param) <- c(cls, "Parameter", class(param)) param } @@ -242,12 +239,12 @@ Parameter <- function(name, type, domain, label, condition, transf, #' - `param_ord()` creates an ordinal parameter. #' - `param_real()` creates a real-valued parameter. #' - `param_int()` creates an integer parameter. -#' +#' #' @param ... one or more parameters created by `param_int()`, `param_real()`, `param_ord()`, or `param_cat()`. -#' @param forbidden (`expression()|character(1)`)\cr String or list of expressions that define forbidden parameter configurations. +#' @param forbidden `expression()|character(1)`\cr String or list of expressions that define forbidden parameter configurations. #' @inheritParams readParameters #' -#' @return (`ParameterSpace`) +#' @return `ParameterSpace` #' @name parameters #' @examples #' digits <- 4L @@ -307,7 +304,7 @@ ParameterSpace <- R6::R6Class("ParameterSpace", cloneable = TRUE, lock_class = T self$nbParameters <- length(.params) self$nbFixed <- sum(self$isFixed) self$nbVariable <- sum(!self$isFixed) - + # Obtain the variables in each condition self$depends <- lapply(self$domains, all.vars) # Check that domain dependencies are OK. @@ -337,6 +334,16 @@ ParameterSpace <- R6::R6Class("ParameterSpace", cloneable = TRUE, lock_class = T self$forbidden <- sapply(forbidden, compile_forbidden) } + # Optimize always TRUE conditions. + cond_names <- names(which(!unlist(lapply(self$conditions, is.logical)))) + for (p in cond_names) { + deps <- self$depends[[p]] + # p depends on deps and they are both fixed and always active. + if (all(self$isFixed[deps]) && all(sapply(self$conditions[deps], isTRUE, USE.NAMES=FALSE))) { + self$conditions[[p]] <- eval(self$conditions[[p]], envir = self$domains[deps], enclos = NULL) + } + } + # Print the hierarchy vector: if (verbose >= 1L) { cat ("# --- Parameters Hierarchy ---\n") @@ -353,7 +360,7 @@ ParameterSpace <- R6::R6Class("ParameterSpace", cloneable = TRUE, lock_class = T forbid_configurations = function(x) { # FIXME: This copies the whole list. Avoid the copy and just append. - self$forbidden <- c(self$forbidden, + self$forbidden <- c(self$forbidden, buildForbiddenExp(configurations = x[, self$names, drop=FALSE])) }, as_character = function() { @@ -379,7 +386,8 @@ ParameterSpace <- R6::R6Class("ParameterSpace", cloneable = TRUE, lock_class = T condition <- p[["condition"]] condition <- if (isTRUE(condition)) "" else paste0(" | ", condition) transf <- p[["transform"]] - if (!is.null(transf) && transf != "") type <- paste0(type, ",", transf) + if (!is.null(transf) && transf != "") + type <- paste0(type, ",", transf) label <- paste0('"', p[["label"]], '"') output <- paste0(output, sprintf('%*s %*s %s %-15s%s\n', -names_len, p[["name"]], -label_len, label, type, domain, condition)) if (type == "r") @@ -400,29 +408,28 @@ ParameterSpace <- R6::R6Class("ParameterSpace", cloneable = TRUE, lock_class = T }), ) -#' @param name Parameter name (must be alphanumeric). -#' @param values (`character()`) \cr Domain as a vector of strings. -#' @param label Label associated to the parameter. Often used to encode a command-line switch that activates the parameter. -#' @param condition (`expression(1)|character(1)`) \cr Expression that defines when the parameter is active according to the value of other parameters. +#' @param name `character(1)`\cr Parameter name (must be alphanumeric). +#' @param values `character()`\cr Domain as a vector of strings. +#' @param label `character(1)`\cr Label associated to the parameter. Often used to encode a command-line switch that activates the parameter. +#' @param condition `expression(1)|character(1)`\cr Expression that defines when the parameter is active according to the value of other parameters. #' @rdname parameters #' @export param_cat <- function(name = name, values, label = "", condition = TRUE) - Parameter(name = name, type = "c", domain = values, label = label, condition = condition, transf = "") + Parameter(name = name, type = "c", domain = as.character(values), label = label, condition = condition, transf = "") #' @rdname parameters #' @export param_ord <- function(name, values, label = "", condition = TRUE) - Parameter(name = name, type = "o", domain = values, label = label, condition = condition, transf = "") + Parameter(name = name, type = "o", domain = as.character(values), label = label, condition = condition, transf = "") #' @param lower,upper Lower and upper limits of the valid domain. -#' @param transf (`character(1)`) \cr If `"log"`, then the parameter is sampled in a logarithmic scale. +#' @param transf `character(1)`\cr If `"log"`, then the parameter is sampled in a logarithmic scale. #' @inheritParams readParameters #' @rdname parameters #' @export param_real <- function(name, lower, upper, label = "", condition = TRUE, transf = "", digits = 15L) Parameter(name = name, type = "r", domain = c(lower, upper), label = label, condition = condition, transf = transf, digits = digits) -#' @param transf (`character(1)`) \cr If `"log"`, then the parameter is sampled in a logarithmic scale. #' @rdname parameters #' @export param_int <- function(name, lower, upper, label = "", condition = TRUE, transf = "") @@ -431,7 +438,7 @@ param_int <- function(name, lower, upper, label = "", condition = TRUE, transf = transform_from_log <- function(x, transf, lower, upper) { - trLower <- attr(transf, "lower") + trLower <- attr(transf, "lower") trUpper <- attr(transf, "upper") x <- exp(trLower + (trUpper - trLower) * x) clamp(x, lower, upper) @@ -439,7 +446,7 @@ transform_from_log <- function(x, transf, lower, upper) transform_to_log <- function(x, transf) { - trLower <- attr(transf, "lower") + trLower <- attr(transf, "lower") trUpper <- attr(transf, "upper") (log(x) - trLower)/(trUpper - trLower) } @@ -482,7 +489,7 @@ transform_to_log <- function(x, transf) # table(floor(rtnorm(nsamples, mean=1.5, sd=1, lower=1,upper=4)))/nsamples # table(floor(rtnorm(nsamples, mean=3.5, sd=1, lower=1,upper=4)))/nsamples # -# The above reasoning also works for log-transformed domains, because +# The above reasoning also works for log-transformed domains, because # floor() happens in the original domain, not in the log-transformed one, # except for the case of log-transformed negative domains, where we have to # translate by -0.5. @@ -492,7 +499,7 @@ integer_round <- function(x, lower, upper) x <- floor(x) # The probability of this happening is very small, but it happens. x <- pmin.int(upper, x) - irace.assert(all(x >= lower)) + irace_assert(all(x >= lower)) as.integer(x) } @@ -556,7 +563,7 @@ param_quantile.ParamOrd <- param_quantile.ParamCat sample_model.ParamInt <- function(param, n, model, domain = param[["domain"]]) { # Dependent domains could be not available because of inactivity of parameters - # on which they are depedent. In this case, the dependent parameter becomes + # on which they are depedent. In this case, the dependent parameter becomes # not active and we return NA. if (anyNA(domain)) return(NA_integer_) @@ -580,7 +587,7 @@ sample_model.ParamInt <- function(param, n, model, domain = param[["domain"]]) sample_unif.ParamInt <- function(param, n, domain = param[["domain"]]) { # Dependent domains could be not available because of inactivity of parameters - # on which they are depedent. In this case, the dependent parameter becomes + # on which they are depedent. In this case, the dependent parameter becomes # not active and we return NA. if (anyNA(domain)) return(NA_integer_) @@ -595,6 +602,11 @@ sample_unif.ParamInt <- function(param, n, domain = param[["domain"]]) #' @exportS3Method param_quantile.ParamInt <- function(param, probs, domain = param[["domain"]]) { + # Dependent domains could be not available because of inactivity of parameters + # on which they are depedent. In this case, the dependent parameter becomes + # not active and we return NA. + if (anyNA(domain)) + return(NA_integer_) lower <- domain[1L] upper <- domain[2L] probs <- as.numeric(probs) @@ -611,9 +623,10 @@ param_quantile.ParamInt <- function(param, probs, domain = param[["domain"]]) sample_model.ParamReal <- function(param, n, model, domain = param[["domain"]]) { # Dependent domains could be not available because of inactivity of parameters - # on which they are depedent. In this case, the dependent parameter becomes + # on which they are depedent. In this case, the dependent parameter becomes # not active and we return NA. - if (anyNA(domain)) return(NA) + if (anyNA(domain)) + return(NA_real_) lower <- domain[[1L]] upper <- domain[[2L]] transf <- param[["transform"]] @@ -623,7 +636,7 @@ sample_model.ParamReal <- function(param, n, model, domain = param[["domain"]]) else x <- sample_numeric_norm(n, mean, sd = model[[1L]], lower, upper, transf) x <- round(x, digits = param[["digits"]]) - irace.assert(all(x >= lower) && all(x <= upper)) + irace_assert(all(x >= lower) && all(x <= upper)) x } @@ -631,20 +644,26 @@ sample_model.ParamReal <- function(param, n, model, domain = param[["domain"]]) sample_unif.ParamReal <- function(param, n, domain = param[["domain"]]) { # Dependent domains could be not available because of inactivity of parameters - # on which they are depedent. In this case, the dependent parameter becomes + # on which they are depedent. In this case, the dependent parameter becomes # not active and we return NA. - if (anyNA(domain)) return(NA) + if (anyNA(domain)) + return(NA_real_) lower <- domain[[1L]] upper <- domain[[2L]] x <- sample_numeric_unif(n, lower, upper, transf = param[["transform"]]) x <- round(x, digits = param[["digits"]]) - irace.assert(all(x >= lower) && all(x <= upper)) + irace_assert(all(x >= lower) && all(x <= upper)) x } #' @exportS3Method param_quantile.ParamReal <- function(param, probs, domain = param[["domain"]]) { + # Dependent domains could be not available because of inactivity of parameters + # on which they are depedent. In this case, the dependent parameter becomes + # not active and we return NA. + if (anyNA(domain)) + return(NA_real_) lower <- domain[1L] upper <- domain[2L] probs <- as.numeric(probs) @@ -657,14 +676,14 @@ param_quantile.ParamReal <- function(param, probs, domain = param[["domain"]]) } #' Print parameter space in the textual format accepted by irace. -#' -#' @param parameters (`list()`) \cr Data structure containing the parameter +#' +#' @param parameters `ParameterSpace`\cr Data structure containing the parameter #' space definition. The data structure has to similar to the one returned by the #' function [`readParameters`]. #' #' #' @return `character()` -#' +#' #' @seealso [readParameters()] #' @examples #' parameters_table <- ' @@ -681,7 +700,7 @@ param_quantile.ParamReal <- function(param, probs, domain = param[["domain"]]) #' elitistants "--elitistants " i (1, ants) | algorithm == "eas" #' nnls "--nnls " i (5, 50) | localsearch %in% c(1,2,3) #' dlb "--dlb " c (0, 1) | localsearch %in% c(1,2,3) -#' +#' #' [forbidden] #' (alpha == 0.0) & (beta == 0.0) #' ' @@ -694,7 +713,7 @@ printParameters <- function(parameters) #' @export print.ParameterSpace <- function(x, digits = 15L, ...) { - str(x$.params, digits.d = digits) + utils::str(x$.params, digits.d = digits) cat("Forbidden:\n") print(x$forbidden, digits = 15L) } diff --git a/R/path_rel2abs.R b/R/path_rel2abs.R index 1b362cfc..37ac9a98 100644 --- a/R/path_rel2abs.R +++ b/R/path_rel2abs.R @@ -4,12 +4,12 @@ #' using [Sys.which()]. Expansion of `'~'` in Windows follows the definition #' of [fs::path_expand()] rather than [base::path.expand()]. This function #' tries really hard to create canonical paths. -#' +#' #' @param path (`character(1)`) Character string representing a relative path. #' @param cwd (`character(1)`) Current working directory. #' #' @return (`character(1)`) Character string representing the absolute path -#' +#' #' @examples #' path_rel2abs("..") #' @export @@ -19,7 +19,7 @@ path_rel2abs <- function (path, cwd = getwd()) # FIXME: split this function into two, one for executable files than handles # finding executables in the PATH with Sys.which() and another for everything # else that doesn't try to be that smart. - + # We need to do this even with the path returned by `getwd()`. cwd <- fs::path_norm(fs::path_expand(cwd)) @@ -29,8 +29,7 @@ path_rel2abs <- function (path, cwd = getwd()) } else if (!startsWith(path, ".")) { # it may be a command in the path. sys_path <- suppressWarnings(Sys.which(path)) - if (nzchar(sys_path) - && fs::file_access(sys_path, "execute") + if (sys_path != "" && fs::file_access(sys_path, "execute") # fs::is_dir() is broken: https://github.com/r-lib/fs/issues/440 && !file.info(sys_path)$isdir) { path <- as.vector(sys_path) diff --git a/R/psRace.R b/R/psRace.R index 8425c064..2c2563bf 100644 --- a/R/psRace.R +++ b/R/psRace.R @@ -1,44 +1,54 @@ -#' Postselection race +#' Post-selection race #' -#' \code{psRace} performs a postselection race of a set of configurations. +#' \code{psRace} performs a post-selection race of a set of configurations. #' #' @inheritParams has_testing_data -#' +#' #' @param max_experiments `numeric(1)`\cr Number of experiments for the #' post-selection race. If it is equal to or smaller than 1, then it is a #' fraction of the total budget given by #' `iraceResults$scenario$maxExperiments` or `iraceResults$scenario$maxTime / #' iraceResults$state$boundEstimate`. -#' @param conf_ids IDs of the configurations in iraceResults$allConfigurations to be used for ablation. -#' If NULL, the `iteration_best` argument will be to decide. -#' @param iteration_elites If TRUE, only select the best configuration of each iteration. -#' If FALSE, select from all elite configurations of all iterations. `max_experiments` +#' @param conf_ids IDs of the configurations in `iraceResults$allConfigurations` to be used for the post-selection. +#' If `NULL`, then the configurations are automatically selected. +#' @param iteration_elites If `FALSE`, give priority to selecting the configurations that were elite in the last iteration. +#' If `TRUE`, select from all elite configurations of all iterations. This parameter only has an effect when `conf_ids` is not `NULL`. +#' @param psrace_logFile `character(1)`\cr Log file to save the post-selection race log. If `NULL`, the log is saved in `iraceResults$scenario$logFile`. #' -#' @return If iraceLogFile is NULL, it returns a list with the following elements: +#' @return The elite configurations after the post-selection. In addition, if `iraceResults$scenario$logFile` is defined, +#' it saves an updated copy of `iraceResults` in that file, where `iraceResults$psrace_log` is a list with the following elements: #' \describe{ -#' \item{configurations}{Configurations used in the race.} -#' \item{instances}{A matrix with the instances used in the experiments. First column has the -#' instances ids from iraceResults$scenario$instances, second column the seed assigned to the instance.} -#' \item{maxExperiments}{Maximum number of experiments set for the race.} -#' \item{experiments}{A matrix with the results of the experiments (columns are configurations, rows are instances).} +#' \item{configurations}{Configurations used in the post-selection race.} +#' \item{instances}{Data frame with the instances used in the post-selection race. First column has the +#' instances IDs from `iraceResults$scenario$instances`, second column the seed assigned to the instance.} +#' \item{max_experiments}{Configuration budget assigned to the post-selection race.} +#' \item{experiments}{Matrix of results generated by the post-selection race, in the same format as the matrix \code{iraceResults$experiments}. Column names are the configuration IDs and row names are the instance IDs.} #' \item{elites}{Best configurations found in the experiments.} #' } -#' If `iraceLogFile` is provided this list object will be saved in `iraceResults$psrace_log`. #' #' @examples #' \donttest{ -#' logfile <- system.file(package="irace", "exdata", "sann.rda") -#' # Execute the postselection after the execution of irace. Use 10% of the total budget. -#' psRace(logfile, max_experiments=0.1) +#' irace_log <- read_logfile(system.file(package="irace", "exdata", "sann.rda")) +#' # Use a temporary file to not change the original "sann.rda". +#' psrace_logFile <- withr::local_tempfile(fileext = ".Rdata") +#' # Execute the post-selection after the execution of irace. Use 10% of the total budget. +#' psRace(irace_log, max_experiments=0.1, psrace_logFile = psrace_logFile) +#' # Print psrace_log +#' irace_log <- read_logfile(psrace_logFile) +#' str(irace_log$psrace_log) #' } #' @author Leslie Pérez Cáceres and Manuel López-Ibáñez #' @export -psRace <- function(iraceResults, max_experiments, conf_ids = NULL, iteration_elites = FALSE) +psRace <- function(iraceResults, max_experiments, conf_ids = NULL, iteration_elites = FALSE, + psrace_logFile = NULL) { - irace.note("Starting post-selection:\n") + irace_note("Starting post-selection:\n") if (missing(iraceResults)) stop("argument 'iraceResults' is missing.") iraceResults <- read_logfile(iraceResults) - scenario <- iraceResults$scenario + scenario <- iraceResults$scenario + if (!is.null(psrace_logFile)) + scenario$logFile <- psrace_logFile + race_state <- iraceResults$state race_state$initialize(scenario, new = FALSE) # Restores the random seed if (max_experiments <= 0) stop("'max_experiments' must be positive.") @@ -54,11 +64,42 @@ psRace <- function(iraceResults, max_experiments, conf_ids = NULL, iteration_eli # Get selected configurations. if (is.null(conf_ids)) { # FIXME: Handle scenario$maxTime > 0 - irace.assert(scenario$maxTime == 0) + irace_assert(scenario$maxTime == 0) which_max_last <- function(x) 1L + length(x) - which.max(rev(x)) - - get_confs_for_psrace <- function(iraceResults, blockSize, iteration_elites, max_experiments, conf_ids, rejected_ids) { + + # We want to race at least two configurations, so we generate all + # integers between 3 and 2^max_confs - 1L that are not powers of 2, then convert + # it to a bit-string then to a logical vector. + generate_combs_2 <- function(max_confs) { + max_pow <- as.integer(2^max_confs) + pow_2 <- as.integer(2L^(0L:(max_confs-1L))) + s <- seq.int(3L, max_pow - 1L, 2L) + if (max_pow >= 8L) + s <- c(s, setdiff(seq.int(6L, max_pow - 2L, 2L), pow_2[-1L:-3L])) + lapply(s, function(z) as.logical((z %/% pow_2) %% 2L)) + } + # A version of the above that is not limited to two configurations. + generate_combs_1 <- function(max_confs) { + max_pow <- as.integer(2^max_confs) + pow_2 <- as.integer(2L^(0L:(max_confs-1L))) + lapply(seq.int(1L, max_pow - 1L, 1L), + function(z) as.logical((z %/% pow_2) %% 2L)) + } + # FIXME: Is this really faster than head(x, n=max_len) ? + truncate_conf_needs <- function(x, max_len) { + if (length(x) <= max_len) return(x) + x[seq_len(max_len)] + } + + get_confs_for_psrace <- function(iraceResults, iteration_elites, max_experiments, conf_ids, rejected_ids) { + scenario <- iraceResults$scenario + report_selected <- + if (scenario$quiet) do_nothing + else function(conf_ids, conf_needs) + cat(sep="", "# Configurations selected: ", paste0(collapse=", ", conf_ids), + ".\n# Pending instances: ", paste0(collapse=", ", conf_needs), ".\n") + allElites <- iraceResults$allElites experiments <- iraceResults$experiments conf_ids <- if (iteration_elites) unlist(rev(allElites)) else allElites[[length(allElites)]] @@ -70,37 +111,93 @@ psRace <- function(iraceResults, max_experiments, conf_ids = NULL, iteration_eli if (length(rejected_ids)) conf_ids <- setdiff(conf_ids, rejected_ids) experiments <- experiments[, conf_ids, drop = FALSE] - conf_needs <- colSums(is.na(experiments)) + conf_needs <- matrixStats::colCounts(experiments, value = NA, useNames = TRUE) + n_done <- nrow(experiments) - min(conf_needs) # Remove any configuration that needs more than max_experiments. conf_needs <- conf_needs[conf_needs <= max_experiments] - # We want to evaluate in at least scenario$blockSize instances more. - n_confs <- floor(max_experiments / blockSize) - # If we have n_confs that have been evaluated in all instances, select those. - if (sum(conf_needs == 0L) >= n_confs) { - conf_needs <- conf_needs[conf_needs == 0L][1:n_confs] - conf_ids <- names(conf_needs) - irace.note("Configurations selected: ", paste0(collapse=", ", conf_ids), ".\n") - irace.note("Pending instances: ", paste0(collapse=", ", conf_needs), ".\n") - return(conf_ids) + if (length(conf_needs) == 1L) { + if (!scenario$quiet) + cat("# Insufficient budget to race more than one configuration!\n") + return(names(conf_needs)) } - # So we need a new instance, that will require blockSize for each configuration. - conf_ids <- names(conf_needs) - conf_needs <- conf_needs + scenario$blockSize - total_needs <- sum(conf_needs) - while (total_needs > max_experiments) { - to_remove <- which_max_last(conf_needs) - total_needs <- total_needs - conf_needs[to_remove] - conf_ids <- conf_ids[-to_remove] - conf_needs <- conf_needs[-to_remove] - irace.assert(length(conf_ids) > 1L, eval_after=save(iraceResults, file="bug-conf_ids.Rdata")) + if (!scenario$deterministic || n_done < length(scenario$instances)) { + # We want to evaluate in at least n_new instances more. + n_new <- scenario$eachTest * scenario$blockSize + n_confs <- floor(max_experiments / n_new) + # If we have n_confs that have been evaluated in all instances, select those. + if (n_confs > 1L && sum(conf_needs == 0L) >= n_confs) { + conf_needs <- conf_needs[conf_needs == 0L][1:n_confs] + conf_ids <- names(conf_needs) + report_selected(conf_ids, conf_needs) + irace_assert(length(conf_ids) > 1L, eval_after = { + cat("n_confs: ", n_confs, "\nn_new:", n_new, "\n") + print(conf_needs) + save(iraceResults, file="bug-conf_ids.Rdata") + }) + return(conf_ids) + } + # Let's try first to evaluate on new instances. + conf_needs_new <- truncate_conf_needs(conf_needs + n_new, max_len = 16L) + irace_assert(length(conf_needs_new) > 1L, eval_after={ + cat("max_experiments: ", max_experiments, "\nn_new: ", n_new, "\nconf_needs:") + print(conf_needs) + save(iraceResults, file="bug-conf_ids.Rdata") + }) + combs <- generate_combs_2(length(conf_needs_new)) + left <- sapply(combs, function(x) max_experiments - sum(conf_needs_new[x]), USE.NAMES=FALSE) + irace_assert(!is.null(left) && length(left) > 0L && !anyNA(left), + eval_after= { + cat("max_experiments: ", max_experiments, "\n") + cat("length(left): ", length(left), "\n") + cat("sum(is.na(left)): ", sum(is.na(left)), "\n") + save(iraceResults, file="bug-conf_ids.Rdata") + }) + + if (any(left >= 0L)) { # We have enough budget to evaluate on a new instance. + # Select the combination that will allow to evaluate the most configurations. + combs <- combs[left >= 0] + n <- sapply(combs, sum, USE.NAMES=FALSE) + winner <- which.max(n) + conf_needs_new <- conf_needs_new[combs[[winner]]] + conf_ids <- names(conf_needs_new) + report_selected(conf_ids, conf_needs_new) + irace_assert(length(conf_ids) > 1L, eval_after = { + cat("max_experiments: ", max_experiments, "\n") + cat("length(left): ", length(left), "\n") + cat("sum(is.na(left)): ", sum(is.na(left)), "\n") + cat("winner: ", winner, "\n") + cat("combs[[winner]]: ", paste0(collapse=",", combs[[winner]]), "\n") + save(iraceResults, file="bug-conf_ids.Rdata") + }) + return(conf_ids) + } } - irace.note("Configurations selected: ", paste0(collapse=", ", conf_ids), ".\n") - irace.note("Pending instances: ", paste0(collapse=", ", conf_needs), ".\n") - conf_ids + # We do not have enough budget to see new instances, so we need consider + # at least 1 configuration that has NA values, but we still include the + # configurations that have been evaluated on all instances. + conf_needs_zero <- conf_needs[conf_needs == 0L] + conf_needs <- truncate_conf_needs(conf_needs[conf_needs > 0L], 16L) + combs <- generate_combs_1(length(conf_needs)) + left <- sapply(combs, function(x) max_experiments - sum(conf_needs[x]), USE.NAMES=FALSE) + irace_assert(any(left >= 0), eval_after=save(iraceResults, file="bug-conf_ids.Rdata")) + # Select the combination that will allow us to evaluate the most configurations. + combs <- combs[left >= 0] + n <- sapply(combs, sum, USE.NAMES=FALSE) + winner <- which.max(n) + conf_needs <- c(conf_needs_zero, conf_needs[combs[[winner]]]) + conf_ids <- names(conf_needs) + report_selected(conf_ids, conf_needs) + irace_assert(length(conf_ids) > 1L, eval_after = { + print(conf_needs) + cat("winner: ", winner, "\n") + cat("combs[[winner]]: ", paste0(collapse=",", combs[[winner]]), "\n") + save(iraceResults, file="bug-conf_ids.Rdata") + }) + return(conf_ids) } - conf_ids <- get_confs_for_psrace(iraceResults, scenario$blockSize, iteration_elites, - max_experiments, conf_ids = conf_ids, rejected_ids = race_state$rejected_ids) - irace.assert(length(conf_ids) > 1L, eval_after = { + conf_ids <- get_confs_for_psrace(iraceResults, iteration_elites, max_experiments, + conf_ids = conf_ids, rejected_ids = race_state$rejected_ids) + irace_assert(length(conf_ids) >= 1L, eval_after = { # Debug what happened if the assert failed. rejected_ids <- race_state$rejected_ids cat("blockSize:", scenario$blockSize, "\niteration_elites: ", as.character(iteration_elites), @@ -119,48 +216,54 @@ psRace <- function(iraceResults, max_experiments, conf_ids = NULL, iteration_eli if (length(rejected_ids)) conf_ids <- setdiff(conf_ids, rejected_ids) cat("conf_ids4:", paste0(collapse=", ", conf_ids), "\n") - experiments <- - conf_needs <- colSums(is.na(experiments[, conf_ids, drop = FALSE])) + conf_needs <- matrixStats::colCounts(experiments[, conf_ids, drop = FALSE], value = NA) cat("conf_needs:", paste0(collapse=", ", conf_needs), "\n") }) - + if (length(conf_ids) == 1L) { + # If we cannot run post-selection, we just return the elite configurations. + allElites <- iraceResults$allElites + return(allElites[[length(allElites)]]) + } + } else if (length(conf_ids) <= 1L) { - irace.error ("The number configurations provided should be larger than 1.") + irace_error ("The number configurations provided should be larger than 1.") } else if (length(race_state$rejected_ids) && any(conf_ids %in% race_state$rejected_ids)) { - irace.error ("Some configuration IDs provided were rejected in the previous run: ", + irace_error ("Some configuration IDs provided were rejected in the previous run: ", paste0(collapse=", ", intersect(conf_ids, race_state$rejected_ids)), ".") } if (!all(conf_ids %in% iraceResults$allConfigurations[[".ID."]])) { - irace.error("Some configuration IDs provided cannot be found in the configurations: ", + irace_error("Some configuration IDs provided cannot be found in the configurations: ", paste0(collapse=", ", setdiff(conf_ids, iraceResults$allConfigurations[[".ID."]])), ".") } elite_configurations <- iraceResults$allConfigurations[iraceResults$allConfigurations[[".ID."]] %in% conf_ids, , drop=FALSE] # Generate new instances. generateInstances(race_state, scenario, max_experiments / nrow(elite_configurations), update = TRUE) - elite_data <- if (scenario$elitist) - iraceResults$experiments[, as.character(elite_configurations[[".ID."]]), drop=FALSE] else NULL - race_state$next_instance <- nrow(iraceResults$experiments) + 1L + elite_data <- iraceResults$experiments[, as.character(elite_configurations[[".ID."]]), drop=FALSE] + race_state$next_instance <- nrow(elite_data) + 1L - cat(sep="", - "# Seed: ", race_state$seed, + irace_note("seed: ", race_state$seed, "\n# Configurations: ", nrow(elite_configurations), "\n# Available experiments: ", max_experiments, "\n# minSurvival: 1\n") - + # We do not want to stop because of this limit. + scenario$elitistLimit <- 0L + # FIXME: elitist_race should not require setting this, but it currently does. + scenario$elitist <- TRUE raceResults <- elitist_race(race_state, maxExp = max_experiments, minSurvival = 1L, configurations = elite_configurations, scenario = scenario, - elite.data = elite_data, - elitist_new_instances = 0L) + elite_data = elite_data, + elitist_new_instances = 0L, + # FIXME: This should be nrow(elite_data) + 1L if we have budget to evaluate on new instances. + firstTest = nrow(elite_data) / scenario$blockSize) elite_configurations <- extractElites(raceResults$configurations, - nbElites = race_state$recovery_info$minSurvival, - debugLevel = scenario$debugLevel) - irace.note("Elite configurations (first number is the configuration ID;", + nbElites = race_state$minSurvival, debugLevel = scenario$debugLevel) + irace_note("Elite configurations (first number is the configuration ID;", " listed from best to worst according to the ", test.type.order.str(scenario$testType), "):\n") if (!scenario$quiet) @@ -174,23 +277,21 @@ psRace <- function(iraceResults, max_experiments, conf_ids = NULL, iteration_eli indexIteration <- 1L + max(race_state$experiment_log[["iteration"]]) # We add indexIteration as an additional column. set(raceResults$experiment_log, j = "iteration", value = indexIteration) - race_state$experiment_log <- rbind(race_state$experiment_log, raceResults$experiment_log) + race_state$experiment_log <- rbindlist(list(race_state$experiment_log, raceResults$experiment_log), fill=TRUE, use.names=TRUE) # Merge new results. iraceResults$experiments <- merge_matrix(iraceResults$experiments, raceResults$experiments) iraceResults$iterationElites[indexIteration] <- elite_configurations[[".ID."]][1L] iraceResults$allElites[[indexIteration]] <- elite_configurations[[".ID."]] - race_state$elite_configurations <- elite_configurations + iraceResults$scenario <- scenario iraceResults$state <- race_state - irace_save_logfile(iraceResults, scenario) + # FIXME: This log should contain only information of what was done in the + # psRace and avoid duplicating info from iraceResults. + iraceResults$psrace_log <- list(configurations = elite_configurations, + instances = race_state$instances_log[seq_len(nrow(raceResults$experiments)), , drop = FALSE], + max_experiments = max_experiments, + experiments = raceResults$experiments, + elites = elite_configurations[[".ID."]]) + save_irace_logfile(iraceResults, logfile = scenario$logFile) } - - # FIXME: This log should contain only information of what was done in the - # psRace and avoid duplicating info from iraceResults. - psrace_log <- list(configurations = elite_configurations, - instances = race_state$instances_log[seq_nrow(iraceResults$experiments), , drop = FALSE], - max_experiments = max_experiments, - experiments = iraceResults$experiments, - elites = elite_configurations[[".ID."]]) - iraceResults$psrace_log <- psrace_log elite_configurations } diff --git a/R/race-wrapper.R b/R/race-wrapper.R index 43dd9074..008d94c2 100644 --- a/R/race-wrapper.R +++ b/R/race-wrapper.R @@ -5,13 +5,13 @@ #' parameters. It builds a string with the switches and the values that can #' be used as a command line to call the program to be tuned, thus generating #' one candidate configuration. -#' -#' +#' +#' #' @param values A vector containing the value of each parameter for the #' candidate configuration. #' @param switches A vector containing the switches of each paramter (in an #' order that corresponds to the values vector). -#' +#' #' @return A string concatenating each element of `switches` and #' `values` for all parameters with a space between each pair of #' parameters (but none between the switches and the corresponding values). @@ -28,12 +28,12 @@ #' parameters <- iraceResults$scenario$parameters #' apply(allConfigurations[1:10, unlist(parameters$names)], 1, buildCommandLine, #' unlist(parameters$switches)) -#' +#' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export buildCommandLine <- function(values, switches) { - irace.assert(length(values) == length(switches)) + irace_assert(length(values) == length(switches)) values <- as.list(values) sel <- !is.na(values) switches <- switches[sel] @@ -45,17 +45,17 @@ buildCommandLine <- function(values, switches) # numeric vector. parse_output <- function(outputRaw, verbose) { - if (verbose) { cat (outputRaw, sep = "\n") } - + if (verbose) + cat(paste0(outputRaw, collapse = "\n")) # Initialize output as raw. If it is empty stays like this. output <- outputRaw # strsplit crashes if outputRaw == character(0) - if (length(outputRaw) > 0) { + if (length(outputRaw)) { outputRaw <- paste0(outputRaw, collapse = "\n") - output <- strsplit(trim(outputRaw), "[[:space:]]+")[[1]] + output <- strsplit(trim(outputRaw), "[[:space:]]+")[[1L]] } # suppressWarnings to avoid messages about NAs introduced by coercion - suppressWarnings (as.numeric (output)) + suppressWarnings(as.numeric(output)) } target_error <- function(err_msg, output, scenario, target_runner_call, @@ -71,8 +71,7 @@ target_error <- function(err_msg, output, scenario, target_runner_call, } if (is.null(output$outputRaw)) { # Message for a function call. - # FIXME: Ideally, we should print the list as R would print it. - output$outputRaw <- toString(output) + output$outputRaw <- deparse1(output) advice_txt <- paste0( "This is not a bug in irace, but means that something failed in ", "a call to the targetRunner or targetEvaluator functions provided by the user.", @@ -85,12 +84,13 @@ target_error <- function(err_msg, output, scenario, target_runner_call, " Try to run the command(s) above from the execution directory '", scenario$execDir, "' to investigate the issue. See also Appendix B (targetRunner troubleshooting checklist) of the User Guide (https://cran.r-project.org/package=irace/vignettes/irace-package.pdf).") } - irace.error(err_msg, "\n", .irace_msg_prefix, + irace_error(err_msg, "\n", .irace_msg_prefix, "The output was:\n", paste(output$outputRaw, collapse = "\n"), "\n", .irace_msg_prefix, advice_txt) } -check_output_target_evaluator <- function (output, scenario, target_runner_call = NULL, bound = NULL) +check_output_target_evaluator <- function (output, scenario, target_runner_time, + target_runner_call, bound) { if (!is.list(output)) { output <- list() @@ -106,15 +106,23 @@ check_output_target_evaluator <- function (output, scenario, target_runner_call } else if (is_na_nowarn(output$cost)) { err_msg <- "The output of targetEvaluator is not numeric!" } - if (scenario$batchmode != 0 && scenario$maxTime > 0) { - if (is.null (output$time)) { - err_msg <- "When batchmode != 0 and maxTime > 0, the output of targetEvaluator must be two numbers 'cost time'!" + if (scenario$maxTime > 0 || scenario$capping) { + if (scenario$batchmode != 0) { + if (is.null (output$time)) + err_msg <- "When batchmode != 0 and maxTime > 0, the output of targetEvaluator must be two numbers 'cost time'!" + # With scenario$capping == TRUE, we may have pre-executed targetRunner + # (which_elite_exe) that already have recorded the time, so when we + # reach this point, we may not have 'time'. + } else if (!scenario$capping && is.null(target_runner_time) && is.null(output$time)) { + err_msg <- "Either targetRunner or targetEvaluator must return 'time' !" } } if (is.null(output$time)) { - output$time <- NA + output$time <- target_runner_time } else { - if (is_na_nowarn(output$time)) { + if (!is.null(target_runner_time)) { + err_msg <- "Both targetRunner and targetEvaluator cannot return 'time' !" + } else if (is_na_nowarn(output$time)) { err_msg <- "The time returned by targetEvaluator is not numeric!" } else if (is.infinite(output$time)) { err_msg <- "The time returned by targetEvaluator is not finite!" @@ -130,10 +138,12 @@ check_output_target_evaluator <- function (output, scenario, target_runner_call } } - if (!is.null(err_msg)) { + if (is.null(err_msg)) { + output$error <- NULL + } else { target_error (err_msg, output, scenario, - target_runner_call = target_runner_call, - target_evaluator_call = output$call) + target_runner_call = target_runner_call, + target_evaluator_call = output$call) } output } @@ -144,7 +154,7 @@ check_output_target_evaluator <- function (output, scenario, target_runner_call #' invoked if `targetEvaluator` is a string (by default #' `targetEvaluator` is `NULL` and this function is not invoked). You can use it as #' an advanced example of how to create your own `targetEvaluator` function. -#' +#' #' @param experiment A list describing the experiment. It contains at least: #' \describe{ #' \item{`id_configuration`}{An alphanumeric string that uniquely identifies a configuration;} @@ -155,8 +165,6 @@ check_output_target_evaluator <- function (output, scenario, target_runner_call #' \item{`bound`}{(only when `capping` is enabled) Time bound for the execution;} #' \item{`configuration`}{1-row data frame with a column per parameter #' name;} -#' \item{`switches`}{Vector of parameter switches (labels) in the order -#' of parameters used in `configuration`.} #' } #' @param num_configurations Number of configurations alive in the race. #' @param all_conf_id Vector of configuration IDs of the alive configurations. @@ -165,22 +173,22 @@ check_output_target_evaluator <- function (output, scenario, target_runner_call #' corresponds to this call to `targetEvaluator`. This is used for #' providing extra information to the user, for example, in case #' `targetEvaluator` fails. -#' +#' #' @return The function `targetEvaluator` must return a list with one element #' `"cost"`, the numerical value corresponding to the cost measure of the #' given configuration on the given instance. -#' +#' #' The return list may also contain the following optional elements that are used #' by \pkg{irace} for reporting errors in `targetEvaluator`: #' \describe{ #' \item{`error`}{is a string used to report an error;} #' \item{`outputRaw`}{is a string used to report the raw output of calls to #' an external program or function;} -#' \item{`call`}{is a string used to report how `targetRunner` called +#' \item{`call`}{is a string used to report how `targetRunner` called #' an external program or function.} #' } #' -#' +#' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export target_evaluator_default <- function(experiment, num_configurations, all_conf_id, @@ -194,7 +202,7 @@ target_evaluator_default <- function(experiment, num_configurations, all_conf_id debugLevel <- scenario$debugLevel targetEvaluator <- scenario$targetEvaluator if (as.logical(file.access(targetEvaluator, mode = 1))) { - irace.error ("targetEvaluator", shQuote(targetEvaluator), + irace_error ("targetEvaluator", shQuote(targetEvaluator), "cannot be found or is not executable!\n") } all_conf_id <- paste0(all_conf_id, collapse = " ") @@ -223,15 +231,15 @@ target_evaluator_default <- function(experiment, num_configurations, all_conf_id call = paste(targetEvaluator, args, collapse=" ")) } -#' Check the output of the target runner and repair it if possible. If the +#' Check the output of the target runner and repair it if possible. If the #' output is incorrect, this function will throw an error. -#' +#' #' @param output The output from target runner. #' @inheritParams defaultScenario #' @param bound Optional time bound that the target runner should have respected. -#' +#' #' @return The output with its contents repaired. -#' +#' #' @export check_output_target_runner <- function(output, scenario, bound = NULL) { @@ -240,30 +248,28 @@ check_output_target_runner <- function(output, scenario, bound = NULL) target_error ("The output of targetRunner must be a list", output, scenario, target_runner_call = NULL) return(output) } - + err_msg <- output$error if (is.null(err_msg)) { - if (!is.null (output$cost)) { - if (is_na_or_empty(output$cost)) { - err_msg <- "The cost returned by targetRunner is not numeric!" - } + if (is.null(output$cost)) { + output$cost <- NULL # make sure to delete it. + } else if (is_na_or_empty(output$cost)) { + err_msg <- "The cost returned by targetRunner is not numeric!" } if (is.null(output$time)) { - output$time <- NA + output$time <- NULL # make sure to delete it. + } else if (is.na(output$time)) { + err_msg <- paste0("The time returned by targetRunner is not numeric!") + } else if (is.infinite(output$time)) { + err_msg <- paste0("The time returned by targetRunner is not finite!") + } else if (output$time <= 0) { + err_msg <- paste0("The value of time (", output$time, ") returned by targetRunner must be strictly positive!") } else { - if (is.na(output$time)) { - err_msg <- paste0("The time returned by targetRunner is not numeric!") - } else if (is.infinite(output$time)) { - err_msg <- paste0("The time returned by targetRunner is not finite!") - } else if (output$time <= 0) { - err_msg <- paste0("The value of time (", output$time, ") returned by targetRunner must be strictly positive!") - } else { - # Fix time. - output$time <- max(output$time, scenario$minMeasurableTime) - if (!is.null(bound) && !is.na(bound) && bound > 0 && bound + scenario$minMeasurableTime < output$time) { - err_msg <- paste0("The time returned by targetRunner (", output$time, ") does not respect the given bound of ", bound, "!") - } + # Fix time. + output$time <- max(output$time, scenario$minMeasurableTime) + if (!is.null(bound) && !is.na(bound) && bound > 0 && bound + scenario$minMeasurableTime < output$time) { + err_msg <- paste0("The time returned by targetRunner (", output$time, ") does not respect the given bound of ", bound, "!") } } if (is.null(err_msg)) { @@ -272,23 +278,25 @@ check_output_target_runner <- function(output, scenario, bound = NULL) # unless using batchmode, in that case targetRunner returns neither the # time nor the cost. if (scenario$batchmode != 0) { - if (!is.na(output$time) || !is.null(output$cost)) { + if (!is.null(output$time) || !is.null(output$cost)) { err_msg <- "When batchmode != 0, the output of targetRunner must not contain a cost nor a time!" } - } else if (scenario$maxTime > 0 && is.na(output$time)) { + } else if (scenario$maxTime > 0 && is.null(output$time)) { err_msg <- "The output of targetRunner must be one number 'time'!" } else if (!is.null(output$cost)) { err_msg <- "The output of targetRunner must be empty or just one number 'time'!" } - } else if (scenario$maxTime > 0 && (is.null(output$cost) || is.na(output$time))) { + } else if (scenario$maxTime > 0 && (is.null(output$cost) || is.null(output$time))) { err_msg <- "The output of targetRunner must be two numbers 'cost time'!" } else if (scenario$maxExperiments > 0 && is.null(output$cost)) { err_msg <- "The output of targetRunner must be one number 'cost'!" - } + } } } - if (!is.null(err_msg)) { + if (is.null(err_msg)) { + output$error <- NULL + } else { target_error (err_msg, output, scenario, target_runner_call = output$call) } output @@ -303,13 +311,13 @@ exec_target_runner <- function(experiment, scenario, target_runner) x <- target_runner(experiment, scenario) check_output_target_runner(x, scenario, bound = experiment$bound) } - + retries <- scenario$targetRunnerRetries while (retries > 0L) { output <- try (doit(experiment, scenario)) if (!inherits(output, "try-error") && is.null(output$error)) return (output) - irace.note("Retrying (", retries, " left).\n") + irace_note("Retrying (", retries, " left).\n") retries <- retries - 1L } doit(experiment, scenario) @@ -346,7 +354,7 @@ parse.aclib.output <- function(outputRaw) } else { error <- paste0("Not valid AClib output status (", status, ")") } - return(list(status = status, cost = cost, time = runtime, error = error)) + list(status = status, cost = cost, time = runtime, error = error) } target_runner_aclib <- function(experiment, scenario) @@ -356,13 +364,13 @@ target_runner_aclib <- function(experiment, scenario) cmd <- res$cmd output <- res$output args <- res$args - + err_msg <- output$error if (is.null(err_msg)) { return(c(parse.aclib.output(output$output), list(outputRaw = output$output, call = paste(cmd, args)))) } - + list(cost = NULL, time = NULL, error = err_msg, outputRaw = output$output, call = paste(cmd, args)) } @@ -373,7 +381,7 @@ check_target_cmdline <- function(target_cmdline, launcher, capping) if (capping) required <- c(required, "bound") for (x in required) { if (!grepl(paste0("{", x, "}"), target_cmdline, fixed=TRUE)) - irace.error("targetCmdline '", target_cmdline, "' must contain '{", x, "}'") + irace_error("targetCmdline '", target_cmdline, "' must contain '{", x, "}'") } } @@ -394,7 +402,7 @@ expand_target_cmdline <- function(target_cmdline, experiment, targetRunner, targ } target_cmdline } - + run_target_runner <- function(experiment, scenario) { configuration_id <- experiment$id_configuration @@ -402,16 +410,17 @@ run_target_runner <- function(experiment, scenario) seed <- experiment$seed configuration <- experiment$configuration instance <- experiment$instance - switches <- experiment$switches bound <- experiment$bound + switches <- scenario$parameters$switches[names(configuration)] + targetRunner <- scenario[["targetRunner"]] debugLevel <- scenario$debugLevel - + if (scenario$aclib) { # FIXME: Use targetCmdline for this has_value <- !is.na(configuration) - # [] [] ... [--cutoff ] [--instance ] + # [] [] ... [--cutoff ] [--instance ] # [--seed ] --config [-param_name_1 value_1] [-param_name_2 value_2] ... args <- paste("--instance", instance, "--seed", seed, "--config", paste0("-", switches[has_value], " ", configuration[has_value], @@ -425,7 +434,7 @@ run_target_runner <- function(experiment, scenario) targetRunnerLauncher <- scenario$targetRunnerLauncher if (!is.null.or.empty(targetRunnerLauncher)) targetRunner <- targetRunnerLauncher - + output <- runcommand(targetRunner, args, configuration_id, debugLevel, timeout = scenario$targetRunnerTimeout) list(cmd=targetRunner, output=output, args=args) } @@ -433,7 +442,7 @@ run_target_runner <- function(experiment, scenario) #' Default `targetRunner` function. #' #' Use it as an advanced example of how to create your own `targetRunner` function. -#' +#' #' @param experiment A list describing the experiment. It contains at least: #' \describe{ #' \item{`id_configuration`}{An alphanumeric string that uniquely identifies a configuration;} @@ -444,17 +453,15 @@ run_target_runner <- function(experiment, scenario) #' \item{`bound`}{(only when `capping` is enabled) Time bound for the execution;} #' \item{`configuration`}{1-row data frame with a column per parameter #' name;} -#' \item{`switches`}{Vector of parameter switches (labels) in the order -#' of parameters used in `configuration`.} #' } #' @inheritParams defaultScenario -#' +#' #' @return If `targetEvaluator` is `NULL`, then the `targetRunner` #' function must return a list with at least one element `"cost"`, #' the numerical value corresponding to the evaluation of the given #' configuration on the given instance. -#' -#' If the scenario option `maxTime` is non-zero or if `capping` is enabled +#' +#' If the scenario option `maxTime` is non-zero or if `capping` is enabled #' then the list must contain at least another element `"time"` that reports the #' execution time for this call to `targetRunner`. #' The return list may also contain the following optional elements that are used @@ -463,11 +470,11 @@ run_target_runner <- function(experiment, scenario) #' \item{`error`}{is a string used to report an error;} #' \item{`outputRaw`}{is a string used to report the raw output of calls to #' an external program or function;} -#' \item{`call`}{is a string used to report how `targetRunner` called +#' \item{`call`}{is a string used to report how `targetRunner` called #' an external program or function.} #' } #' -#' +#' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export target_runner_default <- function(experiment, scenario) @@ -476,17 +483,17 @@ target_runner_default <- function(experiment, scenario) cmd <- res$cmd output <- res$output args <- res$args - + debugLevel <- scenario$debugLevel cost <- time <- NULL err_msg <- output$error if (is.null(err_msg)) { v_output <- parse_output(output$output, verbose = (debugLevel >= 2L)) if (length(v_output) == 1L) { - if (!is.null(scenario$targetEvaluator)) { - time <- v_output[1L] - } else { + if (is.null(scenario$targetEvaluator)) { cost <- v_output[1L] + } else { + time <- v_output[1L] } } else if (length(v_output) == 2L) { cost <- v_output[1L] @@ -507,21 +514,21 @@ execute_experiments <- function(race_state, experiments, scenario) target_runner <- race_state$target_runner execDir <- scenario$execDir if (!fs::dir_exists(execDir)) - irace.error ("Execution directory '", execDir, "' is not found or not a directory\n") + irace_error ("Execution directory '", execDir, "' is not found or not a directory\n") withr::local_dir(execDir) - + if (!is.null(scenario$targetRunnerParallel)) { # FIXME: We should remove the exec_target_runner parameter from # targetRunnerParallel and do lapply(target_output, - # check_output_target_runner, scenario=scenario) after it to make sure the output is valid. + # check_output_target_runner, scenario=scenario) after it to make sure the output is valid. # User-defined parallelization target_output <- scenario$targetRunnerParallel(experiments, exec_target_runner, scenario = scenario, target_runner = target_runner) if (length(target_output) != length(experiments)) { - irace.error("Stopping because the output of targetRunnerParallel is missing elements. The output was:\n", - paste0(capture.output(str(target_output)), collapse="\n")) + irace_error("Stopping because the output of targetRunnerParallel is missing elements. The output was:\n", + paste0(utils::capture.output(utils::str(target_output)), collapse="\n")) } } else if (scenario$batchmode != 0L) { target_output <- cluster_lapply(experiments, scenario = scenario) @@ -551,11 +558,11 @@ execute_experiments <- function(race_state, experiments, scenario) # each configuration and repetitions may occur. cat(unique(unlist(target_output[sapply( target_output, inherits, "try-error")])), file = stderr(), sep = "") - irace.error("A slave process terminated with a fatal error") + irace_error("A slave process terminated with a fatal error") } } else { if (.Platform$OS.type == 'windows') { - irace.assert(!is.null(race_state$cluster)) + irace_assert(!is.null(race_state$cluster)) if (scenario$loadBalancing) { target_output <- parallel::parLapplyLB(race_state$cluster, experiments, exec_target_runner, @@ -589,7 +596,7 @@ execute_experiments <- function(race_state, experiments, scenario) cat(unique(unlist( target_output[sapply( target_output, inherits, "try-error")])), file = stderr()) - irace.error("A child process triggered a fatal error") + irace_error("A child process triggered a fatal error") } } } @@ -602,11 +609,9 @@ execute_experiments <- function(race_state, experiments, scenario) target_output } -execute_evaluator <- function(target_evaluator, experiments, scenario, target_output, configurations_id) +execute_evaluator <- function(target_evaluator, experiments, scenario, target_output) { - ## FIXME: We do not need the configurations_id argument: - irace.assert(isTRUE(all.equal(configurations_id, - sapply(experiments, getElement, "id_configuration")))) + configurations_id <- unique(unlist_element(experiments, "id_configuration")) nconfs <- length(configurations_id) # Evaluate configurations sequentially. for (k in seq_along(experiments)) { @@ -615,12 +620,11 @@ execute_evaluator <- function(target_evaluator, experiments, scenario, target_ou output <- target_evaluator(experiment = experiment, num_configurations = nconfs, all_conf_id = configurations_id, scenario = scenario, target_runner_call = target_runner_call) - output <- check_output_target_evaluator(output, scenario, target_runner_call = target_runner_call, bound = experiment$bound) + output <- check_output_target_evaluator(output, scenario, target_runner_time = target_output[[k]]$time, + target_runner_call = target_runner_call, bound = experiment$bound) target_output[[k]]$cost <- output$cost - if (is.null(target_output[[k]]$call)) - target_output[[k]]$call <- output$call - if (is.null(target_output[[k]]$time) || !is.null.or.na(output$time)) - target_output[[k]]$time <- output$time + # targetEvaluator may return time, for example for batchmode != 0. + target_output[[k]]$time <- output$time } target_output } diff --git a/R/race.R b/R/race.R index 1ccce318..c9a2a14b 100644 --- a/R/race.R +++ b/R/race.R @@ -1,7 +1,7 @@ # ---------------------------------------- -*- mode: r; mode: font-lock -*- # # race.R Racing methods for the selection of the best # # ------------------------------------------------------------------------- # - + # ========================================================================= # # Racing methods for the selection of the best # # ------------------------------------------------------------------------- # @@ -31,42 +31,37 @@ # $Id: race.R,v 1.54 2005/03/30 12:40:42 mbiro Exp $ # createExperimentList <- function(configurations, parameters, - instances, instances_ID, seeds, - bounds = NULL) + instances, instances_ID, seeds, bounds) { - instances <- instances[instances_ID] n_configurations <- nrow(configurations) - n_instances <- length(instances) - pnames <- parameters$names configurations_id <- configurations[[".ID."]] + pnames <- parameters$names configurations <- as.list(configurations[, pnames, drop=FALSE]) # FIXME: How to do this faster? # - maybe purrr::transpose() ? - # - mlr3misc::transpose_list() uses .mapply(list, configurations, list(), which is slower. + # - mlr3misc::transpose_list() uses .mapply(list, configurations, list(), which is slower. configurations <- lapply(seq_len(n_configurations), function(i) lapply(configurations, `[`, i)) - dots <- list(id_configuration = rep(configurations_id, each=n_instances), id_instance = instances_ID, - seed = seeds, configuration = rep(configurations, each = n_instances), - instance = instances) + dots <- list(id_configuration = configurations_id, configuration = configurations) if (!is.null(bounds)) { # There must be a bound for each configuration. # FIXME: There must be a bound for each configuration AND each instance. - irace.assert(length(bounds) == n_configurations) - dots$bound <- rep.int(bounds, n_instances) + irace_assert(length(bounds) == n_configurations) + dots$bound <- bounds } - dots$switches <- list(parameters$switches) - .mapply(list, dots, MoreArgs = NULL) + configurations <- .mapply(list, dots = dots, MoreArgs = NULL) + instances <- instances[instances_ID] + n_instances <- length(instances_ID) + instances <- .mapply(list, + dots = list(id_instance = instances_ID, seed = seeds, instance = instances), + MoreArgs = NULL) + .mapply(c, dots = list( + rep(instances, each = n_configurations), + rep(configurations, times = n_instances)), MoreArgs = NULL) } -## Executes a list of configurations in a particular instance -## configurations: description having the id of the configuration -## instance.idx: index of the instance,seed pair in race_state$instances_log -## bounds: execution bounds (if needed). -## which_alive: index of the configurations that are still alive -## which_exe: index of the alive configurations that should be executed -race_wrapper <- function(race_state, configurations, instance_idx, bounds = NULL, - # FIXME: we actually only need which_exps, not - # which_alive nor which_exe - which_alive, which_exe, scenario) + +race_wrapper_helper <- function(race_state, configurations, instance_idx, bounds, + is_exe, scenario) { # Experiment list to execute experiments <- createExperimentList(configurations, parameters = scenario$parameters, @@ -75,41 +70,83 @@ race_wrapper <- function(race_state, configurations, instance_idx, bounds = NULL seeds = race_state$instances_log[["seed"]][instance_idx], bounds = bounds) - target_output <- vector("list", length(experiments)) - # Execute commands - if (length(which_exe)) { - # which_exe values are within 1:nbConfigurations, whereas experiments - # indices are within 1:length(which_alive). The following line converts - # from one to the other. - which_exps <- which(which_alive %in% which_exe) - irace.assert(length(which_exps) == length(which_exe)) - target_output[which_exps] <- execute_experiments(race_state, experiments[which_exps], scenario) + irace_assert(length(is_exe) == length(experiments)) + instance_idx <- rep(instance_idx, each = nrow(configurations)) + if (race_state$recovery_mode) { + # With targetEvaluator or if everything is executed, we recover everything. + if (!is.null(scenario$targetEvaluator) || all(is_exe)) { + configuration_id <- unlist_element(experiments, "id_configuration") + target_output <- race_state$recover_output(instance_idx, configuration_id) + } else { + irace_assert(any(is_exe)) + configuration_id <- unlist_element(experiments[is_exe], "id_configuration") + target_output <- race_state$recover_output(instance_idx[is_exe], configuration_id) + } + } else { # !recovery_mode + # We cannot let targetRunner or targetEvaluator modify our random seed, so we save it. + withr::local_preserve_seed() + target_output <- vector("list", length(experiments)) + # Execute experiments for which is_exe is TRUE: + if (any(is_exe)) + target_output[is_exe] <- execute_experiments(race_state, experiments[is_exe], scenario) + + # If targetEvaluator is NULL, then target_output must contain the right + # output already. Otherwise, targetEvaluator considers all experiments. + if (!is.null(scenario$targetEvaluator)) { + target_output <- execute_evaluator(race_state$target_evaluator, experiments, scenario, target_output) + } else if (any(!is_exe)) { + experiments <- experiments[is_exe] + instance_idx <- instance_idx[is_exe] + } + target_output <- rbindlist(target_output, fill=TRUE, use.names=TRUE) + set(target_output, j = setdiff(colnames(target_output), c("cost", "time")), value = NULL) + if ("time" %notin% colnames(target_output)) + set(target_output, j = "time", value = NA) + set(target_output, j = "configuration", value = unlist_element(experiments, "id_configuration")) + set(target_output, j = "instance", value = instance_idx) + if (!is.null(bounds)) + set(target_output, j = "bound", value = unlist_element(experiments, "bound")) } + target_output +} - # targetEvaluator may be NULL. If so, target_output must contain the right - # output already. Otherwise, targetEvaluator always re-evaluates. - if (!is.null(scenario$targetEvaluator)) - target_output <- execute_evaluator(race_state$target_evaluator, experiments, scenario, target_output, - configurations_id = configurations[[".ID."]]) - +## Executes a list of configurations in a particular instance +## configurations: description having the id of the configuration +## instance.idx: index of the instance,seed pair in race_state$instances_log +## bounds: execution bounds (NULL if not needed). +## is_exe: Boolean vector that determines which experiments need to executed. +race_wrapper <- function(race_state, configurations, instance_idx, bounds, + is_exe, scenario) +{ + target_output <- race_wrapper_helper(race_state, configurations, instance_idx, + bounds, is_exe, scenario) + race_state$update_race_experiment_log(target_output, scenario) target_output } +experiments_output_to_matrix <- function(output, scenario) +{ + if (scenario$capping) + output[["cost"]] <- applyPAR(output[["cost"]], boundMax = scenario$boundMax, boundPar = scenario$boundPar) + as.matrix(dcast(output[, c("instance", "configuration", "cost")], instance ~ configuration, value.var = "cost"), + rownames = "instance") +} + aux2.friedman <- function(y, I, alive, conf.level = 0.95) { dropped.any <- FALSE n <- nrow(y) k <- length(I) - r <- rowRanks(y, cols = I, ties.method = "average") - R <- colSums2(r) + r <- rowRanks(y, cols = I, ties.method = "average", useNames = FALSE) + R <- colSums2(r, useNames = FALSE) o <- order(R) - best <- I[o[1]] + best <- I[o[1L]] TIES <- tapply(c(r), row(r), table) - STATISTIC <- ((12 * sum((R - n * (k + 1) / 2)^2)) / - (n * k * (k + 1) + STATISTIC <- ((12 * sum((R - n * (k + 1L) / 2)^2)) / + (n * k * (k + 1L) - (sum(unlist(lapply(TIES, function (u) {u^3 - u}))) / - (k - 1)))) - PARAMETER <- k - 1 + (k - 1L)))) + PARAMETER <- k - 1L PVAL <- pchisq(STATISTIC, PARAMETER, lower.tail = FALSE) #names(STATISTIC) <- "Friedman chi-squared" #names(PARAMETER) <- "df" @@ -119,11 +156,11 @@ aux2.friedman <- function(y, I, alive, conf.level = 0.95) # This formula for multiple comparisons comes from Conover, "Practical # Nonparametric Statistics", 1999, pages 369-371. A <- sum(as.vector(r)^2) - t <- qt(1 - alpha / 2, df = (n - 1) * (k - 1)) * - (2 * (n * A - sum(R^2)) / ((n - 1) * (k - 1)))^(1 / 2) + t <- qt(1 - alpha / 2, df = (n - 1L) * (k - 1L)) * + (2 * (n * A - sum(R^2)) / ((n - 1L) * (k - 1L)))^(1 / 2) J <- best - for (j in 2:k) { - if (abs(R[o[j]] - R[o[1]]) > t) { + for (j in 2L:k) { + if (abs(R[o[j]] - R[o[1L]]) > t) { break } else { J <- c(J, I[o[j]]) @@ -132,7 +169,7 @@ aux2.friedman <- function(y, I, alive, conf.level = 0.95) alive[-J] <- FALSE dropped.any <- TRUE } - irace.assert(I[which.min(R)] == best) + irace_assert(I[which.min(R)] == best) list(ranks = R, alive = alive, dropped.any = dropped.any, p.value = PVAL) } @@ -143,13 +180,13 @@ aux_friedman <- function(results, alive, which_alive, conf.level) # If more then 2 configurations are left, use Friedman return(aux2.friedman(results, which_alive, alive, conf.level = conf.level)) } - + ranks <- NULL dropped.any <- TRUE PVAL <- 0 # If only 2 configurations are left, switch to Wilcoxon - V1 <- results[, which_alive[1]] - V2 <- results[, which_alive[2]] + V1 <- results[, which_alive[1L]] + V2 <- results[, which_alive[2L]] diffs <- V1 - V2 # Avoid the test if the answer is obvious if (all(diffs <= 0)) { @@ -165,14 +202,14 @@ aux_friedman <- function(results, alive, which_alive, conf.level) diffs <- outer(diffs, diffs, "+") diffs <- sort(diffs[!lower.tri(diffs)])/2 PVAL <- wilcox.test(V1, V2, paired = TRUE, exact = if (ZEROES || TIES) FALSE else NULL)$p.value - irace.assert(!is.nan(PVAL) & !is.na(PVAL)) + irace_assert(!is.nan(PVAL) & !is.na(PVAL)) if (PVAL >= 1 - conf.level) dropped.any <- FALSE # We use the pseudo median (see wilcox.test.default) ranks <- if (median(diffs) <= 0) c(1L,2L) else c(2L,1L) } if (dropped.any) alive[which_alive[ranks[2L]]] <- FALSE - + list(ranks = ranks, alive = alive, dropped.any = dropped.any, p.value = PVAL) } @@ -180,7 +217,7 @@ aux.one_ttest <- function(results, alive, which_alive, conf.level, adjust = c("none","bonferroni","holm")) { adjust <- match.arg(adjust) - irace.assert(sum(alive) == length(which_alive)) + irace_assert(sum(alive) == length(which_alive)) results <- results[, which_alive] means <- colMeans2(results) best <- which.min(means) @@ -202,13 +239,13 @@ aux.one_ttest <- function(results, alive, which_alive, conf.level, # clearly dominating the other. if (all(results_best <= results_j)) next try(PVAL <- t.test(results_best, results_j, alternative = "less", paired = TRUE)$p.value) - irace.assert(!is.nan(PVAL) & !is.na(PVAL)) + irace_assert(!is.nan(PVAL) & !is.na(PVAL)) pvals[j] <- PVAL } pvals <- p.adjust(pvals, method = adjust) dropj <- which_alive[pvals < 1.0 - conf.level] dropped_any <- length(dropj) > 0 - irace.assert(all(alive[dropj])) + irace_assert(all(alive[dropj])) alive[dropj] <- FALSE list(ranks = means, alive = alive, dropped.any = dropped_any, p.value = min(pvals)) } @@ -217,15 +254,15 @@ aux.ttest <- function(results, alive, which_alive, conf.level, adjust = c("none","bonferroni","holm")) { adjust <- match.arg(adjust) - irace.assert(sum(alive) == length(which_alive)) - - results <- results[, which_alive] - means <- colMeans2(results) + irace_assert(sum(alive) == length(which_alive)) + + means <- colMeans2(results, cols = which_alive, useNames = FALSE) # FIXME: break ties using median or ranks? best <- which.min(means) mean_best <- means[best] pvals <- sapply(means, function(x) as.numeric(isTRUE( - all.equal.numeric(mean_best[[1]], x[[1]], check.attributes = FALSE)))) + all.equal.numeric(mean_best[[1L]], x[[1L]], check.attributes = FALSE)))) + results <- results[, which_alive] results_best <- results[, best] var_best <- var(results_best) which_test <- which(pvals < 1.0) @@ -241,43 +278,32 @@ aux.ttest <- function(results, alive, which_alive, conf.level, # clearly dominating the other. if (all(results_best <= results_j)) next try(PVAL <- t.test(results_best, results_j, paired = TRUE)$p.value) - irace.assert(!is.nan(PVAL) & !is.na(PVAL)) + irace_assert(!is.nan(PVAL) & !is.na(PVAL)) pvals[j] <- PVAL } pvals <- p.adjust(pvals, method = adjust) dropj <- which_alive[pvals < 1.0 - conf.level] - dropped_any <- length(dropj) > 0 - irace.assert(all(alive[dropj])) + dropped_any <- length(dropj) > 0L + irace_assert(all(alive[dropj])) alive[dropj] <- FALSE list(ranks = means, alive = alive, dropped.any = dropped_any, p.value = min(pvals)) } -table_hline <- function(widths) { - s <- "+" - for (w in widths) { - s <- paste0(s, strrep("-", w), "+") - } - paste0(s, "\n") -} +table_hline <- function(widths) + paste0(collapse="+", c("",strrep("-", widths), "\n")) + +table_sprintf <- function(text, widths) + paste0(collapse="|", c("", sprintf("%*s",widths, text), "\n")) -table_sprint <- function(text, widths) { - s <- "|" - for (i in seq_along(text)) { - s <- paste0(s, sprintf("%*s", widths[i], text[i]), "|") - } - paste0(s, "\n") -} .nocap_table_fields_width <- as.integer(c(1, 11, 11, 11, 16, 11, 8, 5, 4, 6)) .nocap_colum_names <- c(" ", "Instance", "Alive", "Best", "Mean best", "Exp so far", "W time", "rho", "KenW", "Qvar") .capping_table_fields_width <- as.integer(c(1, 11, 8, 11, 11, 16, 11, 8, 5, 4, 6)) .capping_colum_names <- c(" ", "Instance", "Bound", "Alive", "Best", "Mean best", "Exp so far", "W time", "rho", "KenW", "Qvar") -capping_hline <- table_hline(.capping_table_fields_width) -capping_header <- table_sprint(.capping_colum_names, .capping_table_fields_width) -nocap_hline <- table_hline(.nocap_table_fields_width) -nocap_header <- table_sprint(.nocap_colum_names, .nocap_table_fields_width) -race_common_header <- "# Markers: +.capping_hline <- table_hline(.capping_table_fields_width) +.nocap_hline <- table_hline(.nocap_table_fields_width) +.race_common_header <- "# Markers: x No test is performed. c Configurations are discarded only due to capping. - The test is performed and some configurations are discarded. @@ -285,41 +311,50 @@ race_common_header <- "# Markers: ! The test is performed and configurations could be discarded but elite configurations are preserved. . All alive configurations are elite and nothing is discarded.\n\n" +.nocap_header <- paste0(.race_common_header, .nocap_hline, + table_sprintf(.nocap_colum_names, .nocap_table_fields_width), .nocap_hline) +.capping_header <- paste0(.race_common_header, .capping_hline, + table_sprintf(.capping_colum_names, .capping_table_fields_width), .capping_hline) + race_print_header_nocap <- function() - cat(sep = "", race_common_header, nocap_hline, nocap_header, nocap_hline) + cat(sep = "", .nocap_header) race_print_header_cap <- function() - cat(sep = "", race_common_header, capping_hline, capping_header, capping_hline) + cat(sep = "", .capping_header) + +elapsed_wctime_str <- function(now, start) { + if (now <= start) return("00:00:00") + elapsed <- difftime(now, start, units = "secs") + # FIXME: Maybe better and faster if we only print seconds? + format(.POSIXct(elapsed, tz="GMT"), "%H:%M:%S") +} race_print_task <- function(res.symb, Results, instance, current_task, - alive, + which_alive, id_best, best, experiments_used, start_time, bound, capping) { - elapsed_wctime_str <- function(now, start) { - if (now <= start) return("00:00:00") - elapsed <- difftime(now, start, units = "secs") - # FIXME: Maybe better and faster if we only print seconds? - format(.POSIXct(elapsed, tz="GMT"), "%H:%M:%S") + cat(sprintf("|%s|%11d|", res.symb, instance)) + if (capping) { + if (is.null(bound)) + cat(" NA|") + else + cat(sprintf("%8.2f|", bound)) } # FIXME: This is the mean of the best, but perhaps it should # be the sum of ranks in the case of test == friedman? mean_best <- mean(Results[, best]) time_str <- elapsed_wctime_str(Sys.time(), start_time) - cat(sprintf("|%s|%11d|", res.symb, instance)) - if (capping) { - if (is.null(bound)) cat(" NA|") else cat(sprintf("%8.2f|", bound)) - } cat(sprintf(paste0("%11d|%11d|", .irace.format.perf, "|%11d|%s"), - sum(alive), id_best, mean_best, experiments_used, time_str)) - - if (current_task > 1L && sum(alive) > 1L) { - res <- Results[seq_len(current_task), alive, drop = FALSE] + length(which_alive), id_best, mean_best, experiments_used, time_str)) + + if (current_task > 1L && length(which_alive) > 1L) { + res <- Results[, which_alive, drop = FALSE] conc <- concordance(res) qvar <- dataVariance(res) # FIXME: We would like to use %+#4.2f but this causes problems with @@ -334,8 +369,8 @@ race_print_footer <- function(bestconf, mean_best, break_msg, debug_level, cappi old_best_id) { cat(sep = "", - if (capping) capping_hline else nocap_hline, - if (debug_level >= 1) paste0("# Stopped because ", break_msg, "\n"), + if (capping) .capping_hline else .nocap_hline, + if (debug_level >= 1L) paste0("# Stopped because ", break_msg, "\n"), if (!is.null(old_best_id)) paste0("Best configuration for the instances in this race: ", old_best_id, "\n"), sprintf("Best-so-far configuration: %11d", bestconf[[".ID."]][1L]), " mean value: ", @@ -345,7 +380,7 @@ race_print_footer <- function(bestconf, mean_best, break_msg, debug_level, cappi cat("\n") } -## This function calculates an execution bound +## This function calculates an execution bound ## data: matrix columns as configurations and rows instances ## type: ## median: bound based on the configuration's mean median @@ -354,16 +389,15 @@ race_print_footer <- function(bestconf, mean_best, break_msg, debug_level, cappi ## best: bound based on the best configurations's mean executionBound <- function(data, type = "median") { - irace.assert (ncol(data) >= 1L) - + irace_assert (ncol(data) >= 1L) + if (ncol(data) == 1L) { return (mean(data[,1L], na.rm = TRUE)) } # This should never happen because the data used to obtain the execution # bound should be complete, that is, the bounding configurations should have # been executed on all previous instances. - - irace.assert (all(!is.na(data))) + irace_assert (!anyNA(data)) colmeans <- colMeans2(data) bound <- switch (type, median = median(colmeans), @@ -381,9 +415,9 @@ executionBound <- function(data, type = "median") ## mean: bound based on the mean time ## worst: bound based on the worst time ## best: bound based on the best time -instanceBound <- function(data, type="median") +instanceBound <- function(data, type="median") { - irace.assert (all(!is.na(data))) + irace_assert (!anyNA(data)) switch (type, median = median(data), mean = mean(data), @@ -393,32 +427,32 @@ instanceBound <- function(data, type="median") } ## This function returns survivors obtained after applying the dominance elimination -## criterion. +## criterion. ## results: matrix of experiments results (all configurations) ## elites: index of elite configurations ## alive: bolean array of alive configurations -## eps: constant added to the bound to account for the measuring precision -dom.elim <- function(results, elites, alive, scenario, minSurvival, eps = 1e-5) +## eps: constant added to the bound to account for the measuring precision +dom_elim <- function(results, elites, alive, scenario, minSurvival, eps = 1e-5) { - irace.assert(sum(alive) >= minSurvival) which_alive <- which(alive) - cmeans <- colMeans2(results[, alive, drop = FALSE]) - irace.assert(!all(is.na(cmeans))) # Only NA values when calculating mean for dominance. - + irace_assert(length(which_alive) >= minSurvival) + cmeans <- colMeans2(results, cols = which_alive, useNames = FALSE) + irace_assert(!all(is.na(cmeans))) # Only NA values when calculating mean for dominance. + # When there are no protected elites left, select the best configurations to # calculate the bounds. This is quite aggressive and another alternative - # would be to disable dom.elim when elites == 0. - if (length(elites) == 0) { + # would be to disable dom_elim when elites == 0. + if (length(elites) == 0L) { # In the case we have only two alive configurations only one can be elite. - if (sum(alive) <= 2L) + if (length(which_alive) <= 2L) elites <- which_alive[which.min(cmeans)] else elites <- which_alive[order(cmeans, na.last = TRUE, decreasing = FALSE)[seq_len(minSurvival)]] } - + bound <- executionBound(results[, elites, drop = FALSE], type = scenario$cappingType) alive[which_alive] <- ((bound + eps) >= cmeans) - return (alive) + alive } ## This function applies PARX (X=boundPar) to all experiments @@ -428,64 +462,68 @@ applyPAR <- function(results, boundMax, boundPar) # We do not want to change Inf or -Inf because those represent rejection. if (boundPar != 1) results[is.finite(results) & results >= boundMax] <- boundMax * boundPar - return(results) + results } ## This function calculates the execution time allowed for the executions ## of configurations based on previous execution times. -## It returns a list with two elements: -## * elite.bound : value (mean execution time or per instance execution time) -## used to calculate the maximum execution time (final.bounds) for each configuration. -## * final.bounds[i] : maximum execution time for candidate i on the current instance. -final.execution.bound <- function(experimentsTime, elites, no.configurations, - current_task, which_exe, scenario) +## It returns a list with two elements: +## * elite_bound : value (mean execution time or per instance execution time) +## used to calculate the maximum execution time (final_bounds) for each configuration. +## * final_bounds[i] : maximum execution time for candidate i on the current instance. +final_execution_bound <- function(experimentsTime, elites, current_task, + which_alive, which_exe, scenario) { - minMeasurableTime <- scenario$minMeasurableTime boundMax <- scenario$boundMax - # FIXME: should we use an adjusted boundMax - final.bounds <- rep(boundMax, no.configurations) - total.time <- current_task * boundMax - elite.bound <- boundMax - # Elite candidates can have NA values due to the rejection + final_bounds <- rep(boundMax, length(which_alive)) + + # Elite candidates can have NA values due to the rejection. if (length(elites)) elites <- elites[!is.na(experimentsTime[current_task,elites])] - + # Only apply bounds when there is previous data - if (length(elites) > 0L && length(which_exe) > 0L) { - # The elite membership is updated before calling this function to know - # the configurations used for calculating the bounds we need to do this. - # Note that some of these configurations could be not elite anymore for the - # elimination phase, given that all their evaluations have been used. - if (scenario$boundType == "instance") { - elite.bound <- instanceBound(experimentsTime[current_task, elites], - type = scenario$cappingType) - final.bounds[which_exe] <- min(elite.bound + minMeasurableTime, boundMax) - final.bounds[which_exe] <- ceiling_digits(final.bounds[which_exe], scenario$boundDigits) - } else { - elite.bound <- executionBound(experimentsTime[seq_len(current_task), elites, drop = FALSE], - type = scenario$cappingType) - elite.bound <- min(elite.bound, boundMax) - total.time <- (current_task * elite.bound) + minMeasurableTime - time.left <- total.time - colSums2(experimentsTime, rows = seq_len(current_task), cols = which_exe, na.rm = TRUE) - final.bounds[which_exe] <- sapply(time.left, min, boundMax) - # We round up the bounds up to the specified number of digits. This may - # be necessary if the target-algorithm does not support higher precision. - final.bounds[which_exe] <- ceiling_digits(final.bounds[which_exe], scenario$boundDigits) - # There are cases in which a small negative budget is used. For example: - # assuming candidates 1 and 2 are elite maxbound=80 - # executionTime <- matrix(c(0.010,0.0170,0.010, 24,28,27, 0.010,0.017,NA), - # byrow=TRUE, ncol=3, nrow=3, dimnames=list(c(1,2,3),c(1,2,3))) - # current_task <- 3; boundDigits=0 - # elite.bound <- irace:::executionBound(executionTime[1:current_task,1:2], "median") - # total.time <- elite.bound * current_task + 0.01 - # time.left <- total.time - colSums2(executionTime[1:current_task,3,drop=FALSE], na.rm=TRUE) - # We set the execution time to the elite.bound this should be enough - # to eliminate a bad candidate for the next task. - final.bounds[final.bounds <= 0] <- elite.bound - irace.assert(all(final.bounds > 0)) - } + if (length(elites) == 0L || length(which_exe) == 0L) { + # FIXME: should we use an adjusted boundMax ? + return(list(final_bounds = final_bounds, elite_bound = boundMax)) + } + minMeasurableTime <- scenario$minMeasurableTime + + # The elite membership is updated before calling this function to know + # the configurations used for calculating the bounds we need to do this. + # Note that some of these configurations could be not elite anymore for the + # elimination phase, given that all their evaluations have been used. + if (scenario$boundType == "instance") { + elite_bound <- instanceBound(experimentsTime[current_task, elites], + type = scenario$cappingType) + final_bounds_exe <- min(elite_bound + minMeasurableTime, boundMax) + final_bounds_exe <- ceiling_digits(final_bounds_exe, scenario$boundDigits) + } else { + elite_bound <- executionBound(experimentsTime[seq_len(current_task), elites, drop = FALSE], + type = scenario$cappingType) + elite_bound <- min(elite_bound, boundMax) + total_time <- (current_task * elite_bound) + minMeasurableTime + final_bounds_exe <- total_time - colSums2(experimentsTime, rows = seq_len(current_task), cols = which_exe, na.rm = TRUE, useNames = FALSE) + # There are cases in which a small negative budget is used. For example: + # assuming candidates 1 and 2 are elite: + # maxBound <- 80 + # executionTime <- matrix(c(0.010,0.0170,0.010, 24,28,27, 0.010,0.017,NA), + # byrow=TRUE, ncol=3, nrow=3, dimnames=list(c(1,2,3),c(1,2,3))) + # current_task <- 3; boundDigits = 0 + # elite_bound <- irace:::executionBound(executionTime[1:current_task,1:2], "median") + # total_time <- elite_bound * current_task + 0.01 + # final_bounds_exe <- total_time - colSums2(executionTime[1:current_task,3,drop=FALSE], na.rm=TRUE) + # + # We set the execution time to the elite_bound, this should be enough + # to eliminate a bad candidate for the next task. + final_bounds_exe[final_bounds_exe <= 0] <- elite_bound + # We round up the bounds up to the specified number of digits. This may + # be necessary if the target-algorithm does not support higher precision. + final_bounds_exe <- ceiling_digits(final_bounds_exe, scenario$boundDigits) + final_bounds_exe[final_bounds_exe > boundMax] <- boundMax + irace_assert(all(final_bounds_exe > 0)) } - list(final.bounds = final.bounds, elite.bound = elite.bound) + final_bounds[which(which_alive %in% which_exe)] <- final_bounds_exe + list(final_bounds = final_bounds, elite_bound = elite_bound) } get_ranks <- function(x, test) @@ -497,106 +535,93 @@ get_ranks <- function(x, test) overall_ranks <- function(x, test) { if (ncol(x) == 1L) return(1L) - + ninstances <- colSums2(!is.na(x)) - uniq.ninstances <- sort(unique(ninstances), decreasing = TRUE) - last.r <- 0L - ranks <- rep(Inf, ncol(x)) + uniq_ninstances <- sort(unique(ninstances), decreasing = TRUE) + last_r <- 0L + ranks <- rep_len(Inf, ncol(x)) # Iterate from the largest to the lowest number of instances. - for (k in uniq.ninstances) { + for (k in uniq_ninstances) { confs <- which(ninstances == k) - irace.assert(all(is.infinite(ranks[confs]))) + irace_assert(all(is.infinite(ranks[confs]))) r <- 1L if (length(confs) > 1L) { # Select only non-NA rows y <- x[, confs, drop = FALSE] y <- y[complete.cases(y), , drop = FALSE] - irace.assert(!any(is.na(y))) r <- get_ranks(y, test = test) } - r <- r + last.r - last.r <- max(r) + r <- r + last_r + last_r <- max(r) ranks[confs] <- r } ranks } # Remove one elite count from every configuration not executed. -update_is_elite <- function(is.elite, which_exe) +update_is_elite <- function(is_elite, which_exe) { - which.notexecuted <- setdiff(which(is.elite > 0L), which_exe) - is.elite[which.notexecuted] <- is.elite[which.notexecuted] - 1L - irace.assert (all(is.elite >= 0L)) - is.elite + which_notexecuted <- setdiff(which(is_elite > 0L), which_exe) + is_elite[which_notexecuted] <- is_elite[which_notexecuted] - 1L + irace_assert (all(is_elite >= 0L)) + is_elite } -update_elite_safe <- function(Results, is.elite) +update_elite_safe <- function(Results, is_elite) { - elites <- is.elite > 0L + elites <- is_elite > 0L if (!any(elites)) return(0L) # All elites rejected. - max(which(rowAnys(!is.na(Results[, elites, drop=FALSE])))) + max(which(rowAnyNotNAs(Results, cols = elites))) } generateTimeMatrix <- function(elite_ids, experiment_log) { - is_elite <- experiment_log[["configuration"]] %in% elite_ids + # Silence CRAN warnings + configuration <- bound <- NULL # Remove everything that we don't need. - experiment_log <- experiment_log[is_elite, c("configuration", "instance", "time", "bound")] - experiment_log[["time"]] <- pmin(experiment_log[["time"]], experiment_log[["bound"]]) - # FIXME: It would be better to use spread() from tidyr - resultsTime <- reshape(as.data.frame(experiment_log), direction = "wide", - idvar = "instance", timevar = "configuration", - drop = "bound") - rownames(resultsTime) <- resultsTime$instance - resultsTime <- resultsTime[order(resultsTime$instance), , drop = FALSE] - colnames(resultsTime) <- substring(colnames(resultsTime), nchar("time.") + 1L) - as.matrix(resultsTime[, as.character(elite_ids), drop = FALSE]) + experiment_log <- experiment_log[(configuration %in% elite_ids) & !is.na(time), + c("configuration", "instance", "time", "bound")] + experiment_log[, time := pmin.int(time, bound)] + experiment_log[, bound := NULL] + time_matrix <- dcast(experiment_log, instance ~ configuration, value.var = "time") + setcolorder(time_matrix, neworder = as.character(elite_ids)) + as.matrix(time_matrix, rownames = "instance") } -race <- function(race_state, maxExp = 0L, +race <- function(race_state, maxExp, minSurvival = 1L, configurations, scenario) - elitist_race(race_state, maxExp = maxExp, + elitist_race(race_state = race_state, maxExp = maxExp, minSurvival = minSurvival, - elite.data = NULL, + elite_data = NULL, configurations = configurations, scenario = scenario, elitist_new_instances = 0L) -elitist_race <- function(race_state, maxExp = 0L, +elitist_race <- function(race_state, maxExp, minSurvival = 1L, - elite.data = NULL, + elite_data = NULL, configurations, scenario, - elitist_new_instances) + elitist_new_instances, + firstTest = scenario$firstTest) { blockSize <- scenario$blockSize - # FIXME: We should take this from scenario. However, this value should be - # zero for the first iteration. - ## FIXME2: Probably, instead of this, we should keep elite_safe in the race_state. - stat.test <- scenario$testType conf.level <- scenario$confidence - first.test <- blockSize * scenario$firstTest - each.test <- blockSize * scenario$eachTest + firstTest <- blockSize * firstTest + eachTest <- blockSize * scenario$eachTest elitist <- scenario$elitist capping <- scenario$capping - no.configurations <- nrow(configurations) - experiment_log <- data.table(instance=integer(0), configuration=integer(0), time=numeric(0), bound=numeric(0)) - alive <- rep(TRUE, no.configurations) - is.rejected <- rep(FALSE, no.configurations) - + n_configurations <- nrow(configurations) + alive <- rep_len(TRUE, n_configurations) + is_rejected <- logical(n_configurations) + ## FIXME: Remove argument checking. This must have been done by the caller. - # Check argument: maxExp - if (!missing(maxExp) - && (!is.numeric(maxExp) || length(maxExp)!=1L || !is.finite(maxExp))) - stop("maxExp must be an single number") - maxExp <- if (maxExp > 0) floor(maxExp) else 0L - if (maxExp && no.configurations > maxExp) - irace.error("Max number of experiments is smaller than number of configurations") - - if (no.configurations <= minSurvival) { - irace.error("Not enough configurations (", no.configurations, + irace_assert(maxExp > 0L) + + if (n_configurations <= minSurvival) { + irace_error("Not enough configurations (", n_configurations, ") for a race (minSurvival=", minSurvival, ")") } @@ -611,617 +636,606 @@ elitist_race <- function(race_state, maxExp = 0L, } else { print_header <- if (capping) race_print_header_cap else race_print_header_nocap print_task <- race_print_task - print_footer <- race_print_footer + print_footer <- race_print_footer } - + + stat_test <- scenario$testType + do_test <- switch(stat_test, + friedman = function(results, alive, which_alive, conf.level = scenario$confidence) + aux_friedman(results, alive, which_alive, conf.level = conf.level), + t.none = function(results, alive, which_alive, conf.level = scenario$confidence) + aux.ttest(results, alive, which_alive, conf.level = conf.level, adjust = "none"), + t.holm = function(results, alive, which_alive, conf.level = scenario$confidence) + aux.ttest(results, alive, which_alive, conf.level = conf.level, adjust = "holm"), + t.bonferroni = function(results, alive, which_alive, conf.level = scenario$confidence) + aux.ttest(results, alive, which_alive, conf.level = conf.level, adjust = "bonferroni")) + do_test <- bytecompile(do_test) + # Create the instance list according to the algorithm selected. if (elitist) { + # FIXME: we should sample taking into account the block-size, so we sample blocks, not instances. + irace_assert((race_state$next_instance - 1L) %% blockSize == 0, + eval_after={cat("next_instance:", race_state$next_instance, ", block_size:", blockSize, "\n")}) race_instances <- elitist_init_instances(race_state, deterministic = scenario$deterministic, - sampleInstances = scenario$sampleInstances, elitist_new_instances = elitist_new_instances, block_size = blockSize) + sampleInstances = scenario$sampleInstances, elitist_new_instances = elitist_new_instances) # It may be reduced further by elitist_init_instances() elitist_new_instances <- min(elitist_new_instances, race_state$elitist_new_instances) - } else + all_elite_instances_evaluated <- function() { + if (race_state$next_instance == 1L) return(TRUE) + evaluated <- !is.na(Results[, alive, drop=FALSE]) + # All instances that have been previously seen have been evaluated by at + # least one configuration + all(rowAnys(evaluated)) && + # and the number of instances evaluated per configuration is a multiple + # of eachTest (which is scenario$blockSize * scenario$eachTest). + all(colSums2(evaluated) %% eachTest == 0L) + } + } else { race_instances <- no_elitist_init_instances(race_state, deterministic = scenario$deterministic) - - irace.assert(!anyDuplicated(race_instances)) - irace.assert(identical(sort(race_instances), seq_along(race_instances))) - no.tasks <- length(race_instances) + all_elite_instances_evaluated <- function() TRUE + } + + irace_assert(!anyDuplicated(race_instances)) + irace_assert(identical(sort(race_instances), seq_along(race_instances))) + nb_tasks <- length(race_instances) # Initialize some variables... experiments_used <- 0L - # is.elite[i] : number of instances to be seen in this race on which i has - # been previously evaluated. - is.elite <- rep(0L, no.configurations) - if (is.null(elite.data)) { + ## FIXME: Probably, instead of this, we should keep elite_safe in the race_state. + if (is.null(elite_data)) { elite_safe <- 0L elite_instances_ID <- NULL } else { - irace.assert(race_state$next_instance - 1L == nrow(elite.data)) + irace_assert(race_state$next_instance - 1L == nrow(elite_data)) # There must be a non-NA entry for each instance. - irace.assert(all(rowAnys(!is.na(elite.data))), - eval_after = { print(elite.data)}) + irace_assert(all(rowAnyNotNAs(elite_data)), + eval_after = { print(elite_data)}) # There must be a non-NA entry for each configuration. - irace.assert(all(colAnys(!is.na(elite.data))), + irace_assert(all(colAnyNotNAs(elite_data)), eval_after = { - cat("elite.data:\n") - print(elite.data) + cat("elite_data:\n") + print(elite_data) cat("experiment_log:\n") print(race_state$experiment_log) }) - + # elite_safe: maximum instance number for which any configuration may be # considered elite. After evaluating this instance, no configuration is # elite. - elite_safe <- elitist_new_instances + nrow(elite.data) + elite_safe <- elitist_new_instances + nrow(elite_data) elite_instances_ID <- as.character(race_instances[seq_len(elite_safe)]) } configurations_ID <- as.character(configurations[[".ID."]]) - Results <- matrix(NA, + Results <- matrix(NA_real_, nrow = elite_safe, - ncol = no.configurations, + ncol = n_configurations, dimnames = list(elite_instances_ID, configurations_ID)) if (capping) - experimentsTime <- matrix(NA, + experimentsTime <- matrix(NA_real_, nrow = elite_safe, - ncol = no.configurations, + ncol = n_configurations, dimnames = list(elite_instances_ID, configurations_ID)) - if (! is.null(elite.data)) { - Results[rownames(elite.data), colnames(elite.data)] <- elite.data - + if (is.null(elite_data)) { + # is_elite[i] : number of instances to be seen in this race on which i has + # been previously evaluated. + is_elite <- integer(n_configurations) + } else { + Results[rownames(elite_data), colnames(elite_data)] <- elite_data + irace_assert(all(colnames(elite_data) %chin% configurations_ID)) if (capping) { - tmp <- generateTimeMatrix(elite_ids = colnames(elite.data), + # Temporarily use only 0 or 1, we will calculate the real value below. + is_elite <- as.integer(configurations_ID %chin% colnames(elite_data)) + tmp <- generateTimeMatrix(elite_ids = colnames(elite_data), experiment_log = race_state$experiment_log) experimentsTime[rownames(tmp), colnames(tmp)] <- tmp # Preliminary execution of elite configurations to calculate # the execution bound of initial configurations (capping only). if (elitist_new_instances > 0L) { - irace.assert(elitist_new_instances %% blockSize == 0L) + irace_assert(elitist_new_instances %% blockSize == 0L) # FIXME: This should go into its own function. - n_elite <- ncol(elite.data) - which_elites <- which(rep(TRUE, n_elite)) - irace.note("Preliminary execution of ", n_elite, + n_elite <- ncol(elite_data) + which_elites <- which(is_elite > 0L, useNames=FALSE) + irace_assert(identical(which_elites, seq_len(n_elite))) + irace_note("Preliminary execution of ", n_elite, " elite configuration(s) over ", elitist_new_instances, " instance(s).\n") + # FIXME: Launch all seq_len(elitist_new_instances) experiments in parallel. for (k in seq_len(elitist_new_instances)) { output <- race_wrapper (race_state, - configurations = configurations[which_elites, , drop = FALSE], + configurations = configurations[which_elites, , drop = FALSE], instance_idx = race_instances[k], bounds = rep(scenario$boundMax, n_elite), - which_alive = which_elites, - which_exe = which_elites, - scenario = scenario) - # Extract results - # FIXME: check what would happen in case of having the target evaluator - # MANUEL: Note how similar is this to what we do in do.experiments(), - # perhaps we can create a function that takes output and experiment_log - # and returns experiment_log. - # LESLIE: Yes you are right, Ill do it once we figure out the rest! - vcost <- unlist(lapply(output, "[[", "cost")) - irace.assert(length(vcost) == n_elite) - vcost <- applyPAR(vcost, boundMax = scenario$boundMax, boundPar = scenario$boundPar) - Results[k, seq_len(n_elite)] <- vcost - vtimes <- unlist(lapply(output, "[[", "time")) - irace.assert(length(vtimes) == n_elite) - experimentsTime[k, which_elites] <- vtimes # capping is enabled - experiment_log <- update_experiment_log(experiment_log, - instance=race_instances[k], - configuration = configurations[[".ID."]][which_elites], - time = vtimes, bound = scenario$boundMax) - experiments_used <- experiments_used + n_elite - + # which_exe values are within 1:nbConfigurations, whereas experiments + # indices are within 1:length(which_alive). The following line converts + # from one to the other. + is_exe = rep_len(TRUE, n_elite), scenario = scenario) + # Extract results: + irace_assert(length(output[["cost"]]) == n_elite) + Results[k, which_elites] <- applyPAR(output[["cost"]], boundMax = scenario$boundMax, boundPar = scenario$boundPar) + irace_assert(!anyNA(output[["time"]])) + experimentsTime[k, which_elites] <- output[["time"]] # capping is enabled + irace_assert(all.equal(configurations[[".ID."]][which_elites], output[["configuration"]])) + irace_assert(all.equal(output[["bound"]], rep(scenario$boundMax, n_elite))) + irace_assert(all.equal(unique(output[["instance"]]), race_instances[k])) + experiments_used <- experiments_used + n_elite + # We remove elite configurations that are rejected given that # is not possible to calculate the bounds. - rejected <- is.infinite(Results[k, which_elites]) - is.rejected[which_elites] <- rejected + rejected <- is.infinite(output[["cost"]]) + irace_assert(all.equal(as.vector(is.infinite(Results[k, which_elites])), rejected), eval_after={ + cat("rejected:\n") + print(output[["cost"]]) + print(rejected) + cat("Results:\n") + print(Results[k, which_elites]) + print(is.infinite(Results[k, which_elites])) + }) + is_rejected[which_elites] <- rejected + is_elite[rejected] <- 0L which_elites <- which_elites[!rejected] n_elite <- length(which_elites) - + # If all elite are eliminated we stop execution of initial instances. if (n_elite == 0L) { - irace.note ("All elite configurations are rejected. Execution of non-elites will be not bounded.\n") + irace_note ("All elite configurations are rejected. Execution of non-elites will be not bounded.\n") break } } - if (any(is.rejected)) { - irace.note ("Immediately rejected configurations: ", - paste0(configurations[is.rejected, ".ID."], - collapse = ", ") , "\n") - alive[is.rejected] <- FALSE + if (any(is_rejected)) { + irace_note ("Immediately rejected configurations: ", + paste0(configurations[[".ID."]][is_rejected], collapse = ", ") , "\n") + alive[is_rejected] <- FALSE # Calculate the maximum instance that has any non-NA value. # FIXME: Use update_elite_safe() if (n_elite > 0L) elite_safe <- max(which(rowAnys(!is.na(Results[, which_elites, drop=FALSE])))) else elite_safe <- 0L + irace_assert(identical(update_elite_safe(Results, is_elite), elite_safe)) + # FIXME: If sum(alive) <= minSurvival, we stop later but we will have + # elites that have not been evaluated in any instance, which is + # bad. Ideally we would sample new configurations here but that is + # too hard to do in iterated-racing. } } - } + } # end if(capping) # Compute the elite membership. - is.elite <- colSums2(!is.na(Results)) + is_elite <- colSums2(!is.na(Results)) # Remove rejected configurations. - is.elite[is.rejected] <- 0L + is_elite[is_rejected] <- 0L } - no_elimination <- 0L # number of tasks without elimination. - print_header() - if (elitist) - all_elite_instances_evaluated <- function() { - if (race_state$next_instance == 1L) return(TRUE) - evaluated <- !is.na(Results[, alive, drop=FALSE]) - # All instances that have been previously seen have been evaluated by at - # least one configuration - all(rowAnys(evaluated)) && - # and the number of instances evaluated per configuration is a multiple - # of blockSize. - all(colSums2(evaluated) %% blockSize == 0L) - } - else - all_elite_instances_evaluated <- function() TRUE - # Start main loop + no_elimination <- 0L # number of tasks without elimination. + print_header() + # Start main loop. break_msg <- NULL - best <- NA - for (current_task in seq_len(no.tasks)) { - which_alive <- which(alive) - nbAlive <- length(which_alive) - which_exe <- which_alive + best <- NA_integer_ + which_alive <- which(alive) + nb_alive <- length(which_alive) - if (elitist && any(is.elite > 0L)) { + for (current_task in seq_len(nb_tasks)) { + which_exe <- which_alive + if (elitist && any(is_elite > 0L)) { # Filter configurations that do not need to be executed (elites). # This is valid only for previous iteration instances. - irace.assert(current_task <= elite_safe) + irace_assert(current_task <= elite_safe) # Execute everything that is alive and not yet executed. which_exe <- which(alive & is.na(Results[current_task, ])) if (length(which_exe) == 0L) { - is.elite <- update_is_elite(is.elite, which_exe) + is_elite <- update_is_elite(is_elite, which_exe) # LESLIE: This is the case in which there are only elite configurations alive - # and we are still in the previous instances execution, but we can still + # and we are still in the previous instances execution, but we can still # continue with the race. (This is only possible because the early termination # criterion is disabled) ## MANUEL: So what is the reason to not immediately terminate here? Is ## there a reason to continue? if (current_task == 1L) { # We may reach this point in the first iteration so we need to calculate best. - if (sum(alive) == 1L) { + if (nb_alive == 1L) { best <- which_alive } else { - tmpResults <- Results[1, which_alive, drop = FALSE] - irace.assert(!any(is.na(tmpResults))) + tmpResults <- Results[1L, which_alive, drop = FALSE] + irace_assert(!anyNA(tmpResults)) # which.min returns only the first minimum. - best <- which_alive[which.min(get_ranks(tmpResults, test = stat.test))] + best <- which_alive[which.min(get_ranks(tmpResults, test = stat_test))] } } if (is.na(best)) { - dump.frames(dumpto = "best_crash", to.file = TRUE, + utils::dump.frames(dumpto = "best_crash", to.file = TRUE, include.GlobalEnv = TRUE) - irace.assert(!is.na(best)) + irace_assert(!is.na(best)) } id_best <- configurations[[".ID."]][best] print_task(".", Results[seq_len(current_task), , drop = FALSE], race_instances[current_task], - current_task, alive = alive, + current_task, which_alive = which_alive, id_best = id_best, best = best, experiments_used, start_time = Sys.time(), # FIXME: Why do we pass NA as bound? Why not pass the actual bound if any? - bound = NA, capping = capping) + bound = NA_real_, capping = capping) next } } - # We stop when we have less configurations than required. - if (nbAlive <= minSurvival && all_elite_instances_evaluated()) { - # Stop race if we have less or equal than the minimum number of - # configurations. - break_msg <- paste0("number of alive configurations (", nbAlive, - ") <= minimum number of configurations (", - minSurvival, ")") - break - } - # LESLIE: FIXME: Stopping deactivated by Thomas suggestion. Remove second - # condition to restore. - # LESLIE: Should we keep the early termination disabled? The difference between keeping it or - # not is that elite configurations could be eliminated later - # LESLIE: I think we should remove this - ## We continue running if (1) we have not reached the first.test or (2) - ## there are instances previously seen that have not been evaluated on any - ## alive configuration. - if (current_task > first.test) { - #if ((current_task > first.test && !capping) - # MANUEL: This is new and I'm not sure what it does. - # LESLIE: When using capping, we dont finish any race until all - # previous instances have been executed (this makes sure that all non-elite - # configurations execute all the previous instances) - # || (capping && (current_task > elite_safe))) { - # MANUEL: How is this even possible? - # LESLIE: It can be that the capping eliminate all but one configuration - # (which should be an elite one) after we finish the new instances to be evaluated, - # we allow the race to be finished. Maybe it wold be better: sum(is.elite) == sum(alive) - # instead of nbAlive == 1, - # LESLIE:Removing this because now is ponitless because of the elite candidates previos - # execution - # || (current_task > elitist_new_instances && nbAlive == 1)))) { - # If we just did a test, check that we have enough budget to reach the - # next test. - if (maxExp && ( (current_task - 1L) %% each.test) == 0L - && experiments_used + length(which_exe) * each.test > maxExp - && all_elite_instances_evaluated()) { + if (all_elite_instances_evaluated()) { + # We stop when we have less configurations than required. + if (nb_alive <= minSurvival) { + # Stop race if we have less or equal than the minimum number of + # configurations. + break_msg <- paste0("number of alive configurations (", nb_alive, + ") <= minimum number of configurations (", minSurvival, ")") + break + } + ## We continue running if (1) we have not reached the firstTest or (2) + ## there are instances previously seen that have not been evaluated on any + ## alive configuration. If we just did a test, check that we have enough + ## budget to reach the next test. + # FIXME: In post-selection racing, we want to consume all budget, so we + # should discard configurations until we have 2. + if (current_task > firstTest && ( (current_task - 1L) %% eachTest) == 0L + && experiments_used + length(which_exe) * eachTest > maxExp) { break_msg <- paste0("experiments for next test (", - experiments_used + length(which_exe) * each.test, - ") > max experiments (", maxExp, ")") + experiments_used + length(which_exe) * eachTest, + ") > max experiments (", maxExp, ")") break } - } - - if (elitist) { - if (scenario$elitistLimit != 0L && no_elimination >= scenario$elitistLimit - && all_elite_instances_evaluated()) { + + if (elitist && scenario$elitistLimit != 0L && no_elimination >= scenario$elitistLimit) { break_msg <- paste0("tests without elimination (", no_elimination, - ") >= elitistLimit (", scenario$elitistLimit, ")") + ") >= elitistLimit (", scenario$elitistLimit, ")") break } -## This is not needed anymore... -# else if (current_task > initial.tests && nbAlive <= minSurvival) { -# # We can stop the race ONLY when we pass the elite_safe -# # this is because how we are recovering the data from -# # previous runs (based on iteration). -# break_msg <- paste0("number of alive configurations (", nbAlive, -# ") less or equal than minimum number (", -# minSurvival, ")") -# break -# } } - - + if (nrow(Results) < current_task) { - Results <- rbind(Results, rep(NA, ncol(Results))) + Results <- rbind(Results, rep_len(NA_real_, ncol(Results))) rownames(Results) <- race_instances[seq_nrow(Results)] if (capping) { - experimentsTime <- rbind(experimentsTime, rep(NA, ncol(experimentsTime))) + experimentsTime <- rbind(experimentsTime, rep_len(NA_real_, ncol(experimentsTime))) rownames(experimentsTime) <- race_instances[seq_nrow(experimentsTime)] } } start_time <- Sys.time() - - # Execution bounds calculation (capping only) - final.bounds <- elite.bound <- NULL + # Calculate bounds for executing if needed. - which.elite.exe <- intersect(which_exe, which(is.elite > 0)) - irace.assert(setequal(which.elite.exe, which(is.elite & is.na(Results[current_task,])))) + which_elite_exe <- intersect(which_exe, which(is_elite > 0L)) + irace_assert(setequal(which_elite_exe, which(is_elite & is.na(Results[current_task,])))) if (capping) { # Pre-execute elite configurations that are not yet executed in the current instance. - if (length(which.elite.exe)) { + if (length(which_elite_exe)) { # FIXME: This should go into its own function output <- race_wrapper(race_state, - configurations = configurations[which.elite.exe, , drop = FALSE], - instance_idx = race_instances[current_task], - # FIXME: What if we already have a bound for this instance? - bounds = rep(scenario$boundMax, length(which.elite.exe)), - # MANUEL: How does this work for target-evaluator? - # We are telling race_wrapper that only some elites are alive! - which_alive = which.elite.exe, - which_exe = which.elite.exe, - scenario = scenario) + configurations = configurations[which_elite_exe, , drop = FALSE], + instance_idx = race_instances[current_task], + # FIXME: What if we already have a bound for this instance? + bounds = rep(scenario$boundMax, length(which_elite_exe)), + is_exe = rep_len(TRUE, length(which_elite_exe)), scenario = scenario) # Extract results - vcost <- unlist(lapply(output, "[[", "cost")) - irace.assert(length(vcost) == length(which.elite.exe)) - vcost <- applyPAR(vcost, boundMax = scenario$boundMax, boundPar = scenario$boundPar) - Results[current_task, which.elite.exe] <- vcost - vtimes <- unlist(lapply(output, "[[", "time")) - irace.assert(length(vtimes) == length(which.elite.exe)) - experimentsTime[current_task, which.elite.exe] <- vtimes - experiment_log <- update_experiment_log(experiment_log, - instance = race_instances[current_task], - configuration = configurations[which.elite.exe, ".ID."], - time = vtimes, bound = scenario$boundMax) - experiments_used <- experiments_used + length(which.elite.exe) - + irace_assert(length(output[["cost"]]) == length(which_elite_exe)) + Results[current_task, which_elite_exe] <- applyPAR(output[["cost"]], boundMax = scenario$boundMax, boundPar = scenario$boundPar) + irace_assert(!anyNA(output[["time"]])) + irace_assert(all.equal(configurations[which_elite_exe, ".ID."], output[["configuration"]])) + experimentsTime[current_task, which_elite_exe] <- output[["time"]] + irace_assert(all.equal(unique(output[["instance"]]), race_instances[current_task])) + experiments_used <- experiments_used + length(which_elite_exe) + # We remove elite configurations that are rejected given that # is not possible to calculate the bounds - rejected <- is.infinite(Results[current_task, which.elite.exe]) + rejected <- is.infinite(output[["cost"]]) if (any(rejected)) { - irace.note ("Immediately rejected configurations: ", - paste0(configurations[which.elite.exe[rejected], ".ID."], - collapse = ", ") , "\n") - is.rejected[which.elite.exe] <- rejected - is.elite[is.rejected] <- 0L - alive[which.elite.exe] <- !rejected + irace_note("Immediately rejected configurations: ", + paste0(configurations[[".ID."]][which_elite_exe[rejected]], + collapse = ", ") , "\n") + is_rejected[which_elite_exe] <- rejected + is_elite[is_rejected] <- 0L + alive[which_elite_exe] <- !rejected if (!any(alive)) { - # FIXME: Only report this error if (all(is.rejected)); otherwise restore non-rejected non-alive ones - irace.error("All configurations have been immediately rejected (all of them returned Inf) !") + # FIXME: Only report this error if (all(is_rejected)); otherwise + # restore non-rejected non-alive ones. Restoring a non-alive + # configuration is difficult. We need to evaluate it in all the + # instances that it has missed. + irace_error("All configurations have been immediately rejected (all of them returned Inf) !") } which_alive <- which(alive) - nbAlive <- length(which_alive) - elite_safe <- update_elite_safe(Results, is.elite) + nb_alive <- length(which_alive) + elite_safe <- update_elite_safe(Results, is_elite) } - which_exe <- setdiff(which_exe, which.elite.exe) + which_exe <- setdiff(which_exe, which_elite_exe) # FIXME: There is similar code above. Can we merge these code paths? if (length(which_exe) == 0L) { - is.elite <- update_is_elite(is.elite, which.elite.exe) + is_elite <- update_is_elite(is_elite, which_elite_exe) if (current_task == 1L) { # We may reach this point in the first iteration so we need to calculate best. - if (sum(alive) == 1L) { + if (nb_alive == 1L) { best <- which_alive } else { - tmpResults <- Results[1, which_alive, drop = FALSE] - irace.assert(!any(is.na(tmpResults))) + tmpResults <- Results[1L, which_alive, drop = FALSE] + irace_assert(!anyNA(tmpResults)) # which.min returns only the first minimum. - best <- which_alive[which.min(get_ranks(tmpResults, test = stat.test))] + best <- which_alive[which.min(get_ranks(tmpResults, test = stat_test))] } } if (is.na(best)) { - dump.frames(dumpto = "best_crash", to.file = TRUE, + utils::dump.frames(dumpto = "best_crash", to.file = TRUE, include.GlobalEnv = TRUE) - irace.assert(!is.na(best)) + irace_assert(!is.na(best)) } id_best <- configurations[[".ID."]][best] print_task(".", Results[seq_len(current_task), , drop = FALSE], race_instances[current_task], - current_task, alive = alive, + current_task, which_alive = which_alive, id_best = id_best, best = best, experiments_used, start_time = start_time, # FIXME: Why do we pass NA as bound? Why not pass the actual bound if any? - bound = if (is.null(scenario$boundMax)) NA else scenario$boundMax, capping) + bound = if (is.null(scenario$boundMax)) NA_real_ else scenario$boundMax, capping) next } } - - all.bounds <- final.execution.bound(experimentsTime, - elites = which(is.elite > 0), - no.configurations, current_task, + all_bounds <- final_execution_bound(experimentsTime, + elites = which(is_elite > 0L), + current_task, which_alive, which_exe, scenario) - final.bounds <- all.bounds$final.bounds - elite.bound <- all.bounds$elite.bound + final_bounds <- all_bounds$final_bounds + elite_bound <- all_bounds$elite_bound } else { - final.bounds <- rep(scenario$boundMax, no.configurations) + final_bounds <- rep(scenario$boundMax, length(which_alive)) + elite_bound <- NULL } - - # Execute experiments + + # Execute experiments. output <- race_wrapper(race_state, configurations = configurations[which_alive, , drop = FALSE], instance_idx = race_instances[current_task], - # FIXME: Why are we keeping final.bounds values for configurations that are dead? - # Also, do we use the final.bounds of which_alive or only the ones of which_exe? - bounds = final.bounds[which_alive], - which_alive = which_alive, which_exe = which_exe, + bounds = final_bounds, + is_exe = which_alive %in% which_exe, scenario = scenario) - # Extract results - vcost <- unlist(lapply(output, "[[", "cost")) - # If the experiment was executed or target.evaluator exists - # then the result is in the output. - ## Currently, targetEvaluator always re-evaluates, which implies that the - ## value may change without counting as an evaluation. We do this to allow online normalization. - which_exps <- if (is.null(scenario$targetEvaluator)) which_exe else which_alive - irace.assert(length(vcost) == length(which_exps)) # Set max execution bound to timed out executions which have execution - # times smaller than boundMax and implement parX if required + # times smaller than boundMax and implement parX if required. + vcost <- output[["cost"]] + # Output is not indexed in the same way as configurations. + which_has_time <- which(which_alive %in% which_exe) + # With !is.null(scenario$targetEvaluator) we will have duplicated (instance, configuration) in output. + irace_assert(all.equal(output[["bound"]], if (is.null(scenario$targetEvaluator)) final_bounds[which_has_time] + else final_bounds)) if (capping) { vcost <- applyPAR(vcost, boundMax = scenario$boundMax, boundPar = scenario$boundPar) - if (scenario$boundAsTimeout) - vcost[(vcost >= final.bounds[which_exps]) & (vcost < scenario$boundMax)] <- scenario$boundMax - } - Results[current_task, which_exps] <- vcost + if (scenario$boundAsTimeout) { + timeout_bounds <- if (is.null(scenario$targetEvaluator)) final_bounds[which_has_time] + else final_bounds + irace_assert(all.equal(output[["bound"]], timeout_bounds)) + # We do not want to change Inf or -Inf because those represent rejection. + vcost[is.finite(vcost) & (vcost >= timeout_bounds) & (vcost < scenario$boundMax)] <- scenario$boundMax + } - # Output is not indexed in the same way as configurations. - which_exps <- which(which_alive %in% which_exe) - irace.assert(length(which_exps) == length(which_exe)) - vtimes <- unlist(lapply(output[which_exps], "[[", "time")) - irace.assert(length(vtimes) == length(which_exe)) - if (capping) { + # If targetEvaluator was used, we do not update the times because no + # evaluation actually happened, only the cost values possibly changed. + vtimes <- if (is.null(scenario$targetEvaluator)) output[["time"]] else output[["time"]][which_has_time] # Correct higher execution times. - # final.bounds indexes are 1:nbConfigurations, vtimes are 1:length(which_alive) - experimentsTime[current_task, which_exps] <- pmin(vtimes, final.bounds[which_exe]) + irace_assert(all.equal(if (is.null(scenario$targetEvaluator)) output[["bound"]] else output[["bound"]][which_has_time], final_bounds[which_has_time])) + experimentsTime[current_task, which_has_time] <- pmin(vtimes, final_bounds[which_has_time]) } - experiment_log <- update_experiment_log(experiment_log, - instance = race_instances[current_task], - configuration = configurations[which_exe, ".ID."], - time = vtimes, bound = if (is.null(final.bounds)) NA else final.bounds[which_exe]) - - irace.assert(anyDuplicated(experiment_log[, c("instance", "configuration")]) == 0L, - eval_after = { - print(experiment_log) - print(mget(ls())) - }) + ## Currently, targetEvaluator always re-evaluates, which implies that the + ## value may change without counting as an evaluation. We do this to allow online normalization. + which_has_cost <- if (is.null(scenario$targetEvaluator)) which_exe else which_alive + irace_assert(length(output[["cost"]]) == length(which_has_cost)) + Results[current_task, which_has_cost] <- vcost experiments_used <- experiments_used + length(which_exe) + # We update the elites that have been executed. - is.elite <- update_is_elite(is.elite, which.elite.exe) + is_elite <- update_is_elite(is_elite, which_elite_exe) ## Drop bad configurations. ## Infinite values denote immediate rejection of a configuration. + # FIXME: Should this be which_has_cost? rejected <- is.infinite(Results[current_task, which_exe]) if (any(rejected)) { - irace.note ("Immediately rejected configurations: ", + irace_note ("Immediately rejected configurations: ", paste0(configurations[which_exe[rejected], ".ID."], collapse = ", ") , "\n") - is.rejected[which_exe] <- rejected - is.elite[is.rejected] <- 0L - alive[is.rejected] <- FALSE - if (!any(alive)) - irace.error("All configurations have been immediately rejected (all of them returned Inf) !") + is_rejected[which_exe] <- rejected + is_elite[is_rejected] <- 0L + alive[is_rejected] <- FALSE + if (!any(alive)) { + # FIXME: Only report this error if (all(is_rejected)); otherwise + # restore non-rejected non-alive ones. Restoring a non-alive + # configuration is difficult. We need to evaluate it in all the + # instances that it has missed. + irace_error("All configurations have been immediately rejected (all of them returned Inf) !") + } which_alive <- which(alive) - nbAlive <- length(which_alive) + nb_alive <- length(which_alive) # FIXME: Should we stop if (nbAlive <= minSurvival) ??? - elite_safe <- update_elite_safe(Results, is.elite) + elite_safe <- update_elite_safe(Results, is_elite) } - irace.assert(!anyNA(Results[seq_len(current_task), alive, drop=FALSE])) - irace.assert(!any(is.infinite(Results[, alive, drop=FALSE]))) - + irace_assert(!anyNA(Results[seq_len(current_task), alive, drop=FALSE])) + irace_assert(!any(is.infinite(Results[, alive, drop=FALSE]))) + # Variables required to produce output of elimination test. - cap.done <- FALSE #if dominance elimination was performed - test.done <- FALSE #if statistical test elimination was performed - cap.dropped <- FALSE #if capping has drop any configuration - test.dropped <- FALSE #if any candidates has been eliminated by testing - cap.alive <- test.alive <- alive - + cap_done <- FALSE # if dominance elimination was performed + test.done <- FALSE # if statistical test elimination was performed + cap_dropped <- FALSE # if capping has drop any configuration + test.dropped <- FALSE # if any candidates has been eliminated by testing + cap_alive <- test.alive <- alive + ## Dominance elimination (Capping only). # The second condition can be false if we eliminated via immediate # rejection. The third condition ensures that we see the block before capping. - if (capping && sum(alive) > minSurvival && (current_task %% blockSize) == 0L) { - irace.assert(!any(is.elite > 0) == (current_task >= elite_safe)) - cap.alive <- dom.elim(Results[seq_len(current_task), , drop = FALSE], - # Get current elite configurations - elites = which(is.elite > 0L), + if (capping && nb_alive > minSurvival && (current_task %% blockSize) == 0L + && (!scenario$cappingAfterFirstTest || current_task >= firstTest)) { + irace_assert(!any(is_elite > 0L) == (current_task >= elite_safe)) + cap_alive <- dom_elim(Results[seq_len(current_task), , drop = FALSE], + # Get current elite configurations. + elites = which(is_elite > 0L), alive, scenario, minSurvival) - cap.dropped <- sum(alive) > sum(cap.alive) - cap.done <- TRUE + cap_dropped <- nb_alive > sum(cap_alive) + cap_done <- TRUE } - - # We assume that first.test is a multiple of each.test. In any + + # We assume that firstTest is a multiple of eachTest. In any # case, this will only do the first test after the first multiple - # of each.test that is larger than first.test. - if (current_task >= first.test && (current_task %% each.test) == 0L - && nbAlive > 1L) { - irace.assert(sum(alive) == nbAlive) - test.res <- - switch(stat.test, - friedman = aux_friedman(Results[seq_len(current_task), ], alive, which_alive, conf.level), - t.none = aux.ttest(Results[seq_len(current_task), ], alive, which_alive, conf.level, adjust = "none"), - t.holm = aux.ttest(Results[seq_len(current_task), ], alive, which_alive, conf.level, adjust = "holm"), - t.bonferroni = aux.ttest(Results[seq_len(current_task), ], alive, which_alive, conf.level, adjust = "bonferroni")) - - race.ranks <- test.res$ranks - test.alive <- test.res$alive - test.dropped <- sum(alive) > sum(test.alive) + # of eachTest that is larger than firstTest. + if (current_task >= firstTest && (current_task %% eachTest) == 0L + && nb_alive > 1L) { + irace_assert(sum(alive) == nb_alive) + test_res <- do_test(Results[seq_len(current_task), ], alive, which_alive) + # FIXME: This race_ranks is unused. We should check if it matches the one computed below. + race_ranks <- test_res$ranks + test.alive <- test_res$alive + test.dropped <- nb_alive > sum(test.alive) test.done <- TRUE } - + # Merge the result of both eliminations. - prev.sum.alive <- sum(alive) - alive <- cap.alive & test.alive - + prev_nb_alive <- nb_alive + prev_which_alive <- which_alive + alive <- cap_alive & test.alive + # Handle elites when elimination is performed. The elite configurations # can be removed only when they have no more previously-executed instances. - irace.assert(!any(is.elite > 0L) == (current_task >= elite_safe)) - if (!is.null(elite.data) && any(is.elite > 0L)) { - irace.assert (length(alive) == length(is.elite)) - alive <- alive | (is.elite > 0L) + irace_assert(!any(is_elite > 0L) == (current_task >= elite_safe)) + if (!is.null(elite_data) && any(is_elite > 0L)) { + irace_assert (length(alive) == length(is_elite)) + alive <- alive | (is_elite > 0L) } # It may happen that the capping and the test eliminate together all # configurations. In that case, we only trust the capping elimination. if (capping && !any(alive)) { if (scenario$debugLevel >= 2L) { - irace.warning("Elimination tests have eliminated all configurations, keeping the capping results.\n") - irace.note("Alive according to capping:", which(cap.alive), "\n") - irace.note("Alive according to test:", which(test.alive), "\n") + irace_warning("Elimination tests have eliminated all configurations, keeping the capping results.\n") + irace_note("Alive according to capping:", which(cap_alive), "\n") + irace_note("Alive according to test:", which(test.alive), "\n") } - alive <- cap.alive + alive <- cap_alive } - - # Output the result of the elimination test - res.symb <- if (cap.dropped && !test.dropped && prev.sum.alive != sum(alive)) { - "c" # Removed just by capping - } else if (cap.dropped || test.dropped) { - if (prev.sum.alive != sum(alive)) "-" else "!" - } else if (cap.done || test.done) "=" else "x" - + which_alive <- which(alive) + nb_alive <- length(which_alive) + # Output the result of the elimination test. + res.symb <- if (cap_dropped && !test.dropped && prev_nb_alive != nb_alive) { + "c" # Removed just by capping. + } else if (cap_dropped || test.dropped) { + if (prev_nb_alive != nb_alive) "-" else "!" + } else if (cap_done || test.done) "=" else "x" + # Rank alive configurations: order all configurations (eliminated or not) # LESLIE: we have to make the ranking outside: we can have configurations eliminated by capping # that are not eliminated by the test. # MANUEL: I don't understand the above comment. - if (length(which_alive) == 1L) { - race.ranks <- 1L - best <- which_alive + if (length(prev_which_alive) == 1L) { + race_ranks <- 1L + best <- prev_which_alive } else { - tmpResults <- Results[seq_len(current_task), which_alive, drop = FALSE] - irace.assert(!any(is.na(tmpResults))) - race.ranks <- get_ranks(tmpResults, test = stat.test) - # which.min returns only the first minimum. - best <- which_alive[which.min(race.ranks)] + tmpResults <- Results[seq_len(current_task), prev_which_alive, drop = FALSE] + irace_assert(!anyNA(tmpResults)) + race_ranks <- get_ranks(tmpResults, test = stat_test) + # which.min() returns only the first minimum. + best <- prev_which_alive[which.min(race_ranks)] } - - irace.assert(best == which_alive[order(race.ranks)][1L]) - irace.assert(length(race.ranks) == length(which_alive)) - prev_alive <- which_alive - which_alive <- which(alive) + irace_assert(best == prev_which_alive[order(race_ranks)][1L]) + irace_assert(length(race_ranks) == length(prev_which_alive)) + # Remove the ranks of those that are not alive anymore - race.ranks <- race.ranks[which_alive] - irace.assert(length(race.ranks) == sum(alive)) + race_ranks <- race_ranks[which_alive] + irace_assert(length(race_ranks) == nb_alive) id_best <- configurations[[".ID."]][best] print_task(res.symb, Results[seq_len(current_task), , drop = FALSE], race_instances[current_task], - current_task, alive = alive, - id_best = id_best, best = best, experiments_used, start_time = start_time, - bound = elite.bound, capping) - + current_task, which_alive = which_alive, + id_best = id_best, best = best, experiments_used, start_time = start_time, + bound = elite_bound, capping) + if (elitist) { # Compute number of statistical tests without eliminations. - irace.assert(!any(is.elite > 0) == (current_task >= elite_safe)) - if (!any(is.elite > 0) - && current_task > first.test && (current_task %% each.test) == 0L) { - if (length(which_alive) == length(prev_alive)) { - no_elimination <- no_elimination + 1L - } else { - no_elimination <- 0L - } + irace_assert(!any(is_elite > 0L) == (current_task >= elite_safe)) + if (!any(is_elite > 0L) + && current_task > firstTest && (current_task %% eachTest) == 0L) { + no_elimination <- if (nb_alive == prev_nb_alive) no_elimination + 1L else 0L } - } + } } - + if (is.null(break_msg)) - break_msg <- paste0("all instances (", no.tasks, ") evaluated") + break_msg <- paste0("all instances (", nb_tasks, ") evaluated") # Adding this given that when ncandidates = minsurvival+1 # and there one elite configuration that gets discarded in the new instances # execution the race is finished with no executions. - # FIXME: we should handle this better, maybe allowing irace to handle no elite + # FIXME: we should handle this better, maybe allowing irace to handle no elite # in irace() # MANUEL: Leslie, how can we reach this error in normal circumstances? # Can we handle this better? - if (current_task == 1L && all(is.elite == 0L)) - irace.error ("Maximum number configurations immediately rejected reached!") - + if (current_task == 1L && all(is_elite == 0L)) + irace_error ("Maximum number configurations immediately rejected reached!") + # All instances that are not new in this race must have been evaluated by at # least one configuration. - irace.assert(all_elite_instances_evaluated(), + irace_assert(all_elite_instances_evaluated(), eval_after = { print(Results[,alive, drop=FALSE])}) # If we stop the loop before we see all new instances, there may be new # instances that have not been executed by any configuration. - Results <- Results[rowAnys(!is.na(Results)), , drop = FALSE] - race.ranks <- overall_ranks(Results[, alive, drop = FALSE], test = stat.test) + Results <- Results[rowAnyNotNAs(Results), , drop = FALSE] + # If we reject configurations so that sum(alive) <= minSurvival, we may stop + # before we evaluate some configurations in any instance. + if (any(is_rejected)) { + alive <- alive & colAnyNotNAs(Results) + which_alive <- which(alive) + if (!any(alive)) { + # FIXME: Only report this error if (all(is_rejected)); otherwise + # restore non-rejected non-alive ones. Restoring a non-alive + # configuration is difficult. We need to evaluate it in all the + # instances that it has missed. + irace_error("All configurations have been immediately rejected (all of them returned Inf) !") + } + } + + race_ranks <- overall_ranks(Results[, alive, drop = FALSE], test = stat_test) if (!scenario$quiet) { old_best <- best # old_best could be NA. - best <- which_alive[which.min(race.ranks)] + best <- which_alive[which.min(race_ranks)] mean_best <- mean(Results[, best]) print_footer(bestconf = configurations[best, , drop = FALSE], # FIXME: This is the mean of the best, but perhaps it - # should be the sum of ranks in the case of test == - # friedman? + # should be the sum of ranks in the case of test == friedman? mean_best = mean_best, - break_msg = break_msg, debug_level = scenario$debugLevel, + break_msg = break_msg, debug_level = scenario$debugLevel, capping = capping, old_best_id = if (old_best == best || is.na(old_best)) NULL else id_best) } - rejected_ids <- configurations[is.rejected, ".ID."] + rejected_ids <- configurations[is_rejected, ".ID."] scenario$parameters$forbid_configurations( race_state$update_rejected(rejected_ids, configurations) ) # Only return alive ones. - nbAlive <- sum(alive) - configurations <- configurations[as.logical(alive), , drop=FALSE] - irace.assert(nbAlive == nrow(configurations)) - irace.assert(all(configurations[[".ID."]] %not_in% rejected_ids)) # No rejected is alive. + configurations <- configurations[alive, , drop=FALSE] + irace_assert(all(configurations[[".ID."]] %not_in% rejected_ids)) # No rejected is alive. # Assign the proper ranks in the configurations data.frame. - configurations[[".RANK."]] <- race.ranks + configurations[[".RANK."]] <- race_ranks # Now we can sort the data.frame by the rank. configurations <- configurations[order(configurations[[".RANK."]]), , drop=FALSE] if (scenario$debugLevel >= 3L) { - irace.note ("Memory used in race():\n") + irace_note ("Memory used in race():\n") race_state$print_mem_used() } - # nrow(Results) may be smaller, equal or larger than current_task. - irace.assert(nrow(experiment_log) == experiments_used) + local_experiment_log <- race_state$reset_race_experiment_log() + # nrow(Results) may be smaller, equal or larger than current_task. + if (is.null(scenario$targetEvaluator)) { + # With targetEvaluator, we may have the recorded a new cost value but not + # counted it as an experiment used if targetRunner was not called. + irace_assert(anyDuplicated(local_experiment_log[, c("instance", "configuration")]) == 0L, + eval_after = { + print(local_experiment_log) + print(mget(ls())) + }) + irace_assert(nrow(local_experiment_log) == experiments_used) + } list(experiments = Results, - experiment_log = experiment_log, + experiment_log = local_experiment_log, experimentsUsed = experiments_used, configurations = configurations) } - -update_experiment_log <- function(experiment_log, instance, configuration, time, bound) - rbind(experiment_log, - data.table(instance = instance, configuration = configuration, time = time, bound = bound)) - diff --git a/R/race_state.R b/R/race_state.R index 44299876..d520f7fd 100644 --- a/R/race_state.R +++ b/R/race_state.R @@ -5,24 +5,27 @@ RaceState <- R6Class("RaceState", lock_class = TRUE, completed = "Incomplete", elapsed = 0L, elapsed_recovered = 0L, - elite_configurations = NULL, elitist_new_instances = 0L, experiment_log = NULL, instances_log = NULL, - model = NULL, + minSurvival = NULL, next_instance = -1L, + race_experiment_log = NULL, recovery_info = NULL, + recovery_mode = FALSE, rejected_ids = NULL, rng = NULL, seed = NULL, session_info = NULL, target_evaluator = NULL, target_runner = NULL, + timeUsed = 0, + time_next_save = 0, timer = NULL, # Methods. - initialize = function(scenario, new = TRUE) { + initialize = function(scenario, new = TRUE, recover = FALSE) { self$timer <- Timer$new() - self$target_runner <- if (is.function(scenario$targetRunner)) + self$target_runner <- if (is.function(scenario$targetRunner)) bytecompile(scenario$targetRunner) else if (scenario$aclib) target_runner_aclib @@ -34,89 +37,129 @@ RaceState <- R6Class("RaceState", lock_class = TRUE, else target_evaluator_default } - if (is.null(self$experiment_log)) { - self$experiment_log <- data.table(iteration=integer(0), instance=integer(0), - configuration=integer(0), time=numeric(0), bound=numeric(0)) - } - + irace_assert(new || !recover) if (new) { - seed <- scenario$seed - if (is.na(seed)) - seed <- trunc(runif(1, 1, .Machine$integer.max)) - set_random_seed(seed) - self$seed <- seed - self$rng <- get_random_seed() - self$elitist_new_instances <- round_to_next_multiple(scenario$elitistNewInstances, scenario$blockSize) - } else { + # elitist_new_instances must be a multiple of blockSize. + self$elitist_new_instances <- scenario$elitistNewInstances * scenario$blockSize + # We cannot recover if we did not get to initialize self$rng. + if (recover && !is.null(self$rng)) { + restore_random_seed(self$rng) + self$recovery_mode <- TRUE + set(self$experiment_log, j = "iteration", value = NULL) + self$recovery_info <- rbindlist(c(list(self$experiment_log), self$race_experiment_log), use.names = TRUE) + # Reinitialize some state. + self$completed = "Incomplete" + self$elapsed = 0L + self$elapsed_recovered = 0L + self$experiment_log = NULL + self$next_instance = -1L + self$race_experiment_log = NULL + self$rejected_ids = NULL + self$timeUsed = 0 + self$time_next_save = 0 + # Just in case anything is still running. + self$stop_parallel() + } else { + seed <- scenario$seed + if (is.na(seed)) + seed <- trunc(runif(1, 1, .Machine$integer.max)) + set_random_seed(seed) + self$seed <- seed + self$rng <- get_random_seed() + } + } else { # !new self$elapsed_recovered <- self$elapsed restore_random_seed(self$rng) } + + if (is.null(self$experiment_log)) { + self$experiment_log <- data.table(iteration=integer(0), instance=integer(0), + configuration=integer(0), cost = numeric(0), time = numeric(0), + bound = if (is.null(scenario$boundMax)) NULL else numeric(0)) + } + if (scenario$debugLevel >= 3L) { - irace.note("RNGkind: ", paste0(self$rng$rng_kind, collapse = " "), "\n", + irace_note("RNGkind: ", paste0(self$rng$rng_kind, collapse = " "), "\n", "# .Random.seed: ", paste0(self$rng$random_seed, collapse = ", "), "\n") } - # We do this here it is available even if we crash. - self$session_info <- sessionInfo() + # We do this here, so it is available even if we crash. + self$session_info <- utils::sessionInfo() invisible(self) }, - update_experiment_log = function(output, instances, configurations_id, scenario, iteration) { + update_experiment_log = function(output, instances, scenario) { + # FIXME: The instances parameter is not needed. + irace_assert(all.equal(rep(instances, each = length(unique(output[["configuration"]]))), output$instance)) # Extract results - times <- c() - costs <- c() - # FIXME: Convert output to a matrix so that we can skip this loop - for (j in instances) { - vcost <- unlist(lapply(output[[j]], "[[", "cost")) - costs <- c(costs, vcost) - vtimes <- unlist(lapply(output[[j]], "[[", "time")) - irace.assert(!any(is.null(vtimes))) - times <- c(times, vtimes) + self$experiment_log <- rbindlist(list(self$experiment_log, output), use.names=TRUE) + experiments_output_to_matrix(output, scenario) + }, + + save_recovery = function(iraceResults, logfile) { + now <- self$timer$wallclock() + # Do not save to disk too frequently. + if (now >= self$time_next_save) { + # irace_note("Saving recovery info.\n") + iraceResults$state <- self + save_irace_logfile(iraceResults, logfile) + self$time_next_save <- now + .irace_minimum_saving_time } - if (scenario$capping) - costs <- applyPAR(costs, boundMax = scenario$boundMax, boundPar = scenario$boundPar) - - self$experiment_log <- rbind(self$experiment_log, - data.table(iteration = iteration, instance = rep(instances, each = length(configurations_id)), - configuration = rep(configurations_id, times = length(instances)), - time = times, bound = if (is.null(scenario$boundMax)) NA else scenario$boundMax)) - - matrix(costs, nrow = length(instances), ncol = length(configurations_id), - byrow = TRUE, - dimnames = list(instances, as.character(configurations_id))) }, - save_recovery = function(elite_configurations, model, ...) { - self$time_elapsed() - self$rng <- get_random_seed() - self$elite_configurations <- elite_configurations - self$model <- model - self$recovery_info <- list(...) + update_race_experiment_log = function(experiment_log, scenario) { + self$race_experiment_log <- c(self$race_experiment_log, list(experiment_log)) + now <- self$timer$wallclock() + # Do not save to disk too frequently. + if (now >= self$time_next_save) { + # irace_note("Saving recovery info.\n") + iraceResults <- list( + scenario = scenario, + irace_version = irace_version, + state = self) + save_irace_logfile(iraceResults, logfile = scenario$logFile) + self$time_next_save <- now + .irace_minimum_saving_time + } + invisible() + }, + + reset_race_experiment_log = function() { + res <- rbindlist(self$race_experiment_log, use.names=TRUE) + self$race_experiment_log <- NULL + res + }, + + recover_output = function(instance_idx, configuration_id) { + search <- data.table(instance = instance_idx, configuration = configuration_id) + res <- self$recovery_info[search, on = .(instance,configuration), mult="first", nomatch=NULL, which=TRUE] + irace_assert(length(res) == 0L || length(res) == nrow(search)) + if (length(res) == 0L) { + irace_note("Cannot find the following in recovery info:") + print(search[!self$recovery_info, on = .(instance,configuration)]) + irace_error("Recovery terminated.") + } + # Get the rows. + output <- self$recovery_info[res] + # Delete those rows. + self$recovery_info <- self$recovery_info[-res] + if (nrow(self$recovery_info) == 0L) { + irace_note("Recovery completed.\n") + self$recovery_mode <- FALSE + self$recovery_info <- NULL + } + output }, update_rejected = function(rejected_ids, configurations) { if (length(rejected_ids) == 0L) return(NULL) - self$rejected_ids <- c(self$rejected_ids, rejected_ids) + self$rejected_ids <- c(self$rejected_ids, rejected_ids) configurations[configurations[[".ID."]] %in% rejected_ids, , drop = FALSE] }, - - recover = function(scenario) { - self$initialize(scenario, new = FALSE) - restore_random_seed(self$rng) - envir <- parent.frame() - # FIXME: This is a bit annoying, it would be better to keep these within RaceState all the time. - for (name in setdiff(names(formals(self$save_recovery)), "...")) - assign(name, self[[name]], envir = envir) - for (name in names(self$recovery_info)) - assign(name, self$recovery_info[[name]], envir = envir) - self$stop_parallel() - invisible(self) - }, time_elapsed = function() { self$elapsed <- self$timer$elapsed() + self$elapsed_recovered self$elapsed }, - + start_parallel = function(scenario) { parallel <- scenario$parallel data.table::setDTthreads(if (parallel <= 1L) 1L else min(4L, parallel)) @@ -135,16 +178,24 @@ RaceState <- R6Class("RaceState", lock_class = TRUE, # on Windows. We need to use the future package for that: # https://stackoverflow.com/questions/56501937/how-to-print-from-clusterapply self$cluster <- parallel::makeCluster(parallel) - if (scenario$debugLevel >= 1L) irace.note("makeCluster initialized for ", parallel, " jobs.") + if (scenario$debugLevel >= 1L) + irace_note("makeCluster initialized for ", parallel, " jobs.\n") # We export the global environment because the user may have defined # stuff there. There must be a better way to do this, but I cannot # figure it out. R sucks sometimes. parallel::clusterExport(self$cluster, ls(envir=.GlobalEnv)) # In Windows, this needs to be exported, or we get: - ## Error in checkForRemoteErrors(val) : - ## 2 nodes produced errors; first error: could not find function "target.runner" + ## Error in checkForRemoteErrors(val) : + ## 2 nodes produced errors; first error: could not find function "target_runner" parallel::clusterExport(self$cluster, list("target_runner"), envir=self) - # parallel::clusterExport(self$cluster, ls(environment(startParallel)), envir=environment(startParallel)) + if (is.function(scenario$targetRunner) + && !identical(environment(scenario$targetRunner), globalenv())) { + env_target_runner <- environment(scenario$targetRunner) + funglobs <- codetools::findGlobals(self$target_runner, merge=TRUE) + common <- intersect(funglobs, ls(envir=env_target_runner)) + if (length(common)) + parallel::clusterExport(self$cluster, common, envir=env_target_runner) + } } } invisible(self) @@ -160,12 +211,12 @@ RaceState <- R6Class("RaceState", lock_class = TRUE, print_mem_used = function(objects) { object_size_kb <- function (name, envir) - object.size(get(name, envir = envir)) / 1024 - + utils::object.size(get(name, envir = envir)) / 1024 + envir <- parent.frame() if (missing(objects)) objects <- ls(envir = envir, all.names = TRUE) - + x <- sapply(objects, object_size_kb, envir = envir) y <- sapply(names(get(class(self)[1L])$public_fields), object_size_kb, envir = self) names(y) <- paste0("RaceState$", names(y)) @@ -186,17 +237,17 @@ no_elitist_init_instances <- function(self, deterministic) # if next.instance == 1 then this is the first iteration. # If deterministic consider all (do not resample). if (self$next_instance == 1L || deterministic) return(seq_len(max_instances)) - irace.assert(self$next_instance < max_instances) + irace_assert(self$next_instance < max_instances) self$next_instance : max_instances } - -elitist_init_instances <- function(self, deterministic, sampleInstances, elitist_new_instances, block_size) + +elitist_init_instances <- function(self, deterministic, sampleInstances, elitist_new_instances) { max_instances <- nrow(self$instances_log) # if next_instance == 1 then this is the first iteration. next_instance <- self$next_instance if (next_instance == 1L) return(seq_len(max_instances)) # Consider all - + new_instances <- NULL last_new <- next_instance - 1L + elitist_new_instances # Do we need to add new instances? @@ -204,7 +255,7 @@ elitist_init_instances <- function(self, deterministic, sampleInstances, elitist if (last_new > max_instances) { # This may happen if the scenario is deterministic and we would need # more instances than what we have. - irace.assert(deterministic) + irace_assert(deterministic) if (next_instance <= max_instances) { # Add all instances that we have not seen yet as new ones. last_new <- max_instances @@ -217,12 +268,9 @@ elitist_init_instances <- function(self, deterministic, sampleInstances, elitist new_instances <- next_instance : last_new } } - # FIXME: we should sample taking into account the block-size, so we sample blocks, not instances. - irace.assert((next_instance - 1L) %% block_size == 0, - eval_after={cat("next_instance:", next_instance, ", block_size:", block_size, "\n")}) - past_instances <- if (sampleInstances) - sample.int(next_instance - 1L) else seq_len(next_instance - 1L) - + past_instances <- if (sampleInstances) sample.int(next_instance - 1L) + else seq_len(next_instance - 1L) + # new_instances + past_instances + future_instances if (last_new + 1L <= max_instances) { future_instances <- (last_new + 1L) : max_instances diff --git a/R/readConfiguration.R b/R/readConfiguration.R index 908b09ec..b7b32d35 100644 --- a/R/readConfiguration.R +++ b/R/readConfiguration.R @@ -1,18 +1,18 @@ #' Read parameter configurations from a file -#' +#' #' Reads a set of target-algorithm configurations from a file and puts them in #' \pkg{irace} format. The configurations are checked to match the parameters #' description provided. -#' +#' #' @param filename `character(1)`\cr Filename from which the configurations should be read. #' The contents should be readable by `read.table( , header=TRUE)`. #' @param text `character(1)`\cr If \code{file} is not supplied and this is, #' then configurations are read from the value of \code{text} via a text connection. #' @inheritParams readParameters #' @inheritParams printParameters -#' -#' @return A data frame containing the obtained configurations. -#' Each row of the data frame is a candidate configuration, +#' +#' @return A data frame containing the obtained configurations. +#' Each row of the data frame is a candidate configuration, #' the columns correspond to the parameter names in `parameters`. #' #' @details @@ -26,29 +26,29 @@ #' ``` #' The order of the columns does not necessarily have to be the same #' as in the file containing the definition of the parameters. -#' -#' @seealso +#' +#' @seealso #' [readParameters()] to obtain a valid parameter structure from a parameters file. -#' +#' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export readConfigurationsFile <- function(filename, parameters, debugLevel = 0L, text) { if (missing(filename) && !missing(text)) { filename <- NULL - configurationTable <- read.table(text = text, header = TRUE, + configurationTable <- utils::read.table(text = text, header = TRUE, na.strings = c("NA", ""), colClasses = "character", stringsAsFactors = FALSE) } else { # Read the file. - configurationTable <- read.table(filename, header = TRUE, + configurationTable <- utils::read.table(filename, header = TRUE, na.strings = c("NA", ""), colClasses = "character", stringsAsFactors = FALSE) } - irace.assert(is.data.frame(configurationTable)) - irace.note("Read ", nrow(configurationTable), " configuration(s)", + irace_assert(is.data.frame(configurationTable)) + irace_note("Read ", nrow(configurationTable), " configuration(s)", if (is.null(filename)) "\n" else paste0(" from file '", filename, "'\n")) fix_configurations(configurationTable, parameters, debugLevel = debugLevel, filename = filename) @@ -57,11 +57,12 @@ readConfigurationsFile <- function(filename, parameters, debugLevel = 0L, text) fix_configurations <- function(configurations, parameters, debugLevel = 0L, filename = NULL) { conf_error <- function(k, ...) - irace.error("Configuration number ", k, + irace_error("Configuration number ", k, if (is.null(filename)) "" else paste0(" from file '", filename, "'"), ...) - - if (debugLevel >= 2L) print(configurations, digits=15L) + + if (debugLevel >= 2L) + print(configurations, digits=15L) nbConfigurations <- nrow(configurations) namesParameters <- parameters[["names"]] # This ignores fixed parameters unless they are given with a different value. @@ -71,10 +72,10 @@ fix_configurations <- function(configurations, parameters, debugLevel = 0L, file missing <- setdiff(colnames(configurations), namesParameters) if (length(missing) > 0L) { if (is.null(filename)) { - irace.error("The parameter names (", strlimit(paste(missing, collapse=", ")), + irace_error("The parameter names (", strlimit(paste(missing, collapse=", ")), ") do not match the parameter names: ", paste(namesParameters, collapse=", ")) } else { - irace.error("The parameter names (", + irace_error("The parameter names (", strlimit(paste(missing, collapse=", ")), ") given in the first row of file ", filename, " do not match the parameter names: ", @@ -87,11 +88,11 @@ fix_configurations <- function(configurations, parameters, debugLevel = 0L, file missing <- setdiff (varParameters, colnames(configurations)) if (length(missing) > 0) { if (is.null(filename)) { - irace.error("The parameter names (", + irace_error("The parameter names (", strlimit(paste(missing, collapse=", ")), ") are missing from the configurations provided.") } else { - irace.error("The parameter names (", + irace_error("The parameter names (", strlimit(paste(missing, collapse=", ")), ") are missing from the first row of file ", filename) } @@ -100,7 +101,7 @@ fix_configurations <- function(configurations, parameters, debugLevel = 0L, file # Add any missing fixed parameters. missing <- setdiff (namesParameters, colnames(configurations)) if (length(missing) > 0L) { - irace.assert (all(parameters$isFixed[missing])) + irace_assert (all(parameters$isFixed[missing])) configurations <- cbind.data.frame(configurations, parameters$domains[missing], stringsAsFactors = FALSE) } @@ -134,7 +135,7 @@ fix_configurations <- function(configurations, parameters, debugLevel = 0L, file suppressWarnings(as.numeric(configurations[[pname]])), digits = parameters$get(pname)[["digits"]]) } - # Loop over all configurations in configurations. + # Loop over all configurations. # FIXME: Vectorize this loop values <- configurations[[pname]] for (k in seq_len(nbConfigurations)) { @@ -193,10 +194,10 @@ fix_configurations <- function(configurations, parameters, debugLevel = 0L, file } } if (anyDuplicated(configurations)) { - irace.error("Duplicated configurations", + irace_error("Duplicated configurations", if (is.null(filename)) "" else paste0(" in file '", filename, "'"), ":\n", - paste0(capture.output( + paste0(utils::capture.output( configurations[duplicated(configurations), , drop=FALSE]), "\n")) } configurations @@ -206,12 +207,12 @@ compile_forbidden <- function(x) { if (is.null(x) || is.bytecode(x)) return(x) # If we are given an expression, it must be a single one. - irace.assert(is.language(x) && (!is.expression(x) || length(x) == 1L)) + irace_assert(is.language(x) && (!is.expression(x) || length(x) == 1L)) if (is.expression(x)) x <- x[[1L]] # When a is NA and we check a == 5, we would get NA, which is # always FALSE, when we actually want to be TRUE, so we test # is.na() first below. - + # We expect that there will be undefined variables, since the expressions # will be evaluated within a data.frame later. expr <- compiler::compile(substitute(is.na(x) | !(x), list(x = x)), diff --git a/R/readParameters.R b/R/readParameters.R index d0c94b03..5912d173 100644 --- a/R/readParameters.R +++ b/R/readParameters.R @@ -1,6 +1,6 @@ #' Reads the parameters to be tuned by \pkg{irace} from a file or from a #' character string. -#' +#' #' @param file `character(1)`\cr Filename containing the definitions of #' the parameters to be tuned. #' @param digits `integer(1)`\cr The number of decimal places to be considered for real-valued parameters. @@ -71,7 +71,7 @@ #' elitistants "--elitistants " i (1, ants) | algorithm == "eas" #' nnls "--nnls " i (5, 50) | localsearch %in% c(1,2,3) #' dlb "--dlb " c (0, 1) | localsearch %in% c(1,2,3) -#' +#' #' [forbidden] #' (alpha == 0.0) & (beta == 0.0) #' [global] @@ -79,24 +79,24 @@ #' ' #' parameters <- readParameters(text=parameters_table) #' str(parameters) -#' +#' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export readParameters <- function (file, digits = 4L, debugLevel = 0L, text) { if (missing(file) && !missing(text)) { - filename <- NA + filename <- NA_character_ file <- textConnection(text) on.exit(close(file)) } else if (is.character(file)) { filename <- file file.check (file, readable = TRUE, text = "readParameter: parameter file") } else { - irace.error("'file' must be a character string") + irace_error("'file' must be a character string") } digits <- as.integer(digits) - + field.match <- function (line, pattern, delimited = FALSE, sep = "[[:space:]]") { #cat ("pattern:", pattern, "\n") @@ -130,10 +130,10 @@ readParameters <- function (file, digits = 4L, debugLevel = 0L, text) v <- c() str <- trim(str) #cat("string2vector:", str, "\n") - while (nchar (str)) { + while (str != "") { result <- field.match (str, "\"[^\"]*\"", delimited = TRUE, sep="") #cat("result.match: ", result$match,"\n") - if (is.null (result$match)) { + if (is.null(result$match)) { result <- field.match (str, "[^,]+", sep="") #cat("result.match: ", result$match,"\n") } @@ -148,15 +148,15 @@ readParameters <- function (file, digits = 4L, debugLevel = 0L, text) errReadParameters <- function(filename, line, context, ...) { context <- if (is.null (context)) "" else paste0(" when reading: \"", context, "\"") - fileloc <- if (is.na(filename)) "" else paste0("'", filename, "'," ) - irace.error(paste0(..., collapse = ""), " at ", fileloc, "line ", line, context) + fileloc <- if (is.na(filename)) "" else paste0("'", filename, "'," ) + irace_error(paste0(..., collapse = ""), " at ", fileloc, "line ", line, context) } - + warnReadParameters <- function(filename, line, context, ...) { context <- if (is.null (context)) "" else paste0(" when reading: \"", context, "\"") - fileloc <- if (is.na(filename)) "" else paste0("'", filename, "'," ) - irace.warning(paste0(..., collapse = ""), " at ", fileloc, "line ", line, context) + fileloc <- if (is.na(filename)) "" else paste0("'", filename, "'," ) + irace_warning(paste0(..., collapse = ""), " at ", fileloc, "line ", line, context) } parse_condition <- function(s, filename, nbLines, line, context) { @@ -167,18 +167,18 @@ readParameters <- function (file, digits = 4L, debugLevel = 0L, text) } str2expression(s) } - + params <- list() pnames <- c() lines <- readLines(con = file) - # Delete comments + # Delete comments lines <- trim(sub("#.*$", "", lines)) within_global <- FALSE nbLines <- 0L # Parse [global] first. for (line in lines) { nbLines <- nbLines + 1L - if (nchar(line) == 0L) next + if (line == "") next if (grepl("^[[:space:]]*\\[forbidden\\]", line)) { if (within_global) break @@ -190,7 +190,7 @@ readParameters <- function (file, digits = 4L, debugLevel = 0L, text) if (!is.wholenumber(digits) || digits > 15 || digits < 1) errReadParameters(filename, nbLines, line, "'digits' must be an integer within [1, 15]") digits <- as.integer(digits) - } else + } else errReadParameters(filename, nbLines, line, "Unknown global option") lines[nbLines] <- "" # Do not parse it again. next @@ -206,8 +206,8 @@ readParameters <- function (file, digits = 4L, debugLevel = 0L, text) nbLines <- 0L for (line in lines) { nbLines <- nbLines + 1L - if (nchar(line) == 0L) next - + if (line == "") next + if (within_forbidden) { # FIXME: Better error reporting. exp <- parse_condition(line, filename, nbLines, line, " for forbidden expressions") @@ -231,7 +231,7 @@ readParameters <- function (file, digits = 4L, debugLevel = 0L, text) errReadParameters (filename, nbLines, NULL, "Duplicated parameter name '", name, "'") } - + ## Match p_switch (quoted string) result <- field.match (line, "\"[^\"]*\"", delimited = TRUE) p_label <- result$match @@ -240,7 +240,7 @@ readParameters <- function (file, digits = 4L, debugLevel = 0L, text) errReadParameters (filename, nbLines, line, "Parameter label (switch) must be a double-quoted string") } - + ## Match param.type (longer matches must precede shorter ones) result <- field.match (line, c("i,log", "r,log", "c", "i", "r", "o")) param.type <- result$match @@ -259,7 +259,7 @@ readParameters <- function (file, digits = 4L, debugLevel = 0L, text) } else { param.transform <- "" } - + ## Match domain (delimited by parenthesis) # Regexp to detect dependent domains of the type ("min(p1)", 100) result <- field.match (line, "\\([^|]+\\)", delimited = TRUE, sep = "") @@ -286,17 +286,17 @@ readParameters <- function (file, digits = 4L, debugLevel = 0L, text) } else { # type %in% c("c", "o") domain <- string2vector(domain_str) } - ## Match start of conditions + ## Match start of conditions result <- field.match (line, "\\|", sep="") line <- result$line - if (!is.null(result$match) && nchar(result$match)) { + if (!is.null(result$match) && result$match != "") { result <- field.match (line, ".*$", sep="") condition <- result$match - if (is.null(result$match) || !nchar(result$match)) + if (is.null(result$match) || result$match == "") errReadParameters (filename, nbLines, line, "Expected condition after '|'") line <- result$line - } else if (!is.null(result$line) && nchar(result$line)) { + } else if (!is.null(result$line) && result$line != "") { errReadParameters (filename, nbLines, line, "Expected '|' before condition") } else { @@ -310,7 +310,7 @@ readParameters <- function (file, digits = 4L, debugLevel = 0L, text) domain_str, ")"), class = "try-error"), invalid_range = function(c) structure(paste0("Lower bound must be smaller than upper bound in numeric range (", domain_str, ") for parameter '", name, "'"), class="try-error"), - + error = function(c) structure(conditionMessage(c), class="try-error") ) if (inherits(p, "try-error")) errReadParameters(filename, nbLines, NULL, p) @@ -321,19 +321,19 @@ readParameters <- function (file, digits = 4L, debugLevel = 0L, text) # Check that we have read at least one parameter if (length(params) == 0) { if (is.na(filename)) - irace.error("No parameter definition found in the input text") + irace_error("No parameter definition found in the input text") else - irace.error("No parameter definition found, check that the parameter file '", filename, "' is not empty.") + irace_error("No parameter definition found, check that the parameter file '", filename, "' is not empty.") } if (length(forbidden)) { - irace.note(length(forbidden), " expression(s) specifying forbidden configurations read.\n") + irace_note(length(forbidden), " expression(s) specifying forbidden configurations read.\n") check_forbidden_params(forbidden, pnames, filename = filename) } parameters <- do.call(parametersNew, c(params, list(forbidden=forbidden, debugLevel = debugLevel))) if (debugLevel >= 2) { print(parameters, digits = 15L) - irace.note("Parameters have been read\n") + irace_note("Parameters have been read\n") } parameters } @@ -341,7 +341,7 @@ readParameters <- function (file, digits = 4L, debugLevel = 0L, text) #' Read parameters in PCS (AClib) format and write them in irace format. #' #' @inheritParams readParameters -#' +#' #' @return A string representing the parameters in irace format. #' #' @details Either `file` or `text` must be given. If `file` is given, the @@ -356,7 +356,7 @@ readParameters <- function (file, digits = 4L, debugLevel = 0L, text) #' #' @references #' Frank Hutter, Manuel López-Ibáñez, Chris Fawcett, Marius Thomas Lindauer, Holger H. Hoos, Kevin Leyton-Brown, and Thomas Stützle. **AClib: A Benchmark Library for Algorithm Configuration**. In P. M. Pardalos, M. G. C. Resende, C. Vogiatzis, and J. L. Walteros, editors, _Learning and Intelligent Optimization, 8th International Conference, LION 8_, volume 8426 of Lecture Notes in Computer Science, pages 36–40. Springer, Heidelberg, 2014. -#' +#' #' @seealso [readParameters()] #' @examples #' ## Read the parameters directly from text @@ -372,19 +372,19 @@ readParameters <- function (file, digits = 4L, debugLevel = 0L, text) #' rasrank [1, 100][1]i #' elitistants [1, 750][1]i #' nnls [5, 50][5]i -#' dlb {0, 1}[1] +#' dlb {0, 1}[1] #' Conditionals: #' q0 | algorithm in {acs} #' rasrank | algorithm in {ras} #' elitistants | algorithm in {eas} #' nnls | localsearch in {1,2,3} -#' dlb | localsearch in {1,2,3} +#' dlb | localsearch in {1,2,3} #' {alpha=0, beta=0}' #' parameters_table <- read_pcs_file(text=pcs_table) #' cat(parameters_table) #' parameters <- readParameters(text=parameters_table) #' str(parameters) -#' +#' #' @author Manuel López-Ibáñez #' @export read_pcs_file <- function(file, digits = 4L, debugLevel = 0L, text) @@ -397,7 +397,7 @@ read_pcs_file <- function(file, digits = 4L, debugLevel = 0L, text) filename <- file file.check (file, readable = TRUE, text = "read_pcs_file: parameter file") } else { - irace.error("'file' must be a character string") + irace_error("'file' must be a character string") } lines <- readLines(con = file) lines <- trim(lines) # Remove leading and trailing whitespace @@ -411,27 +411,27 @@ read_pcs_file <- function(file, digits = 4L, debugLevel = 0L, text) matches <- regmatches(lines[k], regexec(regex_cond, lines[k], perl=TRUE))[[1L]] stopifnot(length(matches) > 0) - lines[k] <- NA + lines[k] <- NA_character_ conditions[[matches[[2L]]]] <- matches[[3L]] } else if (grepl(regex_forbidden, lines[k], perl=TRUE)) { forbidden <- c(forbidden, sub(regex_forbidden, "\\1", lines[k], perl=TRUE)) - lines[k] <- NA + lines[k] <- NA_character_ } } - + parse_pcs_condition <- function(x, types) { if (is.null(x)) return ("") matches <- regmatches(x, regexec("([^[:space:]]+)[[:space:]]+in[[:space:]]+\\{([^}]+)\\}$", x, perl=TRUE))[[1L]] - if (length(matches) == 0L) irace.error("unknown condition ", x) + if (length(matches) == 0L) irace_error("unknown condition ", x) param <- matches[[2L]] type <- types[[param]] - if (is.null(type)) irace.error("unknown type for ", param, " in condition: ", x) + if (is.null(type)) irace_error("unknown type for ", param, " in condition: ", x) cond <- matches[[3L]] if (type == "c" || type == "o") { cond <- strsplit(cond, ",[[:space:]]*")[[1L]] equal <- (length(cond) == 1L) cond <- paste0('"', cond, '"', collapse=',') - } else { + } else { equal <- grepl(",", cond, fixed=TRUE) } if (equal) @@ -448,7 +448,7 @@ read_pcs_file <- function(file, digits = 4L, debugLevel = 0L, text) matches <- regmatches(line, regexec("^([^[:space:]]+)[[:space:]]+\\[([^,]+),[[:space:]]*([^]]+)\\][[:space:]]*\\[[^]]+\\](i?l?i?)(.*)$", line, perl=TRUE))[[1]] if (length(matches) > 0L) { param_name <- matches[[2L]] - + param_type <- paste0(if(grepl("i", matches[5L], fixed=TRUE)) "i" else "r", if(grepl("l", matches[5L], fixed=TRUE)) ",log" else "") param_types[[param_name]] <- param_type @@ -483,7 +483,7 @@ read_pcs_file <- function(file, digits = 4L, debugLevel = 0L, text) param_name, param_name, param_types[[param_name]], param_domains[[param_name]], cond, param_comments[[param_name]])) next } - irace.error("unrecognized line: ", line) + irace_error("unrecognized line: ", line) } if (length(forbidden) > 0L) { exp <- sapply(forbidden, function(x) { @@ -516,8 +516,7 @@ checkParameters <- function(parameters) ## names(parameters$isDependent) <- parameters$names ## } if (!inherits(parameters, "ParameterSpace")) { - irace.error("parameters must be an object of class 'ParameterSpace'") + irace_error("parameters must be an object of class 'ParameterSpace'") } parameters } - diff --git a/R/scenario.R b/R/scenario.R index ad00750d..3bc6fb00 100644 --- a/R/scenario.R +++ b/R/scenario.R @@ -2,12 +2,12 @@ #' #' The scenario argument is an initial scenario that is overwritten for every #' setting specified in the file to be read. -#' +#' #' @param filename `character(1)`\cr Filename from which the scenario will #' be read. If empty, the default `scenarioFile` is used. An example #' scenario file is provided in `system.file(package="irace", "templates/scenario.txt.tmpl")`. -#' @inheritParams defaultScenario -#' +#' @inheritParams defaultScenario +#' #' @return The scenario list read from the file. The scenario settings not #' present in the file are not present in the list, i.e., they are `NULL`. #' @@ -17,7 +17,7 @@ #' \item{[defaultScenario()]}{returns the default scenario settings of \pkg{irace}.} #' \item{[checkScenario()]}{to check that the scenario is valid.} #' } -#' +#' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export readScenario <- function(filename = "", scenario = list(), @@ -28,11 +28,11 @@ readScenario <- function(filename = "", scenario = list(), include.scenario <- function(rfilename, topfile = filename, envir. = scenario_env) { if (!file.exists (rfilename)) { - irace.error ("The scenario file ", shQuote(rfilename), " included from ", + irace_error ("The scenario file ", shQuote(rfilename), " included from ", shQuote(topfile), " does not exist.") } handle.source.error <- function(e) { - irace.error("Reading scenario file ", shQuote(rfilename), + irace_error("Reading scenario file ", shQuote(rfilename), " included from ", shQuote(topfile), " produced the following errors or warnings:\n", paste0(conditionMessage(e), collapse="\n")) @@ -48,10 +48,10 @@ readScenario <- function(filename = "", scenario = list(), if (!filename_given) { filename <- path_rel2abs(params_def["scenarioFile","default"]) if (file.exists(filename)) { - irace.warning("A default scenario file ", shQuote(filename), + irace_warning("A default scenario file ", shQuote(filename), " has been found and will be read.") } else { - irace.warning ("No scenario file given (use ", + irace_warning ("No scenario file given (use ", params_def["scenarioFile", "short"], " or ", params_def["scenarioFile", "long"], ") and no default scenario file ", shQuote(filename), @@ -62,14 +62,14 @@ readScenario <- function(filename = "", scenario = list(), } else { filename <- path_rel2abs(filename) } - + if (file.exists (filename)) { debug.level <- getOption(".irace.debug.level", default = 0) if (debug.level >= 1) cat("# Reading scenario file", shQuote(filename), ".......") # chdir = TRUE to allow recursive sourcing. handle.source.error <- function(e) { - irace.error("Reading scenario file ", shQuote(filename), + irace_error("Reading scenario file ", shQuote(filename), " produced the following errors or warnings:\n", paste0(conditionMessage(e), collapse="\n")) return(NULL) @@ -79,17 +79,17 @@ readScenario <- function(filename = "", scenario = list(), error = handle.source.error, warning = handle.source.error)) if (debug.level >= 1) cat (" done!\n") } else if (filename_given) { - irace.error ("The scenario file ", shQuote(filename), " does not exist.") + irace_error ("The scenario file ", shQuote(filename), " does not exist.") } - + ## Read scenario file variables. - scenario[["scenarioFile"]] <- filename + scenario[["scenarioFile"]] <- filename # If these are given and relative, they should be relative to the # scenario file (except logFile, which is relative to execDir). pathParams <- setdiff(params_def[params_def[, "type"] == "p", "name"], "logFile") params_names <- params_def[!startsWith(params_def[,"name"], "."), "name"] - + for (param in params_names) { if (exists (param, envir = scenario_env, inherits = FALSE)) { value <- get(param, envir = scenario_env, inherits = FALSE) @@ -104,7 +104,7 @@ readScenario <- function(filename = "", scenario = list(), # We only accept variables that match irace.params.names and if the user # wants to define their own, they should use names starting with ".", which # are ignored by ls() - irace.error("Scenario file ", shQuote(filename), " contains unknown variables: ", + irace_error("Scenario file ", shQuote(filename), " contains unknown variables: ", paste0(unknown_scenario_vars, collapse=", "), "\nMAKE SURE NO VARIABLE NAME IS MISSPELL (for example, 'parameterFile' is correct, while 'parametersFile' is not)", "\nIf you wish to use your own variables in the scenario file, use names beginning with a dot `.'") @@ -116,7 +116,7 @@ setup_test_instances <- function(scenario) { testInstances <- scenario[["testInstances"]] if (is.null.or.empty(testInstances)) { - if (!is.null.or.empty(scenario$testInstancesDir) || + if (!is.null.or.empty(scenario$testInstancesDir) || !is.null.or.empty(scenario$testInstancesFile)) { scenario$testInstancesDir <- path_rel2abs(scenario$testInstancesDir) if (!is.null.or.empty(scenario$testInstancesFile)) { @@ -135,7 +135,7 @@ setup_test_instances <- function(scenario) # Remove useless dimensions testInstances <- c(testInstances) } else { - irace.error("testInstances must be a one-dimensional vector or a list. If your instances are matrices or datasets in R, you can use scenario(testInstances=list(data1, data2, data3))") + irace_error("testInstances must be a one-dimensional vector or a list. If your instances are matrices or datasets in R, you can use scenario(testInstances=list(data1, data2, data3))") } } if (is.null(names(testInstances))) { @@ -151,17 +151,17 @@ setup_test_instances <- function(scenario) #' #' Checks for errors a (possibly incomplete) scenario setup of #' \pkg{irace} and transforms it into a valid scenario. -#' +#' #' @inheritParams defaultScenario -#' +#' #' @return The scenario received as a parameter, possibly corrected. Unset #' scenario settings are set to their default values. #' -#' @details This function checks that the directories and the file names -#' provided and required by the \pkg{irace} exist. It also checks that the -#' settings are of the proper type, e.g. that settings expected to be integers +#' @details This function checks that the directories and the file names +#' provided and required by the \pkg{irace} exist. It also checks that the +#' settings are of the proper type, e.g. that settings expected to be integers #' are really integers. Finally, it also checks that there is no inconsistency -#' between settings. If an error is found that prevents \pkg{irace} from +#' between settings. If an error is found that prevents \pkg{irace} from #' running properly, it will stop with an error. #' #' @seealso @@ -171,7 +171,7 @@ setup_test_instances <- function(scenario) #' \item{[defaultScenario()]}{returns the default scenario settings of \pkg{irace}.} #' \item{[checkScenario()]}{to check that the scenario is valid.} #' } -#' +#' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export ## FIXME: This function should only do checks and return TRUE/FALSE. There @@ -191,9 +191,9 @@ checkScenario <- function(scenario = defaultScenario()) if (is.logical(x)) return(x) if (is.na(x)) return(as.logical(NA)) # We handle "0" and "1" but not "TRUE" and "FALSE". - x <- suppressWarnings(as.integer(x)) + x <- suppressWarnings(as.integer(x)) if (is.na(x) || (x != 0L && x != 1L)) { - irace.error (quote.param(name), " must be either 0 or 1.") + irace_error (quote.param(name), " must be either 0 or 1.") } as.logical(x) } @@ -202,23 +202,23 @@ checkScenario <- function(scenario = defaultScenario()) { valid <- trimws(strsplit(.irace.params.def[x, "domain"],",",fixed=TRUE)[[1L]]) if (scenario[[x]] %not_in% valid) { - irace.error ("Invalid value '", scenario[[x]], "' of ", + irace_error ("Invalid value '", scenario[[x]], "' of ", quote.param(x), ", valid values are: ", paste0(valid, collapse = ", ")) } } - + # Fill possible unset (NULL) with default settings. scenario <- defaultScenario(scenario) # Duplicated entries will cause confusion. dups <- anyDuplicated(names(scenario)) if (dups) - irace.error("scenario contains duplicated entries: ", names(scenario)[dups]) + irace_error("scenario contains duplicated entries: ", names(scenario)[dups]) # We have characters everywhere, set to the right types to avoid problems # later. - + # Boolean control parameters. boolParams <- .irace.params.def[.irace.params.def[, "type"] == "b", "name"] for (p in boolParams) { @@ -228,14 +228,14 @@ checkScenario <- function(scenario = defaultScenario()) realParams <- .irace.params.def[.irace.params.def[, "type"] == "r", "name"] for (param in realParams) { if (is.null.or.empty(scenario[[param]])) - scenario[[param]] <- NA + scenario[[param]] <- NA_real_ if (is.na(scenario[[param]])) next # Allow NA default values scenario[[param]] <- suppressWarnings(as.numeric(scenario[[param]])) if (is.null(scenario[[param]]) || is.na (scenario[[param]]) || scenario[[param]] < 0.0 || scenario[[param]] > 1.0) - irace.error (quote.param(param), " must be a real value within [0, 1].") + irace_error (quote.param(param), " must be a real value within [0, 1].") } # Integer control parameters @@ -243,12 +243,12 @@ checkScenario <- function(scenario = defaultScenario()) for (param in intParams) { p <- scenario[[param]] if (is.null.or.empty(p)) - scenario[[param]] <- NA + scenario[[param]] <- NA_integer_ if (is.na(scenario[[param]])) next # Allow NA default values p <- suppressWarnings(as.numeric(p)) if (is.null(p) || is.na (p) || !is.wholenumber(p) || p < 0) - irace.error (quote.param (param), " must be a non-negative integer.") + irace_error (quote.param (param), " must be a non-negative integer.") scenario[[param]] <- as.integer(p) } @@ -256,7 +256,7 @@ checkScenario <- function(scenario = defaultScenario()) if (any(scenario[what] <= 0L)) { for (op in what) { if (scenario[[op]] <= 0L) - irace.error(quote.param (op), " = ", scenario[[op]], " must be larger than 0.") + irace_error(quote.param (op), " = ", scenario[[op]], " must be larger than 0.") } } } @@ -267,67 +267,67 @@ checkScenario <- function(scenario = defaultScenario()) # Check that the files exist and are readable. scenario$parameterFile <- path_rel2abs(scenario$parameterFile) if (is.null.or.empty(scenario$parameters)) { - irace.note("Reading parameter file '", scenario$parameterFile, "'.\n") + irace_note("Reading parameter file '", scenario$parameterFile, "'.\n") scenario$parameters <- readParameters(file = scenario$parameterFile, # AClib benchmarks use 15 digits digits = if (scenario$aclib) 15L else 4L, debugLevel = scenario$debugLevel) } scenario$parameters <- checkParameters(scenario$parameters) - + scenario$execDir <- path_rel2abs(scenario$execDir) file.check (scenario$execDir, isdir = TRUE, text = paste0("execution directory ", quote.param("execDir"))) options(.irace.execdir = scenario$execDir) - if (!is.null.or.empty(scenario$logFile)) { - scenario$logFile <- path_rel2abs(scenario$logFile, cwd = scenario$execDir) - file.check(scenario$logFile, writeable = TRUE, text = quote.param('logFile')) - } else { + if (is.null.or.empty(scenario$logFile)) { # We cannot use NULL because defaultScenario() would override it. scenario$logFile <- "" + } else { + scenario$logFile <- path_rel2abs(scenario$logFile, cwd = scenario$execDir) + file.check(scenario$logFile, writeable = TRUE, text = quote.param('logFile')) } - if (!is.null.or.empty(scenario$recoveryFile)) { + if (is.null.or.empty(scenario$recoveryFile)) { + # We cannot use NULL because defaultScenario() would override it. + scenario$recoveryFile <- "" + } else { scenario$recoveryFile <- path_rel2abs(scenario$recoveryFile) file.check(scenario$recoveryFile, readable = TRUE, text = paste0("recovery file ", quote.param("recoveryFile"))) - + if (!is.null.or.empty(scenario$logFile) # Must have been set "" above. && scenario$recoveryFile == scenario$logFile) { - irace.error("log file and recovery file should be different '", + irace_error("log file and recovery file should be different '", scenario$logFile, "'") } - } else { - # We cannot use NULL because defaultScenario() would override it. - scenario$recoveryFile <- "" } - + if (is.null.or.empty(scenario$targetRunnerParallel)) { scenario$targetRunnerParallel <- NULL } else if (is.function.name(scenario$targetRunnerParallel)) { scenario$targetRunnerParallel <- get.function(scenario$targetRunnerParallel) } else { - irace.error("'targetRunnerParallel' must be a function") + irace_error("'targetRunnerParallel' must be a function") } if (is.null.or.empty(scenario$repairConfiguration)) scenario$repairConfiguration <- NULL else if (is.function.name(scenario$repairConfiguration)) # Byte-compile it. scenario$repairConfiguration <- bytecompile(get.function(scenario$repairConfiguration)) - else - irace.error("'repairConfiguration' must be a function") - + else + irace_error("'repairConfiguration' must be a function") + if (is.na(scenario$capping)) scenario$capping <- (scenario$elitist && scenario$maxTime > 0 && !is.na(scenario$boundMax) && scenario$boundMax > 0) if (scenario$capping) { - if (!scenario$elitist) - irace.error("When capping == TRUE, elitist must be enabled.") - if (scenario$boundMax <= 0) - irace.error("When capping == TRUE, boundMax (", scenario$boundMax, ") must be > 0") + if (!scenario$elitist) + irace_error("When capping == TRUE, elitist must be enabled.") + if (scenario$boundMax <= 0) + irace_error("When capping == TRUE, boundMax (", scenario$boundMax, ") must be > 0") check.valid.param("cappingType") check.valid.param("boundType") if (scenario$boundPar < 1) - irace.error("Invalid value (", scenario$boundPar, ") ", + irace_error("Invalid value (", scenario$boundPar, ") ", quote.param("boundPar"), " must be >= 1") } else if (scenario$boundMax <= 0 || is.na(scenario$boundMax)) { # no capping scenario$boundMax <- NULL @@ -335,14 +335,14 @@ checkScenario <- function(scenario = defaultScenario()) if (is.function.name(scenario$targetRunner)) { scenario$targetRunner <- get.function(scenario$targetRunner) - irace.assert(is.function(scenario$targetRunner)) + irace_assert(is.function(scenario$targetRunner)) } else if (is.null(scenario$targetRunnerParallel)) { if (is.character(scenario$targetRunner)) { scenario$targetRunner <- path_rel2abs(scenario$targetRunner) if (is.null.or.empty(scenario$targetRunnerLauncher)) { file.check (scenario$targetRunner, executable = TRUE, text = paste0("target runner ", quote.param("targetRunner"))) - } else { + } else { scenario$targetRunnerLauncher <- path_rel2abs(scenario$targetRunnerLauncher) file.check (scenario$targetRunnerLauncher, executable = TRUE, text = paste0("target runner launcher ", quote.param("targetRunnerLauncher"))) @@ -355,7 +355,7 @@ checkScenario <- function(scenario = defaultScenario()) } check_target_cmdline(scenario$targetCmdline, capping = scenario$capping) } else { - irace.error(quote.param ('targetRunner'), " must be a function or an executable program") + irace_error(quote.param ('targetRunner'), " must be a function or an executable program") } } @@ -363,13 +363,13 @@ checkScenario <- function(scenario = defaultScenario()) scenario$targetEvaluator <- NULL } else if (is.function.name(scenario$targetEvaluator)) { scenario$targetEvaluator <- get.function(scenario$targetEvaluator) - irace.assert(is.function(scenario$targetEvaluator)) + irace_assert(is.function(scenario$targetEvaluator)) } else if (is.character(scenario$targetEvaluator)) { scenario$targetEvaluator <- path_rel2abs(scenario$targetEvaluator) file.check (scenario$targetEvaluator, executable = TRUE, text = "target evaluator") } else { - irace.error(quote.param('targetEvaluator'), " must be a function or an executable program") + irace_error(quote.param('targetEvaluator'), " must be a function or an executable program") } # Training instances @@ -380,23 +380,23 @@ checkScenario <- function(scenario = defaultScenario()) } if (is.null.or.empty(scenario$trainInstancesDir) && is.null.or.empty(scenario$trainInstancesFile)) - irace.error("Both ", quote.param ("trainInstancesDir"), " and ", + irace_error("Both ", quote.param ("trainInstancesDir"), " and ", quote.param ("trainInstancesFile"), " are empty: No instances provided") - + scenario$instances <- readInstances(instancesDir = scenario$trainInstancesDir, instancesFile = scenario$trainInstancesFile) } if (length(scenario$instances) == 0L) { - irace.error("No instances found in `scenario$instances`") + irace_error("No instances found in `scenario$instances`") } else if (!is.null(dim(scenario$instances))) { if (length(dim(scenario$instances)) == 1L || (length(dim(scenario$instances)) == 2L && dim(scenario$instances)[1] == 1L)) { # Remove useless dimensions scenario$instances <- c(scenario$instances) } else { - irace.error("Instances must be a one-dimensional vector or a list. If your instances are matrices or datasets in R, you can use scenario(instances=list(data1, data2, data3))") + irace_error("Instances must be a one-dimensional vector or a list. If your instances are matrices or datasets in R, you can use scenario(instances=list(data1, data2, data3))") } } # Testing instances @@ -410,67 +410,67 @@ checkScenario <- function(scenario = defaultScenario()) # We cannot read the configurations here because we need the parameters. # FIXME: We should have the parameters inside scenario. } - + if (is.null.or.empty(scenario$initConfigurations)) { scenario$initConfigurations <- NULL } else if (!is.data.frame(scenario$initConfigurations) && !is.matrix(scenario$initConfigurations)) { - irace.error("if given, 'initConfigurations' must be a matrix or data.frame.") + irace_error("if given, 'initConfigurations' must be a matrix or data.frame.") } - + if (length(scenario$instances) %% scenario$blockSize != 0) { - irace.error("The number of instances (", length(scenario$instances), ") must be a multiple of ", + irace_error("The number of instances (", length(scenario$instances), ") must be a multiple of ", quote.param("blockSize"), ".") } - + if (scenario$firstTest %% scenario$eachTest != 0) { - irace.error(quote.param("firstTest"), " must be a multiple of ", + irace_error(quote.param("firstTest"), " must be a multiple of ", quote.param("eachTest"), ".") } - + if (scenario$mu < scenario$firstTest * scenario$blockSize) { if (scenario$debugLevel >= 1) { - irace.warning("Assuming 'mu = firstTest * blockSize' because 'mu' cannot be smaller.\n") + irace_warning("Assuming 'mu = firstTest * blockSize' because 'mu' cannot be smaller.\n") } scenario$mu <- scenario$firstTest * scenario$blockSize } else if (scenario$mu %% scenario$blockSize != 0) { - irace.error(quote.param("mu"), " must be a multiple of ", + irace_error(quote.param("mu"), " must be a multiple of ", quote.param("blockSize"), ".") } - + if (!is.na(scenario$minExperiments)) scenario$maxExperiments <- max(scenario$maxExperiments, scenario$minExperiments) - + ## Only maxExperiments or maxTime should be set. Negative values are not ## allowed. if (scenario$maxExperiments == 0 && scenario$maxTime == 0) { - irace.error("Tuning budget was not provided. Set ", + irace_error("Tuning budget was not provided. Set ", quote.param("maxExperiments"), " or ", quote.param("maxTime"), ".") } else if (scenario$maxExperiments > 0 && scenario$maxTime > 0) { - irace.error("Two different tuning budgets provided, please set only ", + irace_error("Two different tuning budgets provided, please set only ", quote.param("maxExperiments"), " or only ", quote.param ("maxTime"), ".") } else if (scenario$maxExperiments < 0 ) { - irace.error("Negative budget provided, ", quote.param("maxExperiments"), + irace_error("Negative budget provided, ", quote.param("maxExperiments"), "must be >= 0." ) } else if (scenario$maxTime < 0) { - irace.error("Negative budget provided, ", quote.param("maxTime"), " must be >= 0.") + irace_error("Negative budget provided, ", quote.param("maxTime"), " must be >= 0.") } - - if (scenario$maxTime > 0 && (scenario$budgetEstimation <= 0 || scenario$budgetEstimation >= 1)) - irace.error(quote.param("budgetEstimation"), " must be within (0,1).") + + if (scenario$maxTime > 0 && (scenario$budgetEstimation <= 0 || scenario$budgetEstimation >= 1)) + irace_error(quote.param("budgetEstimation"), " must be within (0,1).") if (scenario$maxTime > 0 && !scenario$elitist) - irace.error(quote.param("maxTime"), " requires using 'elitist=1'") - + irace_error(quote.param("maxTime"), " requires using 'elitist=1'") + if (scenario$deterministic && scenario$firstTest * scenario$blockSize > length(scenario$instances)) { - irace.error("When deterministic == TRUE, the number of instances (", + irace_error("When deterministic == TRUE, the number of instances (", length(scenario$instances), ") cannot be smaller than firstTest (", scenario$firstTest, ") * blockSize (", scenario$blockSize, ")") } if (scenario$mpi && scenario$parallel < 2) - irace.error (quote.param("parallel"), " must be larger than 1 when mpi is enabled.") - + irace_error (quote.param("parallel"), " must be larger than 1 when mpi is enabled.") + if (is.null.or.empty(scenario$batchmode)) scenario$batchmode <- 0 if (scenario$batchmode != 0) { @@ -479,12 +479,12 @@ checkScenario <- function(scenario = defaultScenario()) } # Currently batchmode requires a targetEvaluator if (scenario$batchmode != 0 && is.null(scenario$targetEvaluator)) { - irace.error(quote.param("batchmode"), " requires using ", + irace_error(quote.param("batchmode"), " requires using ", quote.param("targetEvaluator"), ".") } if (scenario$batchmode != 0 && scenario$mpi) { - irace.error(quote.param("mpi"), " and ", quote.param("batchmode"), + irace_error(quote.param("mpi"), " and ", quote.param("batchmode"), " cannot be enabled at the same time.") } @@ -492,7 +492,7 @@ checkScenario <- function(scenario = defaultScenario()) if (scenario$capping) scenario$testType <- "t-test" else scenario$testType <- "f-test" } - + scenario$testType <- switch(tolower(scenario$testType), "f-test" =, # Fall-through @@ -510,7 +510,7 @@ checkScenario <- function(scenario = defaultScenario()) #' Prints the given scenario #' #' @inheritParams defaultScenario -#' +#' #' @seealso #' \describe{ #' \item{[readScenario()]}{for reading a configuration scenario from a file.} @@ -518,7 +518,7 @@ checkScenario <- function(scenario = defaultScenario()) #' \item{[defaultScenario()]}{returns the default scenario settings of \pkg{irace}.} #' \item{[checkScenario()]}{to check that the scenario is valid.} #' } -#' +#' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export printScenario <- function(scenario) @@ -534,7 +534,7 @@ printScenario <- function(scenario) #' Default scenario settings #' #' Return scenario object with default values. -#' +#' #' @param scenario `list()`\cr Data structure containing \pkg{irace} #' settings. The data structure has to be the one returned by the function #' [defaultScenario()] or [readScenario()]. @@ -543,11 +543,11 @@ printScenario <- function(scenario) #' the scenario. This should only be modified by packages that wish to extend #' \pkg{irace}. #' -#' +#' #' @return A list indexed by the \pkg{irace} parameter names, #' containing the default values for each parameter, except for those #' already present in the scenario passed as argument. -#' The scenario list contains the following elements: +#' The scenario list contains the following elements: # __IRACE_OPTIONS__BEGIN__ #' \itemize{ #' \item General options: @@ -556,7 +556,7 @@ printScenario <- function(scenario) #' \item{`execDir`}{Directory where the programs will be run. (Default: `"./"`)} #' \item{`logFile`}{File to save tuning results as an R dataset, either absolute path or relative to execDir. (Default: `"./irace.Rdata"`)} #' \item{`quiet`}{Reduce the output generated by irace to a minimum. (Default: `0`)} -#' \item{`debugLevel`}{Debug level of the output of \code{irace}. Set this to 0 to silence all debug messages. Higher values provide more verbose debug messages. (Default: `0`)} +#' \item{`debugLevel`}{Debug level of the output of `irace`. Set this to 0 to silence all debug messages. Higher values (1, 2 or 3) provide more verbose debug messages. (Default: `0`)} #' \item{`seed`}{Seed of the random number generator (by default, generate a random seed). (Default: `NA`)} #' \item{`repairConfiguration`}{User-defined R function that takes a configuration generated by irace and repairs it. (Default: `""`)} #' \item{`postselection`}{Perform a postselection race after the execution of irace to consume all remaining budget. Value 0 disables the postselection race. (Default: `1`)} @@ -587,59 +587,60 @@ printScenario <- function(scenario) #' \item Target algorithm execution: #' \describe{ #' \item{`targetRunner`}{Executable called for each configuration that executes the target algorithm to be tuned. See the templates and examples provided. (Default: `"./target-runner"`)} -#' \item{`targetRunnerLauncher`}{Executable that will be used to launch the target runner, when \code{targetRunner} cannot be executed directly (e.g., a Python script in Windows). (Default: `""`)} -#' \item{`targetCmdline`}{Command-line arguments provided to \code{targetRunner} (or \code{targetRunnerLauncher} if defined). The substrings \code{\{configurationID\}}, \code{\{instanceID\}}, \code{\{seed\}}, \code{\{instance\}}, and \code{\{bound\}} will be replaced by their corresponding values. The substring \code{\{targetRunnerArgs\}} will be replaced by the concatenation of the switch and value of all active parameters of the particular configuration being evaluated. The substring \code{\{targetRunner\}}, if present, will be replaced by the value of \code{targetRunner} (useful when using \code{targetRunnerLauncher}). (Default: `"{configurationID} {instanceID} {seed} {instance} {bound} {targetRunnerArgs}"`)} -#' \item{`targetRunnerRetries`}{Number of times to retry a call to \code{targetRunner} if the call failed. (Default: `0`)} -#' \item{`targetRunnerTimeout`}{Timeout in seconds of any \code{targetRunner} call (only applies to \code{target-runner} executables not to R functions), ignored if 0. (Default: `0`)} -#' \item{`targetRunnerData`}{Optional data passed to \code{targetRunner}. This is ignored by the default \code{targetRunner} function, but it may be used by custom \code{targetRunner} functions to pass persistent data around. (Default: `""`)} -#' \item{`targetRunnerParallel`}{Optional R function to provide custom parallelization of \code{targetRunner}. (Default: `""`)} +#' \item{`targetRunnerLauncher`}{Executable that will be used to launch the target runner, when `targetRunner` cannot be executed directly (e.g., a Python script in Windows). (Default: `""`)} +#' \item{`targetCmdline`}{Command-line arguments provided to `targetRunner` (or `targetRunnerLauncher` if defined). The substrings `\{configurationID\}`, `\{instanceID\}`, `\{seed\}`, `\{instance\}`, and `\{bound\}` will be replaced by their corresponding values. The substring `\{targetRunnerArgs\}` will be replaced by the concatenation of the switch and value of all active parameters of the particular configuration being evaluated. The substring `\{targetRunner\}`, if present, will be replaced by the value of `targetRunner` (useful when using `targetRunnerLauncher`). (Default: `"{configurationID} {instanceID} {seed} {instance} {bound} {targetRunnerArgs}"`)} +#' \item{`targetRunnerRetries`}{Number of times to retry a call to `targetRunner` if the call failed. (Default: `0`)} +#' \item{`targetRunnerTimeout`}{Timeout in seconds of any `targetRunner` call (only applies to `target-runner` executables not to R functions), ignored if 0. (Default: `0`)} +#' \item{`targetRunnerData`}{Optional data passed to `targetRunner`. This is ignored by the default `targetRunner` function, but it may be used by custom `targetRunner` functions to pass persistent data around. (Default: `""`)} +#' \item{`targetRunnerParallel`}{Optional R function to provide custom parallelization of `targetRunner`. (Default: `""`)} #' \item{`targetEvaluator`}{Optional script or R function that provides a numeric value for each configuration. See templates/target-evaluator.tmpl (Default: `""`)} #' \item{`deterministic`}{If the target algorithm is deterministic, configurations will be evaluated only once per instance. (Default: `0`)} -#' \item{`parallel`}{Number of calls to \code{targetRunner} to execute in parallel. Values \code{0} or \code{1} mean no parallelization. (Default: `0`)} +#' \item{`parallel`}{Number of calls to `targetRunner` to execute in parallel. Values `0` or `1` mean no parallelization. (Default: `0`)} #' \item{`loadBalancing`}{Enable/disable load-balancing when executing experiments in parallel. Load-balancing makes better use of computing resources, but increases communication overhead. If this overhead is large, disabling load-balancing may be faster. (Default: `1`)} -#' \item{`mpi`}{Enable/disable MPI. Use \code{Rmpi} to execute \code{targetRunner} in parallel (parameter \code{parallel} is the number of slaves). (Default: `0`)} -#' \item{`batchmode`}{Specify how irace waits for jobs to finish when \code{targetRunner} submits jobs to a batch cluster: sge, pbs, torque, slurm or htcondor. \code{targetRunner} must submit jobs to the cluster using, for example, \code{qsub}. (Default: `0`)} +#' \item{`mpi`}{Enable/disable MPI. Use `Rmpi` to execute `targetRunner` in parallel (parameter `parallel` is the number of slaves). (Default: `0`)} +#' \item{`batchmode`}{Specify how irace waits for jobs to finish when `targetRunner` submits jobs to a batch cluster: sge, pbs, torque, slurm or htcondor. `targetRunner` must submit jobs to the cluster using, for example, `qsub`. (Default: `0`)} #' } #' \item Initial configurations: #' \describe{ #' \item{`initConfigurations`}{Data frame describing initial configurations (usually read from a file using \code{readConfigurations}). (Default: `""`)} -#' \item{`configurationsFile`}{File that contains a table of initial configurations. If empty or \code{NULL}, all initial configurations are randomly generated. (Default: `""`)} +#' \item{`configurationsFile`}{File that contains a table of initial configurations. If empty or `NULL`, all initial configurations are randomly generated. (Default: `""`)} #' } #' \item Training instances: #' \describe{ #' \item{`instances`}{Character vector of the instances to be used in the \code{targetRunner}. (Default: `""`)} -#' \item{`trainInstancesDir`}{Directory where training instances are located; either absolute path or relative to current directory. If no \code{trainInstancesFiles} is provided, all the files in \code{trainInstancesDir} will be listed as instances. (Default: `""`)} -#' \item{`trainInstancesFile`}{File that contains a list of training instances and optionally additional parameters for them. If \code{trainInstancesDir} is provided, \code{irace} will search for the files in this folder. (Default: `""`)} -#' \item{`blockSize`}{Number of training instances, that make up a `block' in \code{trainInstancesFile}. Elimination of configurations will only be performed after evaluating a complete block and never in the middle of a block. Each block typically contains one instance from each instance class (type or family) and the block size is the number of classes. (Default: `1`)} +#' \item{`trainInstancesDir`}{Directory where training instances are located; either absolute path or relative to current directory. If no `trainInstancesFiles` is provided, all the files in `trainInstancesDir` will be listed as instances. (Default: `""`)} +#' \item{`trainInstancesFile`}{File that contains a list of training instances and optionally additional parameters for them. If `trainInstancesDir` is provided, `irace` will search for the files in this folder. (Default: `""`)} +#' \item{`blockSize`}{Number of training instances, that make up a 'block' in `trainInstancesFile`. Elimination of configurations will only be performed after evaluating a complete block and never in the middle of a block. Each block typically contains one instance from each instance class (type or family) and the block size is the number of classes. The value of `blockSize` will multiply `firstTest`, `eachTest` and `elitistNewInstances`. (Default: `1`)} #' } #' \item Tuning budget: #' \describe{ -#' \item{`maxExperiments`}{Maximum number of runs (invocations of \code{targetRunner}) that will be performed. It determines the maximum budget of experiments for the tuning. (Default: `0`)} -#' \item{`minExperiments`}{Minimum number of runs (invocations of \code{targetRunner}) that will be performed. It determines the minimum budget of experiments for the tuning. The actual budget depends on the number of parameters and \code{minSurvival}. (Default: `NA`)} -#' \item{`maxTime`}{Maximum total execution time for the executions of \code{targetRunner}. \code{targetRunner} must return two values: cost and time. This value and the one returned by \code{targetRunner} must use the same units (seconds, minutes, iterations, evaluations, ...). (Default: `0`)} -#' \item{`budgetEstimation`}{Fraction (smaller than 1) of the budget used to estimate the mean computation time of a configuration. Only used when \code{maxTime} > 0 (Default: `0.05`)} +#' \item{`maxExperiments`}{Maximum number of runs (invocations of `targetRunner`) that will be performed. It determines the maximum budget of experiments for the tuning. (Default: `0`)} +#' \item{`minExperiments`}{Minimum number of runs (invocations of `targetRunner`) that will be performed. It determines the minimum budget of experiments for the tuning. The actual budget depends on the number of parameters and `minSurvival`. (Default: `NA`)} +#' \item{`maxTime`}{Maximum total execution time for the executions of `targetRunner`. `targetRunner` must return two values: cost and time. This value and the one returned by `targetRunner` must use the same units (seconds, minutes, iterations, evaluations, ...). (Default: `0`)} +#' \item{`budgetEstimation`}{Fraction (smaller than 1) of the budget used to estimate the mean computation time of a configuration. Only used when `maxTime` > 0 (Default: `0.05`)} #' \item{`minMeasurableTime`}{Minimum time unit that is still (significantly) measureable. (Default: `0.01`)} #' } #' \item Statistical test: #' \describe{ -#' \item{`testType`}{Statistical test used for elimination. The default value selects \code{t-test} if \code{capping} is enabled or \code{F-test}, otherwise. Valid values are: F-test (Friedman test), t-test (pairwise t-tests with no correction), t-test-bonferroni (t-test with Bonferroni's correction for multiple comparisons), t-test-holm (t-test with Holm's correction for multiple comparisons). (Default: `""`)} -#' \item{`firstTest`}{Number of instances evaluated before the first elimination test. It must be a multiple of \code{eachTest}. (Default: `5`)} +#' \item{`testType`}{Statistical test used for elimination. The default value selects `t-test` if `capping` is enabled or `F-test`, otherwise. Valid values are: F-test (Friedman test), t-test (pairwise t-tests with no correction), t-test-bonferroni (t-test with Bonferroni's correction for multiple comparisons), t-test-holm (t-test with Holm's correction for multiple comparisons). (Default: `""`)} +#' \item{`firstTest`}{Number of instances evaluated before the first elimination test. It must be a multiple of `eachTest`. (Default: `5`)} #' \item{`eachTest`}{Number of instances evaluated between elimination tests. (Default: `1`)} #' \item{`confidence`}{Confidence level for the elimination test. (Default: `0.95`)} #' } #' \item Adaptive capping: #' \describe{ -#' \item{`capping`}{Enable the use of adaptive capping, a technique designed for minimizing the computation time of configurations. Capping is enabled by default if \code{elitist} is active, \code{maxTime > 0} and \code{boundMax > 0}. (Default: `NA`)} +#' \item{`capping`}{Enable the use of adaptive capping, a technique designed for minimizing the computation time of configurations. Capping is enabled by default if `elitist` is active, `maxTime > 0` and `boundMax > 0`. (Default: `NA`)} +#' \item{`cappingAfterFirstTest`}{If set to 1, elimination due to capping only happens after `firstTest` instances are seen. (Default: `0`)} #' \item{`cappingType`}{Measure used to obtain the execution bound from the performance of the elite configurations.\itemize{\item median: Median performance of the elite configurations.\item mean: Mean performance of the elite configurations.\item best: Best performance of the elite configurations.\item worst: Worst performance of the elite configurations.} (Default: `"median"`)} #' \item{`boundType`}{Method to calculate the mean performance of elite configurations.\itemize{\item candidate: Mean execution times across the executed instances and the current one.\item instance: Execution time of the current instance.} (Default: `"candidate"`)} -#' \item{`boundMax`}{Maximum execution bound for \code{targetRunner}. It must be specified when capping is enabled. (Default: `0`)} +#' \item{`boundMax`}{Maximum execution bound for `targetRunner`. It must be specified when capping is enabled. (Default: `0`)} #' \item{`boundDigits`}{Precision used for calculating the execution time. It must be specified when capping is enabled. (Default: `0`)} -#' \item{`boundPar`}{Penalization constant for timed out executions (executions that reach \code{boundMax} execution time). (Default: `1`)} -#' \item{`boundAsTimeout`}{Replace the configuration cost of bounded executions with \code{boundMax}. (Default: `1`)} +#' \item{`boundPar`}{Penalization constant for timed out executions (executions that reach `boundMax` execution time). (Default: `1`)} +#' \item{`boundAsTimeout`}{Replace the configuration cost of bounded executions with `boundMax`. (Default: `1`)} #' } #' \item Recovery: #' \describe{ -#' \item{`recoveryFile`}{Previously saved log file to recover the execution of \code{irace}, either absolute path or relative to the current directory. If empty or \code{NULL}, recovery is not performed. (Default: `""`)} +#' \item{`recoveryFile`}{Previously saved log file to recover the execution of `irace`, either absolute path or relative to the current directory. If empty or `NULL`, recovery is not performed. (Default: `""`)} #' } #' \item Testing: #' \describe{ @@ -669,7 +670,7 @@ defaultScenario <- function(scenario = list(), if (is.null(names(scenario))) { scenario <- setNames(as.list(params_def[params_names,"default"]), params_names) } else if (!all(names(scenario) %in% params_names)) { - irace.error("Unknown scenario settings: ", + irace_error("Unknown scenario settings: ", paste(names(scenario)[!(names(scenario) %in% params_names)], collapse = ", ")) } else { @@ -685,11 +686,20 @@ defaultScenario <- function(scenario = list(), readInstances <- function(instancesDir = NULL, instancesFile = NULL) { if (is.null.or.empty(instancesDir) && is.null.or.empty(instancesFile)) - irace.error("Both instancesDir and instancesFile are empty: No instances provided") - + irace_error("Both instancesDir and instancesFile are empty: No instances provided") + instances <- NULL - - if (!is.null.or.empty(instancesFile)) { + + if (is.null.or.empty(instancesFile)) { + file.check (instancesDir, isdir = TRUE, notempty = TRUE, + text = "instances directory") + # The files are sorted in alphabetical order, on the full path if + # 'full.names = TRUE'. + instances <- list.files (path = instancesDir, full.names = TRUE, + recursive = TRUE) + if (length (instances) == 0) + irace_error("No instances found in `", instancesDir, "'") + } else { file.check (instancesFile, readable = TRUE, text = "instance file") # We do not warn if the last line does not finish with a newline. instances <- readLines (instancesFile, warn = FALSE) @@ -697,19 +707,10 @@ readInstances <- function(instancesDir = NULL, instancesFile = NULL) instances <- sub("^[[:space:]]+", "", instances) # Remove leading whitespace instances <- instances[instances != ""] # Delete empty lines if (is.null.or.empty(instances)) - irace.error("No instances found in '", instancesFile, + irace_error("No instances found in '", instancesFile, "' (whitespace and comments starting with '#' are ignored)") if (!is.null.or.empty(instancesDir)) instances <- paste0 (instancesDir, "/", instances) - } else { - file.check (instancesDir, isdir = TRUE, notempty = TRUE, - text = "instances directory") - # The files are sorted in alphabetical order, on the full path if - # 'full.names = TRUE'. - instances <- list.files (path = instancesDir, full.names = TRUE, - recursive = TRUE) - if (length (instances) == 0) - irace.error("No instances found in `", instancesDir, "'") } instances } @@ -721,6 +722,6 @@ update_scenario <- function(scenario, ...) return(scenario) unknown_scenario_args <- setdiff(names(scenario_args), names(scenario)) if (length(unknown_scenario_args)) - irace.error("Unknown scenario settings given: ", paste0(unknown_scenario_args, collapse=", ")) - modifyList(scenario, scenario_args) + irace_error("Unknown scenario settings given: ", paste0(unknown_scenario_args, collapse=", ")) + utils::modifyList(scenario, scenario_args) } diff --git a/R/testing.R b/R/testing.R index 363415f8..f9d702b5 100644 --- a/R/testing.R +++ b/R/testing.R @@ -25,8 +25,8 @@ testConfigurations <- function(configurations, scenario) testInstances <- scenario[["testInstances"]] instances_id <- names(testInstances) - if (length(testInstances) == 0L) irace.error("No test instances given") - if (is.null(instances_id)) irace.error("testInstances must have names") + if (length(testInstances) == 0L) irace_error("No test instances given") + if (is.null(instances_id)) irace_error("testInstances must have names") # 2147483647 is the maximum value for a 32-bit signed integer. # We use replace = TRUE, because replace = FALSE allocates memory for each possible number. @@ -41,29 +41,32 @@ testConfigurations <- function(configurations, scenario) # Create experiment list experiments <- createExperimentList(configurations, parameters = scenario$parameters, - instances = testInstances, instances_ID = instances_id, seeds = instanceSeed, - bounds = rep(scenario$boundMax, nrow(configurations))) + instances = testInstances, instances_ID = instances_id, seeds = instanceSeed, + # We cannot use rep.int because scenario$boundMax may be NULL. + bounds = rep(scenario$boundMax, nrow(configurations))) race_state <- RaceState$new(scenario) if (scenario$debugLevel >= 3L) { - irace.note ("Memory used before execute_experiments() in testConfigurations():\n") + irace_note ("Memory used before execute_experiments() in testConfigurations():\n") race_state$print_mem_used() } race_state$start_parallel(scenario) on.exit(race_state$stop_parallel()) + # We cannot let targetRunner or targetEvaluator modify our random seed, so we save it. + withr::local_preserve_seed() target_output <- execute_experiments(race_state, experiments, scenario) # targetEvaluator may be NULL. If so, target_output must contain the right # output already. if (!is.null(scenario$targetEvaluator)) target_output <- execute_evaluator(race_state$target_evaluator, experiments, - scenario, target_output, configurations[[".ID."]]) + scenario, target_output) - # FIXME: It would be much faster to get convert target_output$cost to a - # vector, then initialize the matrix with the vector. + # FIXME: It would be much faster to convert target_output to a data.table like we do in race_wrapper(), + # then dcast() to a matrix like we do elsewhere. testResults <- matrix(NA, ncol = nrow(configurations), nrow = length(testInstances), # dimnames = list(rownames, colnames) dimnames = list(instances_id, configurations$.ID.)) - cost <- sapply(target_output, getElement, "cost") + cost <- unlist_element(target_output, "cost") if (scenario$capping) cost <- applyPAR(cost, boundMax = scenario$boundMax, boundPar = scenario$boundPar) # FIXME: Vectorize this loop @@ -72,7 +75,7 @@ testConfigurations <- function(configurations, scenario) colnames(testResults) == experiments[[i]]$id_configuration] <- cost[i] } if (scenario$debugLevel >= 3L) { - irace.note ("Memory used at the end of testConfigurations():\n") + irace_note ("Memory used at the end of testConfigurations():\n") race_state$print_mem_used() } diff --git a/R/timer.R b/R/timer.R index 22128bdd..5510e226 100644 --- a/R/timer.R +++ b/R/timer.R @@ -10,6 +10,9 @@ Timer <- R6::R6Class("Timer", cloneable = TRUE, lock_class = TRUE, lock_objects if (!is.na(x[[5L]])) x[[2L]] <- x[[2L]] + x[[5L]] c(user=x[[1L]], system=x[[2L]], wallclock=x[[3L]]) + }, + wallclock = function() { + proc.time()[[3L]] - self$start[[3L]] }) ) diff --git a/R/utils.R b/R/utils.R index ee665258..09849e75 100644 --- a/R/utils.R +++ b/R/utils.R @@ -1,6 +1,6 @@ # Print a user-level warning message, when the calling context # cannot help the user to understand why the program failed. -irace.warning <- function(...) +irace_warning <- function(...) { if (getOption(".irace.quiet", default=FALSE)) return() warning(paste0(.irace_msg_prefix, ..., collapse=""), @@ -9,7 +9,7 @@ irace.warning <- function(...) # Print a user-level fatal error message, when the calling context # cannot help the user to understand why the program failed. -irace.error <- function(...) +irace_error <- function(...) { # The default is only 1000, which is too small. 8170 is the maximum # value allowed up to R 3.0.2 @@ -22,7 +22,7 @@ irace.error <- function(...) ## R> debugger(iracedump) ## ## See help(dump.frames) for more details. -irace.dump.frames <- function() +irace_dump_frames <- function() { execDir <- getOption(".irace.execdir") if (!is.null(execDir)) cwd <- setwd(execDir) @@ -34,17 +34,18 @@ irace.dump.frames <- function() } # Print an internal fatal error message that signals a bug in irace. -irace.internal.error <- function(...) +irace_internal_error <- function(...) { .irace.bug.report <- paste0(.irace_msg_prefix, "An unexpected condition occurred. ", "Please report this bug to the authors of the irace package ") - op <- options(warning.length = 8170) - if (!base::interactive()) options(error = irace.dump.frames) - on.exit(options(op)) - # 6 to not show anything below irace.assert() - bt <- capture.output(traceback(5)) + op <- list(warning.length = 8170L) + if (!base::interactive()) + op <- c(op, list(error = irace_dump_frames)) + withr::local_options(op) + # 6 to not show anything below irace_assert() + bt <- utils::capture.output(traceback(5)) warnings() stop (.irace_msg_prefix, paste0(..., collapse = "\n"), "\n", paste0(bt, collapse= "\n"), "\n", @@ -52,7 +53,7 @@ irace.internal.error <- function(...) invisible() } -irace.assert <- function(exp, eval_after = NULL) +irace_assert <- function(exp, eval_after = NULL) { # FIXME: It would be great if we could save into a file the state of # the function that called this one. @@ -60,14 +61,14 @@ irace.assert <- function(exp, eval_after = NULL) mc <- sys.call()[[2L]] msg <- paste0("'", deparse(mc), "' is not TRUE") if (!is.null(eval_after)) { - msg_after <- eval.parent(capture.output(eval_after)) + msg_after <- eval.parent(utils::capture.output(eval_after)) msg <- paste0(msg, "\n", paste0(msg_after, collapse="\n")) } - irace.internal.error(msg) + irace_internal_error(msg) invisible() } -irace.note <- function(...) +irace_note <- function(...) { # FIXME: If this was a function within an irace object, we could replace it # when using quiet. @@ -85,50 +86,50 @@ file.check <- function (file, executable = FALSE, readable = executable, READ <- 4 if (!is.character(file) || is.null.or.empty(file)) { - irace.error (text, " ", shQuote(file), " is not a vaild filename") + irace_error (text, " ", shQuote(file), " is not a vaild filename") } file <- path_rel2abs(file) ## The above should remove the trailing separator if present for windows OS ## compatibility, except when we have just C:/, where the trailing separator ## must remain. - + if (!file.exists(file)) { if (writeable) { if (tryCatch({ suppressWarnings(file.create(file) && file.remove(file)) }, error=function(e) FALSE)) return(TRUE) - irace.error("cannot create ", text, " ", shQuote(file)) + irace_error("cannot create ", text, " ", shQuote(file)) return (FALSE) } - irace.error (text, " '", file, "' does not exist") + irace_error (text, " '", file, "' does not exist") return(FALSE) } if (writeable && (file.access(file, mode = WRITE) != 0)) { - irace.error(text, " '", file, "' cannot be written into") + irace_error(text, " '", file, "' cannot be written into") return(FALSE) } - + if (readable && (file.access(file, mode = READ) != 0)) { - irace.error(text, " '", file, "' is not readable") + irace_error(text, " '", file, "' is not readable") return (FALSE) } if (executable && file.access(file, mode = EXEC) != 0) { - irace.error(text, " '", file, "' is not executable") + irace_error(text, " '", file, "' is not executable") return (FALSE) } if (isdir) { if (!file.info(file)$isdir) { - irace.error(text, " '", file, "' is not a directory") + irace_error(text, " '", file, "' is not a directory") return (FALSE) } if (notempty && length(list.files (file, recursive=TRUE)) == 0) { - irace.error(text, " '", file, "' does not contain any file") + irace_error(text, " '", file, "' does not contain any file") return (FALSE) } } else if (file.info(file)$isdir) { - irace.error(text, " '", file, "' is a directory, not a file") + irace_error(text, " '", file, "' is a directory, not a file") return (FALSE) } return (TRUE) @@ -183,10 +184,10 @@ bytecompile <- function(x) # This function is used to trim potentially large strings for printing, since # the maximum error/warning length is 8170 characters (R 3.0.2) -strlimit <- function(s, limit = 5000) +strlimit <- function(s, limit = 5000L) { if (nchar(s) <= limit) return(s) - paste0(substr(s, 1, limit - 3), "...") + paste0(substr(s, 1L, limit - 3L), "...") } #' Update filesystem paths of a scenario consistently. @@ -225,7 +226,7 @@ test.type.order.str <- function(test.type) t.none =, # Fall-throught t.holm =, # Fall-throught t.bonferroni = "mean value", - irace.internal.error ("test.type.order.str() Invalid value '", + irace_internal_error ("test.type.order.str() Invalid value '", test.type, "' of test.type")) @@ -240,34 +241,41 @@ trim <- function(str) trim_trailing(trim_leading(str)) ## This function takes two matrices x and y and merges them such that the ## resulting matrix z has: -# rownames(z) <- setunion(rownames(x), rownames(y)) and +# rownames(z) <- setunion(rownames(x), rownames(y)) and # rownames(z) <- setunion(rownames(x), rownames(y)) and # z[rownames(x), colnames(x)] <- x and z[rownames(y), colnames(y)] <- y, and # z[i, j] <- NA for all i,j not in x nor y. merge_matrix <- function(x, y) { - new.cols <- setdiff(colnames(y), colnames(x)) - new.rows <- setdiff(rownames(y), rownames(x)) + rownames_x <- rownames(x) + colnames_x <- colnames(x) + rownames_y <- rownames(y) + colnames_y <- colnames(y) - if (is.null(rownames(x)) || is.null(colnames(x))) + if (is.null(rownames_x) || is.null(colnames_x)) return(y) - if (is.null(rownames(y)) || is.null(colnames(y))) + if (is.null(rownames_y) || is.null(colnames_y)) return(x) - # Add columns - x <- cbind(x, - matrix(NA, ncol = length(new.cols), nrow = nrow(x), - dimnames = list(rownames(x), new.cols))) - # Add rows - x <- rbind(x, - matrix(NA, ncol = ncol(x), nrow = length(new.rows), - dimnames = list(new.rows, colnames(x)))) - # Update - x[rownames(y), colnames(y)] <- y + row_union <- union(rownames_x, rownames_y) + col_union <- union(colnames_x, colnames_y) + + z <- matrix(NA_real_, nrow = length(row_union), ncol = length(col_union), + dimnames = list(row_union, col_union)) + + # Map row and column names to indices for efficient assignment. + row_idx_x <- chmatch(rownames_x, row_union) + col_idx_x <- chmatch(colnames_x, col_union) + row_idx_y <- chmatch(rownames_y, row_union) + col_idx_y <- chmatch(colnames_y, col_union) + + z[row_idx_x, col_idx_x] <- x + z[row_idx_y, col_idx_y] <- y + # There must be a non-NA entry for each instance. - irace.assert(all(rowAnys(!is.na(x)))) - return(x) + irace_assert(all(rowAnyNotNAs(z))) + return(z) } # FIXME: This may not work when working interactively. For example, @@ -281,7 +289,7 @@ merge_matrix <- function(x, y) ## print(loadedNamespaces()) ## try(print(as.list(get(".__NAMESPACE__.", envir = asNamespace("irace", base.OK = FALSE), ## inherits = FALSE))$path)) -## try(print(path.package("irace"))) +## try(print(path.package("irace"))) # # That is, neither R_LIBS, .libPaths or whether library was called with lib.loc # will affect the slaves. It also happens before we can set those variables on @@ -292,15 +300,14 @@ mpiInit <- function(nslaves, debugLevel = 0) { # Load the Rmpi package if it is not already loaded. if ("Rmpi" %not_in% loadedNamespaces()) { - if (! suppressPackageStartupMessages - (requireNamespace("Rmpi", quietly = TRUE))) - irace.error("The 'Rmpi' package is required for using MPI") - + if (!suppressPackageStartupMessages(requireNamespace("Rmpi", quietly = TRUE))) + irace_error("The 'Rmpi' package is required for using MPI") + # When R exits, finalize MPI. reg.finalizer(environment(Rmpi::mpi.exit), function(e) { # Rmpi already prints a message, so we don't need this. # cat("# Finalize MPI...\n") - if (Rmpi::mpi.comm.size(1) > 0) + if (Rmpi::mpi.comm.size(1) > 0L) # FIXME: dellog == TRUE tries to delete log files, but it does # not take into account that we may have changed directory and # it does not fail gracefully but produces an annoying: @@ -310,8 +317,8 @@ mpiInit <- function(nslaves, debugLevel = 0) # what mpi.finalize does, minus the annoying message: "Exiting Rmpi. Rmpi # cannot be used unless relaunching R", which we do not care about # because this finalizer should only be called when exiting R. - capture.output(Rmpi::mpi.finalize(), - file = if (.Platform$OS.type == 'windows') 'NUL' else '/dev/null') + utils::capture.output(Rmpi::mpi.finalize(), + file = if (.Platform$OS.type == 'windows') 'NUL' else '/dev/null') }, onexit = TRUE) # Create slaves @@ -342,16 +349,15 @@ mpiInit <- function(nslaves, debugLevel = 0) # columns. concordance <- function(data) { - irace.assert (is.matrix(data) && is.numeric(data)) + irace_assert (is.matrix(data) && is.numeric(data)) n <- nrow(data) #judges k <- ncol(data) #objects if (n <= 1L || k <= 1L) - return(list(kendall.w = NA, spearman.rho = NA)) + return(list(kendall.w = NA_real_, spearman.rho = NA_real_)) # Get rankings by rows (per instance) - r <- rowRanks(data, ties.method = "average") - R <- colSums2(r) + r <- rowRanks(data, ties.method = "average", useNames=FALSE) TIES <- c(table(r,row(r))) # If everything is tied, then W=1, perfect homogeneity. if (all(TIES == k)) { @@ -359,48 +365,49 @@ concordance <- function(data) } else { # FIXME: This formula seems slightly different from the one in # friedman.test. Why? - TIES <- sum(TIES^3 - TIES) - W <- ((12 * sum((R - n * (k + 1) / 2)^2)) / - ((n^2 * (k^3 - k)) - (n * TIES))) + TIES <- sum(TIES^3L - TIES) + R <- colSums2(r, useNames=FALSE) + W <- ((12 * sum((R - n * (k + 1) / 2)^2L)) / + ((n^2L * (k^3L - k)) - (n * TIES))) } # Spearman's rho - rho <- (n * W - 1) / (n - 1) + rho <- (n * W - 1) / (n - 1L) ## Same as in friedman test #STATISTIC <- n * (k - 1) * W #PARAMETER <- k - 1 #pvalue <- pchisq(PARAMETER, df = PARAMETER, lower.tail = FALSE) list(kendall.w = W, spearman.rho = rho) -} +} ## FIXME: Move this to the manual page. ## FIXME: Reference! Explain a bit what is computed! -# Calculates Performance similarity of instances +# Calculates Performance similarity of instances # data: matrix with the data, instances in rows (judges), configurations # in columns. -# Returns: variance value [0,1], where 0 is a homogeneous set of instances and +# Returns: variance value [0,1], where 0 is a homogeneous set of instances and # 1 is a heterogeneous set. # FIXME: How to handle missing values? dataVariance <- function(data) { - irace.assert (is.matrix(data) && is.numeric(data)) + irace_assert (is.matrix(data) && is.numeric(data)) # LESLIE: should we rank data?? # MANUEL: We should add the option. - if (nrow(data) <= 1L || ncol(data) <= 1L) return(NA) - + if (nrow(data) <= 1L || ncol(data) <= 1L) return(NA_real_) + # Normalize #datamin <- apply(data,1,min,na.rm=TRUE) #datamax <- apply(data,1,max,na.rm=TRUE) - #normdata <- (data - datamin) / (datamax-datamin) - + #normdata <- (data - datamin) / (datamax-datamin) + #standardize - meandata <- rowMeans2(data) - stddata <- rowSds(data) + meandata <- rowMeans2(data, useNames=FALSE) + stddata <- rowSds(data, useNames=FALSE) # If stddata == 0, then data is constant and it doesn't matter as long as it # is non-zero. stddata[stddata == 0] <- 1 - zscoredata <- (data - meandata) / stddata - + zscoredata <- (data - meandata) / stddata + # FIXME: We could log-tranform if needed # Variance of configurations mean(colVars(zscoredata)) @@ -409,7 +416,7 @@ dataVariance <- function(data) runcommand <- function(command, args, id, debugLevel, timeout = 0) { if (debugLevel >= 2L) { - irace.note (command, " ", args, "\n") + irace_note (command, " ", args, "\n") elapsed <- proc.time()["elapsed"] } err <- NULL @@ -433,11 +440,11 @@ runcommand <- function(command, args, id, debugLevel, timeout = 0) if (!is.null(attr(output, "errmsg"))) err <- paste(sep = "\n", err, attr(output, "errmsg")) if (debugLevel >= 2L) - irace.note ("ERROR (", id, "): ", err, "\n") + irace_note ("ERROR (", id, "): ", err, "\n") return(list(output = output, error = err)) } if (debugLevel >= 2L) { - irace.note ("DONE (", id, ") Elapsed wall-clock seconds: ", + irace_note ("DONE (", id, ") Elapsed wall-clock seconds: ", formatC(proc.time()["elapsed"] - elapsed, format = "f", digits = 2), "\n") } @@ -446,7 +453,7 @@ runcommand <- function(command, args, id, debugLevel, timeout = 0) list(output = output, error = NULL) } -# Safe sampling of vector: +# Safe sampling of vector: resample <- function(x, ...) x[sample.int(length(x), ...)] # Rounds up the number x to the specified number of decimal places 'digits'. @@ -458,7 +465,7 @@ ceiling_digits <- function(x, digits) int_div * multiple + ceiling(div - int_div) * multiple } -# ceil.decimal <- function(x, d) { +# ceil.decimal <- function(x, d) { # # get the significant digits in the integer part. # ssd <- x * 10^(d) # # get the non significant digits @@ -479,17 +486,27 @@ is.sub.path <- function(x, dir, n = nchar(dir)) substr(x, 1L, n) == dir # Same as !(x %in% table). Package data.table has %notin%. "%not_in%" <- function(x, table) is.na(match(x, table)) -irace_save_logfile <- function(iraceResults, scenario) +#' Save the log generated by \pkg{{irace}} to a file (by default `irace.Rdata`). +#' +#' This function may be useful if you are manually editing the log data generated by a run of \pkg{irace}. +#' +#' @param iraceResults `list()`\cr Object created by \pkg{irace} and typically saved in the log file `irace.Rdata`. +#' +#' @param logfile `character(1)`\cr Filename to save `iraceResults`. Usually, this is given by `scenario$logFile`. If `NULL` or `""`, no data is saved. +#' +#' @seealso [read_logfile()] +#' @concept analysis +save_irace_logfile <- function(iraceResults, logfile) { - # FIXME: Raul Santa Maria proposed to only save if sufficient times (>= 1 + # FIXME: Raul Santa Maria proposed to only save if sufficient time (>= 1 # minute) has passed since the last save. We would need an option to force # saving the last one. - if (is.null.or.empty(scenario$logFile)) return(invisible()) + if (is.null.or.empty(logfile)) return(invisible()) # Files produced by `saveRDS` (or `serialize` to a file connection) are not # suitable as an interchange format between machines, for example to download # from a website. The files produced by `save` have a header identifying the # file type and so are better protected against erroneous use. - save(iraceResults, file = scenario$logFile, version = 3L) + save(iraceResults, file = logfile, version = 3L) } valid_iracelog <- function(x) @@ -502,7 +519,7 @@ valid_iracelog <- function(x) #' @param filename Filename that contains the log file saved by irace. Example: `irace.Rdata`. #' #' @param name Optional argument that allows overriding the default name of the object in the file. -#' +#' #' @return (`list()`) #' @examples #' irace_results <- read_logfile(system.file("exdata/irace-acotsp.Rdata", package="irace", @@ -513,18 +530,22 @@ valid_iracelog <- function(x) read_logfile <- function(filename, name = "iraceResults") { if (is_na_or_empty(filename)) - irace.error("read_logfile: 'filename' is NULL or NA.") + irace_error("read_logfile: 'filename' is NULL or NA.") # If filename is already the iraceResults object, just return it. - if (valid_iracelog(filename)) return(filename) + if (valid_iracelog(filename)) + return(filename) if (file.access(filename, mode = 4) != 0) - irace.error("read_logfile: Cannot read file '", filename, "'.") - + irace_error("read_logfile: Cannot read file '", filename, "'.") + load(filename) iraceResults <- get0(name, inherits=FALSE) if (!valid_iracelog(iraceResults)) - irace.error("read_logfile: The file '", filename, "' does not contain the '", name, "' object.") - + irace_error("read_logfile: The file '", filename, "' does not contain the '", name, "' object.") + + # data.table recommends doing this after loading a data.table from a file. + setDT(iraceResults$state$experiment_log) + setDT(iraceResults$state$instances_log) iraceResults } @@ -534,8 +555,11 @@ get_log_clean_version <- function(iraceResults) if (is.null(log_version)) log_version <- iraceResults$irace.version if (is.null(log_version)) - return(package_version("0")) - if (length(gregexpr("\\.", log_version)[[1L]]) > 3) + return(package_version("0.0.0")) + if (log_version == "unknown") # This may happen during development. + return(package_version("1000.0.0")) + if (length(gregexpr("\\.", log_version)[[1L]]) > 3L + || grepl("[a-z]", log_version)) log_version <- sub("\\.[^.]*$", "", log_version) package_version(log_version) } @@ -554,15 +578,13 @@ has_testing_data <- function(iraceResults) { ins <- iraceResults$scenario$testInstances exp <- iraceResults$testing$experiments - !(length(ins) == 0L || - (length(ins) == 1L && (is.na(ins) || nchar(ins) == 0L)) || - length(exp) == 0L || !(is.matrix(exp) || is.data.frame(exp))) + !(length(ins) == 0L + || (length(ins) == 1L && (is.na(ins) || ins == "")) + || length(exp) == 0L || !(is.matrix(exp) || is.data.frame(exp))) } do_nothing <- function(...) invisible() -.irace_tolerance <- sqrt(.Machine$double.eps) - seq_nrow <- function(x) seq_len(nrow(x)) clamp <- function(x, lower, upper) pmax.int(lower, pmin.int(x, upper)) @@ -591,3 +613,13 @@ vlast <- function(x) # between machines. runif_integer <- function(size) sample.int(min(2147483647L, .Machine$integer.max), size = size, replace = TRUE) + +unlist_element <- function(x, element) + unlist(lapply(x, "[[", element, exact=TRUE), recursive=FALSE, use.names=FALSE) + +# Extensions of matrixStats +rowAnyNotNAs <- function(x, rows = NULL, cols = NULL, ..., useNames = FALSE) + !rowAlls(x, rows = rows, cols = cols, value = NA, ..., useNames = useNames) + +colAnyNotNAs <- function(x, rows = NULL, cols = NULL, ..., useNames = FALSE) + !colAlls(x, rows = rows, cols = cols, value = NA, ..., useNames = useNames) diff --git a/TODO b/TODO index 32e8abee..fbf3f481 100644 --- a/TODO +++ b/TODO @@ -52,7 +52,7 @@ TODO General (in order of importance): * Give priority to user-provided configurations or to default/preferred parameter values when choosing between similarly performing configurations. - * maxBound should be a floating-point number. + * boundMax should be a floating-point number. * Write a report at the end that checks if the instances are homogeneous and suggests clusters and sub-configurations. One possibility is to replace diff --git a/cleanup b/cleanup index 6bcb463c..4998d921 100755 --- a/cleanup +++ b/cleanup @@ -1,16 +1,18 @@ #!/bin/sh -rm -f irace-Ex.R config.* confdefs.h \ +rm -f irace.Rdata irace-Ex.R config.* confdefs.h \ src/*.so src/*.o src/*/*.o src/*.gcno src/*/*.gcno src/*.gcov src/config.h src/symbols.rds \ inst/doc/*.blg inst/doc/*.bbl \ - tests/testthat/*.log tests/testthat/*.Rout tests/testthat/Rplots.pdf tests/testthat/iracedump.rda tests/testthat/irace.Rdata + tests/testthat/*.log tests/testthat/*.Rout tests/testthat/Rplots.pdf tests/testthat/iracedump.rda tests/testthat/irace.Rdata \ + examples/vignette-example/Rplots.pdf \ + devel-examples/vignette-example/*.stderr \ + devel-examples/vignette-example/*.stdout \ + devel-examples/vignette-example/irace-acotsp.Rdata \ + devel-examples/vignette-example/examples.Rdata \ + devel-examples/vignette-example/irace-acotsp-stdout.txt \ + devel-examples/vignette-example/mean.pdf \ + devel-examples/vignette-example/sann.rda -rm -fr tests/testthat/run_* tests/testthat/multi_irace* +rm -fr tests/testthat/run_* tests/testthat/multi_irace* \ + autom4te.cache find . -name '*.orig' -o -name '.Rhistory' -o -name '*.rej' | xargs rm -f - -rm -rf autom4te.cache - -rm -f examples/vignette-example/Rplots.pdf -rm -f devel-examples/vignette-example/*.stderr devel-examples/vignette-example/*.stdout - - diff --git a/cran-comments.md b/cran-comments.md index f1c98991..7c908177 100644 --- a/cran-comments.md +++ b/cran-comments.md @@ -1,10 +1,10 @@ Dear CRAN maintainers, -I would like to submit version $VERSION of the irace package to CRAN. +I would like to submit version 4.0 of the irace package to CRAN. The NEWS section corresponding to this version is available from: -https://mlopez-ibanez.github.io/irace/news/index.html#irace-35 +https://mlopez-ibanez.github.io/irace/news/index.html More information about irace may be found at: diff --git a/devel-examples/vignette-example/create-example-file.R b/devel-examples/vignette-example/create-example-file.R index 9eb6cc2c..47c16ce3 100644 --- a/devel-examples/vignette-example/create-example-file.R +++ b/devel-examples/vignette-example/create-example-file.R @@ -1,60 +1,11 @@ library(irace) +library(fs) +library(withr) -iracebin <- system.file(package="irace", "bin/irace") -if (0 != file.access(iracebin, mode=1)) - stop("Error: ", iracebin, " is not executable or not found!") - -system(paste0("nice -n 19 ", iracebin, " --parallel 2 | tee irace-acotsp-stdout.txt 2>&1")) - -iraceResults <- read_logfile("irace-acotsp.Rdata") - -# Change paths -# FIXME: Use irace::scenario_update_paths() -to_change <- c("logFile", "trainInstancesDir", "trainInstancesFile", - "testInstancesDir", "testInstancesFile", "parameterFile", - "targetRunner") -iraceResults$scenario[to_change] <- - lapply(iraceResults$scenario[to_change], - function(x) paste0("./", basename(x))) - -iraceResults$scenario$execDir <- "./" -iraceResults$scenario$instances <- - paste0(iraceResults$scenario$trainInstancesDir, "/", - basename(iraceResults$scenario$instances)) -iraceResults$scenario[["testInstances"]] <- - paste0(iraceResults$scenario$testInstancesDir, "/", - basename(iraceResults$scenario$testInstances)) - -save(iraceResults, file = "irace-acotsp.Rdata", version = 3L) - -############################### -### Small example file -############################### -# Experiment -res <- get_instanceID_seed_pairs(iraceResults, index = 1L, instances = TRUE) -stopifnot(length(res[["instanceID"]]) == 1L) -stopifnot(length(res[["instance"]]) == 1L) -experiment <- list ( - id_configuration = 1, - id_instance = res[["instanceID"]], - seed = res[["seed"]], - configuration = getConfigurationById(iraceResults, 1L, drop.metadata = TRUE), - instance = res[["instance"]], - switches = iraceResults$scenario$parameters$switches) - -# Output -# FIXME: Create this just using functions from the irace package. -output <- list( - list(cost=iraceResults$experiments[1L,1L], time=as.numeric(iraceResults$experimentLog[1L,"time"])), - list(cost=iraceResults$experiments[1L,2L], time=as.numeric(iraceResults$experimentLog[2L,"time"]))) - -# save in the folder -save(experiment, output, file="examples.Rdata", version = 3L) - -## Create sann.rda +###### Create sann.rda cat('**** Creating sann.rda\n') -## We generate 200 instances (in this case, weights): -weights <- rnorm(200, mean = 0.9, sd = 0.02) +## We generate 50 instances (in this case, weights): +weights <- rnorm(50, mean = 0.9, sd = 0.02) ## On this set of instances, we are interested in optimizing two ## parameters of the SANN algorithm: tmax and temp. We setup the @@ -71,12 +22,11 @@ parameters <- readParameters(text = parameters_table) ## configuration on a single instance. For simplicity, we restrict to ## three-dimensional functions and we set the maximum number of ## iterations of SANN to 5000. - -target.runner <- function(experiment, scenario) +target_runner <- function(experiment, scenario) { instance <- experiment$instance configuration <- experiment$configuration - D <- 3 + D <- 3L par <- runif(D, min=-1, max=1) fn <- function(x) { # Functions to be optimized: @@ -88,7 +38,7 @@ target.runner <- function(experiment, scenario) } f_rastrigin <- function (x) sum(x * x - 10 * cos(2 * pi * x) + 10) - + (instance * f_rastrigin(x) + (1 - instance) * f_rosenbrock(x)) } res <- stats::optim(par,fn, method="SANN", @@ -107,10 +57,10 @@ target.runner <- function(experiment, scenario) } ## We define a configuration scenario by setting targetRunner to the -## function define above, instances to the first 100 random weights, and +## function define above, instances to the first 50 random weights, and ## a maximum budget of 1000 calls to targetRunner. -scenario <- list(targetRunner = target.runner, - instances = weights[1:100], +scenario <- list(targetRunner = target_runner, + instances = weights, maxExperiments = 1000, logFile = "./sann.rda", execDir = "./", @@ -126,7 +76,107 @@ iraceResults$scenario$execDir <- "./" iraceResults$scenario$logFile <- "./sann.rda" save(iraceResults, file="sann.rda", version = 3L) +### ACOTSP example + +download_uncompress <- function(url, exdir, flat = FALSE) { + exts <- c(".tar.bz2", ".tar.gz", ".tgz", ".tar.xz", ".zip") + fileext <- exts[endsWith(url, exts)] + tf <- local_tempfile(fileext = fileext) + download.file(url, destfile = tf) + if (fileext == ".zip") { + unzip(tf, exdir = exdir, junkpaths = flat) + } else { + untar(tf, exdir = exdir, restore_times = FALSE, verbose = TRUE) + if (flat) { + files <- untar(tf, list = TRUE) + withr::with_dir(exdir, { + for (p in files) { + if (fs::is_file(p)) { + fs::file_move(p, ".") + } + } + paths <- fs::dir_ls(".", type = "directory") + for (p in paths) { + n <- length(fs::dir_ls(p, recurse = TRUE, type = "file")) + if (n == 0L) fs::dir_delete(p) + } + }) + } + } +} + +invoke_make <- function(args) { + system2("make", args = args) +} + +# Problem instances +setup_tsp_rue_2000 <- function() { + exdir <- fs::path_abs("acotsp") + if (!fs::file_exists(exdir)) { + download_uncompress("https://iridia.ulb.ac.be/supp/IridiaSupp2016-003/scenarios/acotsp/instances.tar.gz", + exdir = exdir) + } +} + +setup_acotsp <- function() { + exdir <- fs::path_abs("acotsp") + download_uncompress("https://github.com/MLopez-Ibanez/ACOTSPQAP/archive/refs/heads/master.zip", exdir = exdir, flat= TRUE) + invoke_make(c("-C", exdir, "acotsp")) +} + +setup_tsp_rue_2000() +setup_acotsp() + +iracebin <- system.file(package="irace", "bin/irace") +if (0 != file.access(iracebin, mode=1)) + stop("Error: ", iracebin, " is not executable or not found!") +system(paste0("nice -n 19 ", iracebin, " --parallel 2 | tee irace-acotsp-stdout.txt 2>&1")) + # Create log-ablation.Rdata cat('**** Running ablation("irace-acotsp.Rdata")\n') -ablation("irace-acotsp.Rdata", parallel = 1) +ablation("irace-acotsp.Rdata", parallel = 2L) + +iraceResults <- read_logfile("irace-acotsp.Rdata") + +# Change paths +# FIXME: Use irace::scenario_update_paths() +to_change <- c("logFile", "trainInstancesDir", "trainInstancesFile", + "testInstancesDir", "testInstancesFile", "parameterFile", + "targetRunner") +iraceResults$scenario[to_change] <- + lapply(iraceResults$scenario[to_change], + function(x) paste0("./", basename(x))) + +iraceResults$scenario$execDir <- "./" +iraceResults$scenario$instances <- + paste0(iraceResults$scenario$trainInstancesDir, "/", + basename(iraceResults$scenario$instances)) +iraceResults$scenario[["testInstances"]] <- + paste0(iraceResults$scenario$testInstancesDir, "/", + basename(iraceResults$scenario$testInstances)) + +save(iraceResults, file = "irace-acotsp.Rdata", version = 3L) + +############################### +### Small example file +############################### +# Experiment +res <- get_instanceID_seed_pairs(iraceResults, index = 1L, instances = TRUE) +stopifnot(length(res[["instanceID"]]) == 1L) +stopifnot(length(res[["instance"]]) == 1L) +experiment <- list ( + id_configuration = 1, + id_instance = res[["instanceID"]], + seed = res[["seed"]], + configuration = getConfigurationById(iraceResults, 1L, drop.metadata = TRUE), + instance = res[["instance"]], + switches = iraceResults$scenario$parameters$switches) + +# Output +# FIXME: Create this just using functions from the irace package. +output <- list( + list(cost=iraceResults$experiments[1L,1L], time=as.numeric(iraceResults$experimentLog[1L,"time"])), + list(cost=iraceResults$experiments[1L,2L], time=as.numeric(iraceResults$experimentLog[2L,"time"]))) +# save in the folder +save(experiment, output, file="examples.Rdata", version = 3L) diff --git a/devel-examples/vignette-example/instances-test.txt b/devel-examples/vignette-example/instances-test.txt deleted file mode 100644 index 1b650022..00000000 --- a/devel-examples/vignette-example/instances-test.txt +++ /dev/null @@ -1,10 +0,0 @@ -1000-10.tsp -1000-11.tsp -1000-12.tsp -1000-13.tsp -1000-14.tsp -1000-15.tsp -1000-16.tsp -1000-17.tsp -1000-18.tsp -1000-19.tsp diff --git a/devel-examples/vignette-example/instances.txt b/devel-examples/vignette-example/instances.txt deleted file mode 100644 index 1d1fbe37..00000000 --- a/devel-examples/vignette-example/instances.txt +++ /dev/null @@ -1,10 +0,0 @@ -1000-1.tsp -1000-2.tsp -1000-3.tsp -1000-4.tsp -1000-5.tsp -1000-6.tsp -1000-7.tsp -1000-8.tsp -1000-9.tsp -1000-10.tsp diff --git a/devel-examples/vignette-example/instances/1000-1.tsp b/devel-examples/vignette-example/instances/1000-1.tsp deleted file mode 100644 index a7819293..00000000 --- a/devel-examples/vignette-example/instances/1000-1.tsp +++ /dev/null @@ -1,1006 +0,0 @@ -NAME : portgen-1000-1 -COMMENT : portgen N=1000, seed=1 -TYPE : TSP -DIMENSION : 1000 -EDGE_WEIGHT_TYPE : EUC_2D -NODE_COORD_SECTION -1 921674 250065 -2 377506 896048 -3 169771 218380 -4 877708 813387 -5 123344 400022 -6 30588 418434 -7 113951 210517 -8 924356 970453 -9 194244 23254 -10 45333 569958 -11 264618 206470 -12 212018 319685 -13 423427 581180 -14 212210 21951 -15 733992 691156 -16 264308 7229 -17 755721 348018 -18 798956 285928 -19 200563 209181 -20 779738 937621 -21 667044 750997 -22 958577 165463 -23 368408 61344 -24 927938 354116 -25 976287 819675 -26 632837 412885 -27 703268 817667 -28 154174 914445 -29 494344 29488 -30 97091 883843 -31 17816 668526 -32 33648 185722 -33 732978 279590 -34 459856 948487 -35 842108 863012 -36 42515 840128 -37 46967 225658 -38 937120 851733 -39 503201 394350 -40 165511 508982 -41 86836 182722 -42 924859 850149 -43 673339 595781 -44 973580 569998 -45 615040 519366 -46 826071 252076 -47 367072 916726 -48 895105 826915 -49 704030 732919 -50 228343 516675 -51 558142 533587 -52 188605 467305 -53 732838 450115 -54 488025 853119 -55 144328 558392 -56 940864 924346 -57 414448 577725 -58 57771 765740 -59 301453 116922 -60 290616 906062 -61 575559 726937 -62 720144 325433 -63 304869 508928 -64 651523 579662 -65 492819 487005 -66 363707 650082 -67 250022 607118 -68 568117 162490 -69 768274 347133 -70 792377 907598 -71 294327 856658 -72 279381 708977 -73 943806 99133 -74 531931 41639 -75 611856 386177 -76 175392 124368 -77 240099 741337 -78 152967 908059 -79 283564 581486 -80 899187 570348 -81 681841 140892 -82 60741 236730 -83 264064 84206 -84 644964 705470 -85 633919 958637 -86 233808 259814 -87 505065 904439 -88 730669 451191 -89 486837 978806 -90 172465 396809 -91 225363 70037 -92 680475 922471 -93 805164 222815 -94 589341 13292 -95 343054 483911 -96 517525 62803 -97 713214 833739 -98 673789 34513 -99 351592 374942 -100 978307 492614 -101 612296 553125 -102 869173 215046 -103 160814 105355 -104 443893 317628 -105 936173 930152 -106 318718 270272 -107 238431 415275 -108 52822 619037 -109 427677 227001 -110 562940 229551 -111 732613 270022 -112 727162 141304 -113 346341 680683 -114 390640 290019 -115 743625 625314 -116 7298 169209 -117 42632 242312 -118 78091 955091 -119 831605 265199 -120 869648 186126 -121 795137 362339 -122 450351 113503 -123 751298 247503 -124 335641 571909 -125 487398 993106 -126 643872 61573 -127 631316 352993 -128 485316 443086 -129 510492 626860 -130 136955 205723 -131 273749 178693 -132 447979 750046 -133 135014 956379 -134 819920 124928 -135 663977 805319 -136 283396 855767 -137 739603 569834 -138 585678 671040 -139 638705 374169 -140 655988 903255 -141 170190 763780 -142 153063 537902 -143 351565 828605 -144 721230 292586 -145 107298 121712 -146 135170 706677 -147 601222 64329 -148 902730 939370 -149 622736 880517 -150 527824 80257 -151 608797 961471 -152 915921 584143 -153 822915 880092 -154 908509 93414 -155 1427 656711 -156 721856 217906 -157 519562 15243 -158 70552 567071 -159 577470 383649 -160 847316 195644 -161 333642 939402 -162 597103 583719 -163 196522 321924 -164 939845 155460 -165 746918 705586 -166 762531 545291 -167 372741 999277 -168 181399 952284 -169 244217 137820 -170 467349 784493 -171 251134 337580 -172 445270 911654 -173 788069 195767 -174 109574 17502 -175 867807 580805 -176 999524 467276 -177 133599 822238 -178 317726 63505 -179 588729 916644 -180 402743 870631 -181 635874 770688 -182 626064 216934 -183 405576 384275 -184 772635 607908 -185 227173 874785 -186 457497 559968 -187 515842 266510 -188 196119 866366 -189 805803 774865 -190 265993 133016 -191 733195 23201 -192 752716 876287 -193 69712 991842 -194 919226 155807 -195 593700 797123 -196 179648 636309 -197 910647 592564 -198 326996 691166 -199 821738 178759 -200 715534 921703 -201 389963 334709 -202 751509 734790 -203 847610 976323 -204 714559 257311 -205 752525 325883 -206 144279 432922 -207 322943 605619 -208 690982 999564 -209 860040 33500 -210 889937 714410 -211 562536 593876 -212 892373 305470 -213 484821 122787 -214 808458 781051 -215 418900 219796 -216 151806 548491 -217 22339 940110 -218 988737 300273 -219 700257 147096 -220 185304 70147 -221 131802 885726 -222 265870 879289 -223 234587 585772 -224 743936 605177 -225 107743 204208 -226 882707 40687 -227 759859 495738 -228 769896 841471 -229 312370 811398 -230 746053 547337 -231 276065 567463 -232 72006 682378 -233 194081 258553 -234 167051 443654 -235 371032 105210 -236 255628 254863 -237 925757 685729 -238 831703 521848 -239 834016 396634 -240 535573 643349 -241 810417 997060 -242 34997 871562 -243 943730 584342 -244 476485 339961 -245 746028 730183 -246 133221 256603 -247 776063 80094 -248 814519 876938 -249 959968 580140 -250 47586 712738 -251 751755 347301 -252 69603 464393 -253 393790 885647 -254 5690 888296 -255 552007 185553 -256 364986 972408 -257 65370 15869 -258 414116 19462 -259 791399 991912 -260 867858 317142 -261 298584 586911 -262 396068 658293 -263 353455 908327 -264 185259 461363 -265 291938 946056 -266 516158 945720 -267 844627 350019 -268 278362 838009 -269 931690 19127 -270 457446 924268 -271 792943 484573 -272 472102 428886 -273 431599 546310 -274 860535 117770 -275 726639 906192 -276 691678 498605 -277 288202 101529 -278 196579 806035 -279 502674 719583 -280 186030 555781 -281 953957 986562 -282 430850 627739 -283 392610 880413 -284 500306 636483 -285 584270 867806 -286 158927 673629 -287 265272 961666 -288 625464 799979 -289 298708 294538 -290 461714 547420 -291 405652 465675 -292 275332 736157 -293 992099 529596 -294 514869 216887 -295 957409 397949 -296 337702 295206 -297 434857 589640 -298 765341 119313 -299 219300 510435 -300 803421 631620 -301 247601 565996 -302 656056 179219 -303 500539 226002 -304 223272 552045 -305 109430 666983 -306 291165 285786 -307 762174 788081 -308 218078 658751 -309 551705 841210 -310 862398 273296 -311 661113 989870 -312 833062 952650 -313 620204 592930 -314 17573 86053 -315 461127 399461 -316 576706 746663 -317 185107 794730 -318 256255 119866 -319 703501 487250 -320 518078 333348 -321 977890 673659 -322 354489 684112 -323 736835 347832 -324 462143 482206 -325 969435 172410 -326 101740 133247 -327 49308 403960 -328 54913 500937 -329 380889 861325 -330 922964 380672 -331 522501 736022 -332 33966 776082 -333 689092 617505 -334 931296 78062 -335 51245 870246 -336 196607 69498 -337 871775 689987 -338 171556 527865 -339 940561 429101 -340 897736 119267 -341 212040 156247 -342 163088 80454 -343 876960 840684 -344 712696 409024 -345 105637 638749 -346 188570 625439 -347 436004 647831 -348 136741 908392 -349 801884 664502 -350 512556 208970 -351 407270 33042 -352 584470 850168 -353 960370 945492 -354 970158 968854 -355 527000 214229 -356 788241 971864 -357 755687 284214 -358 192102 897062 -359 300017 386135 -360 639341 780700 -361 815620 266794 -362 565505 842275 -363 462976 163564 -364 485028 21607 -365 729617 226063 -366 557706 971707 -367 902101 683506 -368 331025 240175 -369 400560 878873 -370 888352 979897 -371 540666 326561 -372 769683 324937 -373 823129 921776 -374 59933 593729 -375 184855 973176 -376 423363 780277 -377 934885 286492 -378 651263 435563 -379 130940 900963 -380 519142 720194 -381 544932 91285 -382 80502 547102 -383 673563 461679 -384 202181 430750 -385 461085 270326 -386 837129 706288 -387 201279 666165 -388 357336 35342 -389 331908 279013 -390 191012 27413 -391 32624 584064 -392 502464 9422 -393 681131 466421 -394 891205 354999 -395 9943 869346 -396 37994 969810 -397 417788 618025 -398 142768 834378 -399 125282 103518 -400 967600 787786 -401 589867 780920 -402 402717 157442 -403 940551 839299 -404 277812 925462 -405 605361 184841 -406 544357 775940 -407 891020 649795 -408 682199 575122 -409 673497 462476 -410 404334 839184 -411 336396 98663 -412 463149 673299 -413 680458 56209 -414 303570 43836 -415 725613 518037 -416 757530 406446 -417 673651 6170 -418 483055 256683 -419 693044 852669 -420 327223 106009 -421 792924 428728 -422 950665 170759 -423 532949 939331 -424 506661 744488 -425 937567 86559 -426 530807 81445 -427 377904 449563 -428 30255 183421 -429 107268 396547 -430 674387 683867 -431 146255 425143 -432 598239 499352 -433 391917 115629 -434 825275 720261 -435 116845 742867 -436 68460 929008 -437 524909 317774 -438 308377 254951 -439 720758 13585 -440 643043 497037 -441 948940 907023 -442 369449 41746 -443 371781 332387 -444 808207 174299 -445 614252 367426 -446 431408 972783 -447 735823 584355 -448 37548 863915 -449 903819 632890 -450 862382 277998 -451 218573 493076 -452 101444 440529 -453 137618 623783 -454 711996 336158 -455 77782 697868 -456 375213 932969 -457 782294 306961 -458 252459 173472 -459 689319 13883 -460 461804 528002 -461 211809 192384 -462 857879 838847 -463 524294 575384 -464 827564 84379 -465 180156 684593 -466 542954 384600 -467 935803 945175 -468 121823 15971 -469 124728 62487 -470 789287 198309 -471 643067 794323 -472 712495 86250 -473 155616 239023 -474 114903 896976 -475 60060 462163 -476 36351 819439 -477 452733 177788 -478 735044 833972 -479 557272 156269 -480 318705 121646 -481 499054 649509 -482 546871 879472 -483 54800 580890 -484 220474 696044 -485 151344 13435 -486 58568 792342 -487 953823 999640 -488 491650 392369 -489 739651 680090 -490 103802 690322 -491 18111 671294 -492 765673 58510 -493 185538 893445 -494 837728 56330 -495 890374 540933 -496 795497 428684 -497 911143 775851 -498 139741 850725 -499 840500 712854 -500 594599 763247 -501 499372 434813 -502 793174 369738 -503 444051 365056 -504 53766 394223 -505 992249 841599 -506 996243 500942 -507 265895 777772 -508 326148 70370 -509 738365 771019 -510 739731 204075 -511 740390 507619 -512 101445 388097 -513 514062 623755 -514 999168 584084 -515 555589 126594 -516 338603 345427 -517 687840 262203 -518 694078 517169 -519 405398 987901 -520 732361 115168 -521 155079 66709 -522 316599 686298 -523 800542 287877 -524 327238 523045 -525 261789 515986 -526 851557 256415 -527 157264 468005 -528 424643 153944 -529 746972 530970 -530 675660 926881 -531 959657 65865 -532 661861 877081 -533 686519 929534 -534 184343 579597 -535 977229 38271 -536 743131 215320 -537 509230 223745 -538 352518 483974 -539 350355 633440 -540 963453 360117 -541 876782 468197 -542 908424 628707 -543 166936 272738 -544 683565 810758 -545 575683 764543 -546 332826 825801 -547 10671 694090 -548 372036 939758 -549 557478 92854 -550 333779 316568 -551 937522 693798 -552 559592 901671 -553 639203 383359 -554 347991 528556 -555 301068 151905 -556 470378 936213 -557 955286 911117 -558 594055 133856 -559 55193 967771 -560 505045 746760 -561 372055 91488 -562 245817 660661 -563 100748 49333 -564 655728 607559 -565 584541 969159 -566 135983 821798 -567 332371 811548 -568 889738 940568 -569 512910 997307 -570 34652 33080 -571 217544 715793 -572 305713 828922 -573 392487 241337 -574 579984 350009 -575 593341 322702 -576 284030 949919 -577 508312 364620 -578 180584 115724 -579 361426 748043 -580 11932 698635 -581 870448 350683 -582 493904 267988 -583 934360 754585 -584 630500 126364 -585 518629 352717 -586 553871 705184 -587 374430 182342 -588 462730 422136 -589 583176 881196 -590 480076 985024 -591 687906 907684 -592 595627 885905 -593 98710 785299 -594 327894 64382 -595 877187 135153 -596 310067 386546 -597 478677 681935 -598 479208 512360 -599 341363 123370 -600 366192 970351 -601 658161 698788 -602 869932 608316 -603 634795 376345 -604 354292 622406 -605 265910 395284 -606 787830 297044 -607 870856 876778 -608 388567 483902 -609 872005 811968 -610 788780 422000 -611 413221 507129 -612 760171 548278 -613 694555 855083 -614 835251 766113 -615 547546 86384 -616 67844 960769 -617 615286 84791 -618 197194 390862 -619 36828 718848 -620 497338 614808 -621 913293 515925 -622 275602 455186 -623 721932 802938 -624 626374 930399 -625 987379 624125 -626 677108 575249 -627 575824 279808 -628 902506 697392 -629 83501 785140 -630 411122 243933 -631 339517 635443 -632 125068 651101 -633 481991 271904 -634 21441 415669 -635 154846 585629 -636 857527 941606 -637 824588 164654 -638 744892 837971 -639 931305 480363 -640 645771 997163 -641 771581 50110 -642 354990 303613 -643 746866 432400 -644 835700 964184 -645 602800 925289 -646 369421 621159 -647 564002 911709 -648 757281 971687 -649 691336 110948 -650 710294 883960 -651 871633 146011 -652 284628 990216 -653 852544 626997 -654 220259 272211 -655 532941 470105 -656 861691 119317 -657 182339 485833 -658 874512 718358 -659 71440 213359 -660 893820 510303 -661 580568 910493 -662 705374 270885 -663 713995 711515 -664 656978 834372 -665 312109 117894 -666 617712 659094 -667 947422 175665 -668 135472 652264 -669 867771 869156 -670 429101 28508 -671 360960 622341 -672 70364 92497 -673 344720 458927 -674 915784 293117 -675 197713 45765 -676 314709 856963 -677 798838 592400 -678 266248 212539 -679 198588 108962 -680 854743 200280 -681 759226 351102 -682 843109 504432 -683 109145 239349 -684 48952 89842 -685 141112 415584 -686 802573 778322 -687 15646 848054 -688 195594 723604 -689 111654 112973 -690 4637 501456 -691 512926 548015 -692 979628 111829 -693 358668 266610 -694 815984 442989 -695 66520 896122 -696 603311 777928 -697 728044 13516 -698 225934 582637 -699 606695 222310 -700 896902 621116 -701 347272 802810 -702 288479 696256 -703 532838 766693 -704 877334 687009 -705 233732 999637 -706 396555 755598 -707 42442 958621 -708 596968 981297 -709 623058 829593 -710 278497 526507 -711 632654 826642 -712 192939 519996 -713 68311 999762 -714 489842 319390 -715 315215 428901 -716 846270 424645 -717 879241 4999 -718 104901 757327 -719 505573 21007 -720 514860 377370 -721 643551 986390 -722 164492 540012 -723 263467 776669 -724 584988 208047 -725 945204 226172 -726 92794 287305 -727 907094 468001 -728 774846 922627 -729 923569 283480 -730 591354 775510 -731 261119 856327 -732 172149 856362 -733 356085 410164 -734 591106 502429 -735 695154 820299 -736 396309 415011 -737 884388 52325 -738 433712 345349 -739 919547 724938 -740 745150 145684 -741 76193 206362 -742 728035 539705 -743 167781 989942 -744 252496 22879 -745 648913 694737 -746 166221 3144 -747 325853 694561 -748 981060 228540 -749 102002 112166 -750 106300 918117 -751 857121 860050 -752 462896 799520 -753 149979 886432 -754 559269 367389 -755 300220 784903 -756 670130 900690 -757 634566 896617 -758 609289 257975 -759 530474 477587 -760 875301 127544 -761 308162 478940 -762 396128 777036 -763 963177 536258 -764 952114 84868 -765 902346 547280 -766 786080 552158 -767 424717 960247 -768 475553 175502 -769 571796 831417 -770 930415 909805 -771 459468 774908 -772 147577 521369 -773 386574 687280 -774 607015 548816 -775 731383 444802 -776 276426 17133 -777 209819 559019 -778 132037 304963 -779 435333 502648 -780 323966 974476 -781 314635 727852 -782 436973 390414 -783 325434 895222 -784 753112 113197 -785 510042 922009 -786 650960 981657 -787 746203 430498 -788 851118 291028 -789 269120 837109 -790 644999 658213 -791 100925 449465 -792 760902 927870 -793 232644 58227 -794 115184 34303 -795 634812 580331 -796 422389 458599 -797 321374 8406 -798 258845 477810 -799 28704 717078 -800 670250 95546 -801 418159 769906 -802 903816 73170 -803 343877 826960 -804 256231 281949 -805 326374 73809 -806 189779 401030 -807 867836 743634 -808 552086 856036 -809 406477 428567 -810 131568 847624 -811 866517 36034 -812 442946 414496 -813 503849 881053 -814 77840 673033 -815 86621 24158 -816 34796 987171 -817 510734 571190 -818 468434 699895 -819 581629 17267 -820 375783 376608 -821 651750 686616 -822 902734 787188 -823 713814 386355 -824 15652 906877 -825 504556 377792 -826 399969 355671 -827 630457 646092 -828 60749 430987 -829 259171 332626 -830 604735 643982 -831 245330 238964 -832 906165 949766 -833 422059 503162 -834 498295 80648 -835 29819 165731 -836 840383 499599 -837 924011 753776 -838 447654 510846 -839 405576 796854 -840 353746 72861 -841 621881 745214 -842 68297 442639 -843 778827 795832 -844 81005 560967 -845 149130 965272 -846 201599 500981 -847 987447 210052 -848 536224 152150 -849 762605 148957 -850 339533 202968 -851 980778 218798 -852 553131 431694 -853 755910 654755 -854 287373 187817 -855 867265 264917 -856 349982 698204 -857 183496 639463 -858 442382 744349 -859 251516 696113 -860 413542 269909 -861 740556 349337 -862 741114 826851 -863 184952 621585 -864 946468 492316 -865 997866 792899 -866 223472 217759 -867 929588 88829 -868 722879 923677 -869 561717 428833 -870 256 34477 -871 544316 384891 -872 147425 879221 -873 224715 852262 -874 759866 160596 -875 25099 914638 -876 205681 270289 -877 151091 546634 -878 979495 763019 -879 289209 464301 -880 708815 832232 -881 93037 858539 -882 187560 832788 -883 720601 965090 -884 550779 304274 -885 414747 590120 -886 984483 90919 -887 671013 498903 -888 64227 470267 -889 198246 194480 -890 847355 421932 -891 332376 482167 -892 783501 165633 -893 699861 364932 -894 30198 96800 -895 368227 757788 -896 372898 257443 -897 14086 410136 -898 49993 453396 -899 713878 648521 -900 814993 754448 -901 654015 565385 -902 313241 603167 -903 582262 723514 -904 486787 985457 -905 846772 614563 -906 732821 192408 -907 96073 951026 -908 459334 835594 -909 844453 777424 -910 782794 267205 -911 251212 902257 -912 489281 660299 -913 936104 419097 -914 777678 67846 -915 916640 340713 -916 983479 212788 -917 347708 232792 -918 689111 139967 -919 386093 832474 -920 706299 864266 -921 520479 252774 -922 314006 101022 -923 506576 470641 -924 768162 353786 -925 474032 630896 -926 675717 646031 -927 731881 474280 -928 770968 441227 -929 217677 80449 -930 914055 442295 -931 337420 654312 -932 279158 982506 -933 94084 480047 -934 878402 995050 -935 444450 988692 -936 67432 490666 -937 303392 151898 -938 591487 605181 -939 170375 15001 -940 889330 494876 -941 201420 697229 -942 153791 474345 -943 3292 329166 -944 933629 365201 -945 138707 209064 -946 261565 391042 -947 388024 717606 -948 796833 29812 -949 949381 162108 -950 509535 901395 -951 300265 753161 -952 464456 979155 -953 429475 978488 -954 492240 257535 -955 470987 441802 -956 507598 852475 -957 941741 704991 -958 180729 946378 -959 266287 561552 -960 185672 64272 -961 530665 716293 -962 485515 543054 -963 688427 314271 -964 26210 324237 -965 722422 612999 -966 112940 912840 -967 544013 447528 -968 987278 348944 -969 755488 448799 -970 293616 56914 -971 62878 372077 -972 179529 74435 -973 678398 545272 -974 905526 844969 -975 29178 482562 -976 3601 625144 -977 439686 896535 -978 788454 387424 -979 209147 16927 -980 991876 80530 -981 222999 43440 -982 963919 414072 -983 378923 135520 -984 672946 867305 -985 26592 635457 -986 40851 421317 -987 532373 703110 -988 60670 905520 -989 276607 588979 -990 754600 301002 -991 105124 9282 -992 332360 641891 -993 389999 69500 -994 948921 129941 -995 68605 851758 -996 675998 888182 -997 422207 658158 -998 16063 641561 -999 839704 476419 -1000 13764 772878 diff --git a/devel-examples/vignette-example/instances/1000-10.tsp b/devel-examples/vignette-example/instances/1000-10.tsp deleted file mode 100644 index 42c4823e..00000000 --- a/devel-examples/vignette-example/instances/1000-10.tsp +++ /dev/null @@ -1,1006 +0,0 @@ -NAME : portgen-1000-10 -COMMENT : portgen N=1000, seed=10 -TYPE : TSP -DIMENSION : 1000 -EDGE_WEIGHT_TYPE : EUC_2D -NODE_COORD_SECTION -1 797517 173948 -2 880140 682535 -3 390634 664426 -4 688253 780117 -5 330270 823990 -6 821952 334380 -7 390702 797652 -8 533664 23775 -9 614815 824750 -10 388273 17724 -11 995203 582941 -12 869100 983626 -13 844494 801657 -14 371160 512074 -15 227626 747832 -16 704962 285102 -17 57551 30845 -18 326186 170483 -19 906064 161269 -20 977932 159077 -21 579501 300761 -22 865547 196728 -23 559623 778668 -24 47395 315769 -25 284328 886416 -26 794205 316469 -27 525917 229830 -28 862460 512415 -29 116397 849294 -30 356349 220151 -31 758361 526984 -32 802184 171193 -33 244488 521191 -34 468832 193974 -35 238029 754995 -36 976379 299046 -37 540422 501857 -38 223519 678734 -39 57024 639269 -40 121166 332079 -41 685260 521865 -42 155725 7474 -43 989470 177977 -44 482917 886358 -45 786356 804995 -46 701650 712090 -47 923239 222937 -48 182698 280454 -49 760339 363689 -50 973209 880888 -51 721643 408125 -52 194602 952249 -53 201155 272340 -54 160743 518442 -55 240359 684482 -56 29497 230039 -57 62938 551354 -58 518501 46271 -59 603745 579247 -60 988494 964033 -61 760851 105142 -62 220764 357141 -63 33351 568253 -64 104443 588172 -65 300701 951178 -66 517990 538582 -67 398910 436683 -68 302581 455220 -69 458927 604371 -70 488973 943199 -71 574231 903670 -72 897863 822322 -73 44143 596507 -74 491325 566098 -75 189585 614444 -76 176011 172166 -77 62988 22030 -78 362897 183061 -79 9215 757919 -80 649668 745935 -81 813412 556371 -82 29468 297160 -83 110251 125826 -84 332176 240615 -85 507211 921993 -86 554945 37647 -87 389661 374050 -88 788022 588685 -89 42154 198734 -90 994243 850290 -91 559037 346524 -92 938504 554766 -93 137765 961618 -94 509114 101749 -95 326432 176754 -96 123044 218312 -97 97160 566979 -98 388253 536584 -99 514008 523812 -100 34300 455457 -101 554352 292591 -102 571855 339294 -103 55406 829486 -104 233662 508221 -105 884264 401279 -106 673947 907466 -107 431486 472913 -108 622890 595100 -109 459211 462488 -110 908907 573666 -111 611818 808363 -112 206314 51753 -113 367641 262354 -114 465792 50367 -115 318643 958535 -116 355023 533932 -117 314469 592964 -118 176343 651571 -119 915037 465590 -120 931875 542665 -121 502407 46625 -122 192842 752766 -123 564936 314680 -124 11997 45406 -125 199338 125898 -126 70792 463641 -127 205169 75764 -128 100434 20419 -129 978121 978891 -130 162951 403835 -131 914449 768071 -132 576346 341599 -133 898871 627321 -134 714624 678720 -135 907977 308210 -136 583103 413804 -137 263149 783008 -138 502873 148176 -139 603194 130549 -140 951318 347221 -141 284233 486901 -142 887415 914807 -143 44086 586951 -144 957586 972870 -145 694092 549021 -146 936946 236316 -147 557612 623665 -148 959561 88603 -149 783476 409833 -150 249892 416759 -151 711486 881447 -152 94088 852117 -153 841665 583891 -154 576225 290361 -155 31678 513483 -156 62833 5250 -157 284798 613930 -158 466888 678132 -159 210459 952680 -160 382037 810268 -161 843845 304790 -162 428828 491217 -163 596723 701655 -164 319716 411031 -165 941342 918982 -166 571950 312832 -167 98871 437835 -168 284387 278982 -169 202102 273485 -170 447918 365953 -171 376492 4905 -172 590833 883824 -173 705175 632156 -174 807488 66394 -175 26941 257906 -176 768886 372444 -177 468490 330909 -178 844809 398653 -179 782576 656252 -180 567730 562683 -181 381789 302740 -182 842442 665724 -183 136991 57927 -184 414417 400974 -185 908755 834732 -186 870643 144064 -187 925739 124130 -188 41381 471401 -189 836299 97918 -190 646408 198070 -191 919078 663464 -192 843301 378659 -193 537193 269209 -194 470390 433146 -195 300844 226460 -196 864564 801127 -197 364730 613186 -198 495309 232427 -199 79166 466702 -200 842442 233773 -201 795856 709569 -202 419985 828871 -203 338827 105422 -204 529142 89830 -205 793716 575599 -206 928262 349430 -207 355407 341270 -208 698118 580661 -209 938010 229256 -210 170414 904563 -211 978760 947715 -212 558532 674981 -213 38876 161074 -214 724078 96868 -215 785302 935959 -216 942258 746469 -217 304202 70809 -218 269807 569648 -219 308056 502031 -220 680541 956532 -221 331199 241134 -222 262731 396281 -223 247699 916849 -224 242595 689748 -225 574309 334235 -226 508348 982298 -227 681399 906483 -228 291514 49387 -229 405367 349176 -230 559063 769179 -231 797366 27111 -232 409289 837184 -233 244399 687128 -234 86699 959126 -235 93570 781268 -236 338065 248261 -237 654946 836179 -238 396214 996462 -239 266315 652048 -240 383466 989489 -241 755706 374902 -242 537804 16123 -243 138593 915147 -244 337179 467018 -245 826409 582679 -246 482948 348929 -247 408460 899273 -248 618466 82937 -249 586187 426552 -250 66 251237 -251 650533 590546 -252 306282 584819 -253 578529 133446 -254 444494 665276 -255 767890 376367 -256 712207 938349 -257 522766 976384 -258 286231 448436 -259 618650 510015 -260 218717 161461 -261 100941 660147 -262 959060 842333 -263 130734 747518 -264 941979 70126 -265 257650 262768 -266 551967 601039 -267 884158 7098 -268 277282 817357 -269 852136 561420 -270 729892 690156 -271 296496 827163 -272 248300 664948 -273 481737 822801 -274 389869 566126 -275 768538 870948 -276 140958 516060 -277 168901 737297 -278 699269 49494 -279 706388 299183 -280 307537 761171 -281 281309 883074 -282 935384 77733 -283 79870 885043 -284 690048 857818 -285 494646 463429 -286 58566 52524 -287 741477 347768 -288 20502 584880 -289 491245 221763 -290 143064 81240 -291 41130 642795 -292 762589 496478 -293 981459 668893 -294 665655 806425 -295 927228 392239 -296 127308 994318 -297 66773 266462 -298 631590 243972 -299 85686 900531 -300 644445 896857 -301 331555 168106 -302 423061 687298 -303 829818 498163 -304 753471 672423 -305 755838 30375 -306 383838 899962 -307 371955 915298 -308 633863 286991 -309 816300 668921 -310 446143 835898 -311 799356 789516 -312 213372 597789 -313 131873 890460 -314 629462 54178 -315 517950 522339 -316 831409 818821 -317 465924 112689 -318 697401 141167 -319 270840 847291 -320 862614 694467 -321 852593 996733 -322 360282 91330 -323 592882 337791 -324 780945 468984 -325 134588 741129 -326 614510 31507 -327 382581 122105 -328 65448 512734 -329 702181 310372 -330 989897 688650 -331 227323 906179 -332 809809 61370 -333 177782 387105 -334 539680 280624 -335 322416 296071 -336 506046 347316 -337 534333 705013 -338 221388 767848 -339 406935 91266 -340 532341 619139 -341 188279 319089 -342 64281 829300 -343 295016 925229 -344 9012 404554 -345 934907 310295 -346 601487 990215 -347 524875 566542 -348 188421 505276 -349 462399 655268 -350 869942 825033 -351 930856 689678 -352 936643 515448 -353 552850 295420 -354 967226 553280 -355 827089 140218 -356 503721 297627 -357 375465 679601 -358 87162 237107 -359 381304 243266 -360 872949 672505 -361 924705 884411 -362 410682 497382 -363 365215 816367 -364 410672 18884 -365 152163 925967 -366 800622 853655 -367 264177 392122 -368 115418 890651 -369 943624 384680 -370 742138 57908 -371 543924 765746 -372 531604 262401 -373 385590 717076 -374 579532 27492 -375 201327 372053 -376 94603 443514 -377 503105 943974 -378 24410 77201 -379 425500 544520 -380 400030 662198 -381 351796 582545 -382 811142 769181 -383 596294 737975 -384 766022 113063 -385 294011 370086 -386 657575 353812 -387 41939 500896 -388 577901 481191 -389 381306 466708 -390 472972 288014 -391 390867 866151 -392 618854 489964 -393 574171 218076 -394 42512 494996 -395 795827 377442 -396 124628 830560 -397 90669 372051 -398 400333 190111 -399 723807 30707 -400 684500 904399 -401 335769 112824 -402 554520 913312 -403 981186 228452 -404 824660 13140 -405 369802 806333 -406 34688 930504 -407 748693 22587 -408 537569 521235 -409 491876 439090 -410 368847 406182 -411 14168 735315 -412 428562 389612 -413 34316 544751 -414 799292 128626 -415 519710 349449 -416 656530 368165 -417 96905 666638 -418 253326 460362 -419 117458 596266 -420 952394 52936 -421 726200 603422 -422 126148 389645 -423 363274 389313 -424 401998 701057 -425 337735 855582 -426 390819 595180 -427 510997 335051 -428 247868 967604 -429 15918 887881 -430 659985 520823 -431 110993 228393 -432 60745 316866 -433 80133 431265 -434 804356 359047 -435 659313 148256 -436 119236 790819 -437 101354 513265 -438 15363 418987 -439 224318 93511 -440 141743 66712 -441 528832 911410 -442 468640 998887 -443 238456 428136 -444 307419 780038 -445 586504 822060 -446 656006 758411 -447 936953 804138 -448 933699 935380 -449 502067 612882 -450 374282 944286 -451 164995 308487 -452 559313 271022 -453 326749 479408 -454 126539 512109 -455 96595 819731 -456 660184 235879 -457 301376 837924 -458 864817 352581 -459 291440 256607 -460 383166 144752 -461 929197 191473 -462 984765 715026 -463 983260 810749 -464 231506 830331 -465 186515 535954 -466 292448 712208 -467 996915 322012 -468 406527 292952 -469 610033 630715 -470 134069 885874 -471 136696 50812 -472 396872 441752 -473 892863 464532 -474 773645 221926 -475 820878 122949 -476 703874 671735 -477 426366 838327 -478 651838 452787 -479 311571 237301 -480 864495 33797 -481 869374 495823 -482 378040 210721 -483 683035 609372 -484 839007 859624 -485 945061 400284 -486 578935 69514 -487 435729 260217 -488 440877 257461 -489 765106 146437 -490 63188 530473 -491 499178 994205 -492 965836 152718 -493 666579 796624 -494 334168 786194 -495 638976 797155 -496 453944 750409 -497 685654 733785 -498 306938 67181 -499 615083 136654 -500 874 635401 -501 699426 627208 -502 158738 290404 -503 623771 709669 -504 705899 273648 -505 171748 855213 -506 118619 525376 -507 598324 67339 -508 579853 118965 -509 810169 644254 -510 903782 615853 -511 994288 702353 -512 858749 309660 -513 700858 951727 -514 910775 145324 -515 636445 731208 -516 551562 491458 -517 974689 207974 -518 411854 973801 -519 395880 898497 -520 572865 547614 -521 986455 689913 -522 882411 23122 -523 802867 751591 -524 891659 375993 -525 32927 355210 -526 156405 469759 -527 500208 269666 -528 83838 207967 -529 652518 950763 -530 878550 649969 -531 313789 807401 -532 700783 624134 -533 868757 428705 -534 642964 575201 -535 264471 828261 -536 227305 434175 -537 611327 548571 -538 459447 524529 -539 202144 589083 -540 225821 492890 -541 299209 960012 -542 266773 986476 -543 417419 744161 -544 790674 350555 -545 339216 983148 -546 330836 820678 -547 634025 744603 -548 320308 552280 -549 78586 333840 -550 563675 278337 -551 549447 302575 -552 150172 540036 -553 56001 196393 -554 202985 513732 -555 852246 339677 -556 417293 301963 -557 611546 895401 -558 319132 493110 -559 173375 956180 -560 303826 316477 -561 350119 309124 -562 11526 986133 -563 278814 924730 -564 284002 71290 -565 492570 263054 -566 321544 688411 -567 736836 886143 -568 75597 997245 -569 348466 371372 -570 667343 924309 -571 570785 834494 -572 46729 22739 -573 633029 21712 -574 809152 647891 -575 465788 395577 -576 268277 7295 -577 841269 300620 -578 956793 861035 -579 565738 264028 -580 464439 58756 -581 847927 831613 -582 846389 927937 -583 768891 582798 -584 255234 588807 -585 262372 297420 -586 683957 525483 -587 490391 908248 -588 48200 342823 -589 467855 710905 -590 29340 417778 -591 358991 19974 -592 606851 433814 -593 415127 489931 -594 842022 808899 -595 117251 492798 -596 742011 759658 -597 108999 369922 -598 240351 45301 -599 344103 138480 -600 974538 290205 -601 553856 98246 -602 618551 48010 -603 36586 248302 -604 400443 407455 -605 885493 466862 -606 19013 756839 -607 146776 971640 -608 316745 88268 -609 722613 476466 -610 687586 723590 -611 238695 116753 -612 614269 972166 -613 743563 585711 -614 906932 442380 -615 871662 799898 -616 942380 60399 -617 825412 562477 -618 398764 602152 -619 873197 635211 -620 117069 326858 -621 767317 365555 -622 121312 393661 -623 254103 625257 -624 145389 136833 -625 626358 654640 -626 138369 901722 -627 266818 174640 -628 347825 493457 -629 272834 56073 -630 649245 434434 -631 375105 765232 -632 290386 558634 -633 699544 653458 -634 635526 753114 -635 717536 691487 -636 942878 585780 -637 850107 32946 -638 585220 336972 -639 849934 439628 -640 624341 250106 -641 312876 850858 -642 793134 437227 -643 424793 177148 -644 770012 266778 -645 862933 745306 -646 966625 120082 -647 917675 425581 -648 383980 181537 -649 515447 88366 -650 808441 917131 -651 775323 705760 -652 512491 376252 -653 341763 287511 -654 108587 829591 -655 749847 170677 -656 723444 6056 -657 193140 903939 -658 467809 255022 -659 847556 864804 -660 174653 518007 -661 138010 547160 -662 944673 800404 -663 916163 237117 -664 73288 473855 -665 691182 297709 -666 228384 20343 -667 689781 453663 -668 526661 306820 -669 657717 889195 -670 969418 169770 -671 329591 905208 -672 92124 344925 -673 607296 419464 -674 175409 117270 -675 509418 146862 -676 108249 41592 -677 397183 510732 -678 688746 754980 -679 15979 58827 -680 849590 34942 -681 629793 219392 -682 860172 580076 -683 841085 818236 -684 913931 848214 -685 296642 48344 -686 79613 730286 -687 355386 27791 -688 409758 96417 -689 149977 433941 -690 111658 161183 -691 221138 14460 -692 624264 656240 -693 667915 8992 -694 160170 109705 -695 612578 708425 -696 392888 809502 -697 592553 921073 -698 90157 599305 -699 549821 64332 -700 935167 510878 -701 269487 741467 -702 5612 348235 -703 925724 93788 -704 417328 740943 -705 842816 679754 -706 594809 906274 -707 446249 141165 -708 642053 820290 -709 626838 939098 -710 489918 241780 -711 268414 849599 -712 913047 785763 -713 778857 338145 -714 724674 7151 -715 102066 315969 -716 679089 409034 -717 591124 431904 -718 566373 314864 -719 568210 483098 -720 14186 847624 -721 382153 221072 -722 619786 370797 -723 440010 543289 -724 896455 806789 -725 142216 752011 -726 874631 542670 -727 962266 619198 -728 831789 860453 -729 150342 573707 -730 781861 610860 -731 525578 934229 -732 726756 995191 -733 297600 373737 -734 286487 75452 -735 701155 98763 -736 923834 820049 -737 796882 737906 -738 367149 725743 -739 887333 293848 -740 953974 918404 -741 187802 150966 -742 225289 491206 -743 790391 744860 -744 682277 595933 -745 134304 192635 -746 28376 492757 -747 781943 915422 -748 923789 562104 -749 424189 881879 -750 3648 714267 -751 655956 602607 -752 852815 223811 -753 564209 723664 -754 317381 471059 -755 828807 86928 -756 178175 554409 -757 439403 589225 -758 582483 32820 -759 152285 811333 -760 71401 735495 -761 949547 404608 -762 71804 986887 -763 442806 321227 -764 967233 573070 -765 173697 643484 -766 408362 416273 -767 465041 867045 -768 740229 633393 -769 711562 636063 -770 908723 757571 -771 592574 870944 -772 524531 398808 -773 243088 623768 -774 420953 795055 -775 472616 602562 -776 594871 851119 -777 708181 360163 -778 305904 239683 -779 137565 985770 -780 483582 930815 -781 12102 681317 -782 562335 71236 -783 494354 307231 -784 29878 40595 -785 346137 958715 -786 611866 357229 -787 338717 468839 -788 140624 98427 -789 696426 711640 -790 680983 203123 -791 183662 981463 -792 89487 242882 -793 631382 727044 -794 853937 393805 -795 372691 432997 -796 603515 670967 -797 289926 950008 -798 145704 235345 -799 532226 55691 -800 258184 144660 -801 927341 709312 -802 114072 269492 -803 418900 613407 -804 761632 465299 -805 728781 578859 -806 385745 743760 -807 613078 50584 -808 327300 341135 -809 391391 612327 -810 925531 259008 -811 775004 974186 -812 782410 201477 -813 31373 902553 -814 243157 69225 -815 49939 527216 -816 336795 231127 -817 982859 102123 -818 817378 439901 -819 368384 38902 -820 915581 290246 -821 335653 241610 -822 468273 113682 -823 657993 629328 -824 888556 88449 -825 918635 243150 -826 992188 463001 -827 5751 730967 -828 807864 696214 -829 726453 11949 -830 452113 978998 -831 245023 722730 -832 549717 438534 -833 243205 144135 -834 275486 499396 -835 392590 697972 -836 452578 302942 -837 693692 682381 -838 266820 312002 -839 968434 51443 -840 393612 335158 -841 176100 231208 -842 617111 70941 -843 282193 614065 -844 681409 544324 -845 858917 673243 -846 164415 868988 -847 646311 217609 -848 837668 32711 -849 547917 785892 -850 846862 345991 -851 660893 837112 -852 694836 583476 -853 67049 760980 -854 845890 934810 -855 448774 193799 -856 14805 182128 -857 153031 778870 -858 814583 376034 -859 76418 332108 -860 600866 210494 -861 596217 489594 -862 652534 46599 -863 37078 615465 -864 608105 110216 -865 615331 505840 -866 466112 33624 -867 602669 199813 -868 320352 993972 -869 78176 838241 -870 256358 906158 -871 537646 349300 -872 943458 648423 -873 77025 674821 -874 216454 599712 -875 180530 222202 -876 424605 437701 -877 170560 341021 -878 879879 627269 -879 234443 495022 -880 263123 73077 -881 682803 7648 -882 678452 542615 -883 656152 665505 -884 238669 504608 -885 701844 139761 -886 159580 476706 -887 151577 378664 -888 785888 158516 -889 319033 311512 -890 166719 409809 -891 381022 113083 -892 847092 542253 -893 823036 458463 -894 355171 60053 -895 543661 654847 -896 755302 573568 -897 136397 116596 -898 746577 60940 -899 197722 564794 -900 862534 918509 -901 355787 904941 -902 432992 770721 -903 841180 311522 -904 590609 628306 -905 517984 421415 -906 272097 174390 -907 951361 608276 -908 317775 109235 -909 871251 561856 -910 796037 595211 -911 467782 673875 -912 642073 783334 -913 783974 254639 -914 43714 380856 -915 537484 474365 -916 567907 690726 -917 793527 745304 -918 137711 206631 -919 161722 238816 -920 224478 713801 -921 587212 793315 -922 264016 948449 -923 187064 81426 -924 931494 353062 -925 332622 491938 -926 17225 816866 -927 27310 388169 -928 350602 665061 -929 111414 687688 -930 633010 634548 -931 149800 351793 -932 403828 804183 -933 834203 478782 -934 910373 2911 -935 421211 236348 -936 177740 518188 -937 229233 304098 -938 577985 650916 -939 646565 253904 -940 432731 118912 -941 143224 356026 -942 747845 902935 -943 324564 216113 -944 286898 989344 -945 911100 658928 -946 296257 158810 -947 817604 988130 -948 536061 69023 -949 564081 959917 -950 370464 536148 -951 434861 677590 -952 920330 213710 -953 348714 661199 -954 69020 124374 -955 63604 134488 -956 378163 122069 -957 776587 974081 -958 338290 990989 -959 534188 415697 -960 268121 765179 -961 914700 950456 -962 632446 885062 -963 801486 500149 -964 597857 15523 -965 955384 916786 -966 581896 522191 -967 190299 298243 -968 740748 21155 -969 579439 773764 -970 564645 333574 -971 681925 871200 -972 721222 145921 -973 744228 345801 -974 526364 932542 -975 186643 35912 -976 471166 548557 -977 4533 453677 -978 954252 912670 -979 487290 622087 -980 472961 327558 -981 81760 295256 -982 559728 730029 -983 452563 506962 -984 400846 630666 -985 229853 992488 -986 464625 601646 -987 229053 232209 -988 294013 366142 -989 945923 178768 -990 930810 888816 -991 12858 975770 -992 542562 627825 -993 835025 286640 -994 962462 460270 -995 845680 233786 -996 620308 948772 -997 543910 572157 -998 868949 80279 -999 642147 489013 -1000 851907 378085 diff --git a/devel-examples/vignette-example/instances/1000-11.tsp b/devel-examples/vignette-example/instances/1000-11.tsp deleted file mode 100644 index 70564694..00000000 --- a/devel-examples/vignette-example/instances/1000-11.tsp +++ /dev/null @@ -1,1006 +0,0 @@ -NAME : portgen-1000-11 -COMMENT : portgen N=1000, seed=11 -TYPE : TSP -DIMENSION : 1000 -EDGE_WEIGHT_TYPE : EUC_2D -NODE_COORD_SECTION -1 894833 498824 -2 269321 547700 -3 192952 602876 -4 333870 554198 -5 242151 648875 -6 798770 991707 -7 532564 85112 -8 934698 251921 -9 772656 580472 -10 426377 511921 -11 965268 291438 -12 830998 835175 -13 669057 603933 -14 499932 122088 -15 726919 754129 -16 198368 315976 -17 757754 995603 -18 606989 824322 -19 873342 378167 -20 999954 405905 -21 791996 584068 -22 744099 422424 -23 358646 413926 -24 60668 755952 -25 985221 671609 -26 589913 639089 -27 61766 275626 -28 941158 578856 -29 741070 273717 -30 940711 368630 -31 729533 955702 -32 554243 836245 -33 856878 214702 -34 247608 110139 -35 726465 520771 -36 191253 16703 -37 595250 754768 -38 922007 326178 -39 229671 555372 -40 894017 90200 -41 862862 226214 -42 181377 358288 -43 24596 242665 -44 761733 921509 -45 138725 392287 -46 576714 763203 -47 651702 479182 -48 214652 775292 -49 988817 989331 -50 500416 32468 -51 184255 505296 -52 861935 895021 -53 808746 363698 -54 457712 703478 -55 251029 698493 -56 817123 819561 -57 134992 548423 -58 791915 966330 -59 304000 75060 -60 621592 81586 -61 225884 258276 -62 609722 693997 -63 336516 685956 -64 154768 700229 -65 946021 558309 -66 868466 526193 -67 304342 195524 -68 273077 43301 -69 91222 632953 -70 566373 58265 -71 938665 686672 -72 299916 57139 -73 166402 318437 -74 153480 957704 -75 142666 528696 -76 620524 288588 -77 43309 942107 -78 164001 658061 -79 200954 666411 -80 621943 765444 -81 272476 824758 -82 137104 192764 -83 759827 130451 -84 519644 77853 -85 382021 473477 -86 812849 346295 -87 932394 92896 -88 461061 937295 -89 214967 667615 -90 529996 678454 -91 485001 488356 -92 78285 180577 -93 285832 43708 -94 389088 111577 -95 435696 142625 -96 523657 13368 -97 250932 92895 -98 245416 592370 -99 754277 207020 -100 596077 229106 -101 103470 485865 -102 427708 464211 -103 43694 132168 -104 210303 862732 -105 656274 120293 -106 268972 89376 -107 230715 479318 -108 241787 259107 -109 573826 44209 -110 947348 167456 -111 376174 312623 -112 481775 152914 -113 370007 326984 -114 918586 468183 -115 49201 328893 -116 726992 352235 -117 11340 409703 -118 409482 395624 -119 257641 598967 -120 938789 26725 -121 469881 344879 -122 164229 268239 -123 766451 211033 -124 531592 98017 -125 722887 918431 -126 673783 286093 -127 157819 267184 -128 502114 751234 -129 474524 18005 -130 54729 648070 -131 874527 611336 -132 923942 629549 -133 650411 924093 -134 925147 962475 -135 712866 30753 -136 727515 475808 -137 321321 28916 -138 493673 90080 -139 154804 214591 -140 650800 618772 -141 852460 900581 -142 413454 401130 -143 454366 115656 -144 428292 381791 -145 759291 485388 -146 470477 295165 -147 886100 908036 -148 299209 994073 -149 23558 135313 -150 774566 676371 -151 56229 317000 -152 447217 104115 -153 65971 773202 -154 872638 756688 -155 812817 386457 -156 322942 92733 -157 258713 569340 -158 177592 579361 -159 725235 15906 -160 330339 656337 -161 900535 789833 -162 187909 36495 -163 974524 410514 -164 28590 217206 -165 962945 720470 -166 217441 398115 -167 401774 264342 -168 295830 759726 -169 641867 844114 -170 223537 875004 -171 390420 412386 -172 51451 102954 -173 584853 680643 -174 107256 849604 -175 933512 888695 -176 965482 806351 -177 172367 54095 -178 458929 658114 -179 915226 182875 -180 808284 306244 -181 131335 28524 -182 533151 937812 -183 996037 910555 -184 41282 155759 -185 984486 496949 -186 472104 875630 -187 82394 441644 -188 690855 94183 -189 617466 133813 -190 577566 316409 -191 495287 845715 -192 408922 656701 -193 589135 188917 -194 864964 463961 -195 268305 385275 -196 718444 486107 -197 859628 726588 -198 402900 514789 -199 329992 609807 -200 412098 490670 -201 63177 973443 -202 272038 617102 -203 393407 119766 -204 397429 515666 -205 464959 270012 -206 793149 451264 -207 914570 423009 -208 587799 645227 -209 168895 806562 -210 534912 481247 -211 580563 431475 -212 743660 493815 -213 433771 498661 -214 603592 465292 -215 48236 571088 -216 696753 101799 -217 668853 307553 -218 523259 44023 -219 931145 985913 -220 68901 943908 -221 20021 58401 -222 929049 787057 -223 804711 286969 -224 742446 365812 -225 292816 904238 -226 911197 864699 -227 561570 841010 -228 793916 961377 -229 304589 964484 -230 93842 349384 -231 188621 411516 -232 446765 521051 -233 249990 734748 -234 522215 127512 -235 618297 300830 -236 902780 803083 -237 513745 630673 -238 570049 715863 -239 869904 902650 -240 699898 472394 -241 194072 639107 -242 371449 698852 -243 382466 285236 -244 655034 147802 -245 57563 788511 -246 521807 803632 -247 367615 768070 -248 41127 216937 -249 544656 298375 -250 217008 88848 -251 417064 839796 -252 665913 820422 -253 710166 272090 -254 493250 862718 -255 458543 508679 -256 306342 156787 -257 906921 305330 -258 827577 384988 -259 43900 678694 -260 479923 33052 -261 190091 223840 -262 910504 529449 -263 883765 62984 -264 137170 693322 -265 920506 297959 -266 222613 7186 -267 444106 191218 -268 166051 37284 -269 732185 66119 -270 871274 997477 -271 241335 976340 -272 667878 24510 -273 598419 297967 -274 893128 838166 -275 884305 978143 -276 79767 851333 -277 377868 919049 -278 866234 409878 -279 395689 474694 -280 654371 672881 -281 539904 427130 -282 991443 461065 -283 267343 330002 -284 488908 882410 -285 706910 529610 -286 491860 205734 -287 794388 501780 -288 953284 338758 -289 845971 991454 -290 663214 473886 -291 667294 662475 -292 38951 247624 -293 758054 795482 -294 15742 983040 -295 923874 836049 -296 548375 849775 -297 359208 341664 -298 505617 35601 -299 181951 166098 -300 71225 259661 -301 451995 901673 -302 174951 410418 -303 310848 417292 -304 812382 130243 -305 160994 70752 -306 394135 412648 -307 550819 818322 -308 124506 690129 -309 67922 649778 -310 955448 231742 -311 148050 322810 -312 811185 447249 -313 77614 590186 -314 30783 383970 -315 190931 535992 -316 526376 715728 -317 830460 592462 -318 79750 254645 -319 111655 220628 -320 123118 67925 -321 727560 365963 -322 27592 692132 -323 687998 225564 -324 38589 911959 -325 264049 915431 -326 4817 797981 -327 975167 535233 -328 733285 736267 -329 71213 582488 -330 330667 56203 -331 305636 591753 -332 7124 93069 -333 343192 28172 -334 385056 858687 -335 130323 898941 -336 651539 155962 -337 385729 40016 -338 226924 350069 -339 347643 275951 -340 713963 341347 -341 518972 448294 -342 53303 134728 -343 230355 934623 -344 708603 737390 -345 249270 51578 -346 869589 252968 -347 90305 224177 -348 416385 571597 -349 980234 987575 -350 465207 337929 -351 877921 762637 -352 197995 922702 -353 396459 556523 -354 744678 840438 -355 304877 798661 -356 27663 333823 -357 333218 279088 -358 186613 52668 -359 501447 782947 -360 676683 771594 -361 47937 397480 -362 393480 792394 -363 21020 888901 -364 957966 463026 -365 643557 670401 -366 605391 507204 -367 971074 915301 -368 313683 185149 -369 115075 774214 -370 948114 177687 -371 433175 925655 -372 60706 477675 -373 3641 472109 -374 859488 297910 -375 203157 527483 -376 613630 517207 -377 344018 794805 -378 732538 370716 -379 791563 282694 -380 609018 211310 -381 441447 970463 -382 892324 127189 -383 365486 102008 -384 273116 855542 -385 275447 714504 -386 193180 203537 -387 579790 149200 -388 157963 530729 -389 53461 598674 -390 59856 650303 -391 97338 675272 -392 854008 432247 -393 228953 634927 -394 614880 843884 -395 549815 211675 -396 912032 259533 -397 498767 233610 -398 984507 229637 -399 345865 911506 -400 319711 472911 -401 418647 260813 -402 238053 552853 -403 430145 938357 -404 663199 911771 -405 565851 97610 -406 755836 947678 -407 732878 397342 -408 299277 181914 -409 471696 658713 -410 142682 135849 -411 756142 361609 -412 535830 802535 -413 295856 932366 -414 965483 26936 -415 719054 219606 -416 867530 141689 -417 32822 962246 -418 894467 149660 -419 942393 456666 -420 132969 47039 -421 163230 956166 -422 701202 413966 -423 455532 550422 -424 723702 696231 -425 937754 52140 -426 264154 318928 -427 192451 100105 -428 605380 276957 -429 227991 275807 -430 658385 280485 -431 995963 206532 -432 778802 518812 -433 934379 799669 -434 246476 318912 -435 941809 748854 -436 458212 775465 -437 720959 90542 -438 871694 437214 -439 169158 435725 -440 197154 18898 -441 704375 689675 -442 368551 438569 -443 223642 660997 -444 362887 514010 -445 27866 94798 -446 903183 623481 -447 514856 384114 -448 588827 387765 -449 235206 610659 -450 542271 18318 -451 381264 287977 -452 499076 918855 -453 347764 574478 -454 950377 753882 -455 876463 944383 -456 914070 713981 -457 247941 563587 -458 377301 372482 -459 691676 394687 -460 929984 546613 -461 564463 635816 -462 776641 923491 -463 367590 170234 -464 276388 802104 -465 742777 297216 -466 486836 415275 -467 559261 252771 -468 104828 990394 -469 441734 804963 -470 61268 851159 -471 969321 968200 -472 584025 481252 -473 530335 267367 -474 846839 591365 -475 16524 418592 -476 111377 433102 -477 867881 245054 -478 531482 965988 -479 728715 246304 -480 814027 357369 -481 132743 145414 -482 692614 25304 -483 975061 945869 -484 129956 766688 -485 33252 109934 -486 525643 100311 -487 378163 511392 -488 435236 131360 -489 767935 531587 -490 392008 401601 -491 441519 30083 -492 988077 385408 -493 164472 341422 -494 722661 533957 -495 277710 158958 -496 860438 675045 -497 771710 951333 -498 325515 869010 -499 590037 72632 -500 46016 398974 -501 499431 315252 -502 199356 614923 -503 977073 81293 -504 445025 482473 -505 80581 190059 -506 243327 194758 -507 968594 655068 -508 496931 457698 -509 373703 741280 -510 699788 106051 -511 355832 57323 -512 720672 634278 -513 610502 210391 -514 900954 763239 -515 534318 353943 -516 686335 285461 -517 451005 201949 -518 158273 246760 -519 61489 333008 -520 888476 706774 -521 967719 981380 -522 834168 171658 -523 803125 803115 -524 954372 137432 -525 340831 115124 -526 968055 826797 -527 538313 692072 -528 712638 213970 -529 864246 997406 -530 456649 730312 -531 19803 112016 -532 593996 373806 -533 222339 261946 -534 360589 796935 -535 851943 693816 -536 503325 236270 -537 400449 584663 -538 137995 529035 -539 519009 28599 -540 921640 396531 -541 346145 903547 -542 306590 804006 -543 334139 574319 -544 691464 77199 -545 979609 896327 -546 886171 264554 -547 481065 194660 -548 203449 731449 -549 580931 249505 -550 33663 274090 -551 284105 925773 -552 215792 944299 -553 768979 64508 -554 520206 734307 -555 357933 138318 -556 522505 787046 -557 17797 560322 -558 844141 755249 -559 630951 399336 -560 170357 490890 -561 681015 111084 -562 763271 577853 -563 409710 577552 -564 20477 456149 -565 815684 73487 -566 8828 784701 -567 670666 783321 -568 874026 559098 -569 885750 746268 -570 959864 578890 -571 943367 292127 -572 906842 488718 -573 215311 775086 -574 501282 903212 -575 784949 625897 -576 710971 124781 -577 433820 960176 -578 265261 499403 -579 255106 432471 -580 70273 209880 -581 178758 773938 -582 774443 779043 -583 194951 230377 -584 880204 529078 -585 345010 69054 -586 253967 727739 -587 614386 544459 -588 779919 556233 -589 677263 803095 -590 312591 910306 -591 322445 588006 -592 385876 605803 -593 894729 234890 -594 10258 891623 -595 588370 643648 -596 678893 356671 -597 401257 890810 -598 324922 215628 -599 677741 362382 -600 708798 659078 -601 97823 698186 -602 590620 874643 -603 303451 122964 -604 738904 828016 -605 65447 30371 -606 489145 363483 -607 844101 426624 -608 530986 822086 -609 372680 883633 -610 454120 979322 -611 552636 517822 -612 820279 685932 -613 971231 555780 -614 137118 739743 -615 241008 656955 -616 817328 849247 -617 737648 282220 -618 421161 958961 -619 743905 959251 -620 74816 72642 -621 862209 126625 -622 437502 609047 -623 91011 161071 -624 536391 715325 -625 919578 769141 -626 78509 937997 -627 121373 51843 -628 841749 248575 -629 960537 308399 -630 453481 344489 -631 379059 779652 -632 753199 992804 -633 168161 362519 -634 925980 235053 -635 335612 369915 -636 285695 657355 -637 964054 684978 -638 900812 614639 -639 396448 768436 -640 844182 722655 -641 595243 828719 -642 286261 896518 -643 277895 37675 -644 96047 744844 -645 114058 58642 -646 32981 508852 -647 623638 704901 -648 786946 204854 -649 162571 752523 -650 708235 476372 -651 764622 767955 -652 871143 196923 -653 173898 249790 -654 651735 224855 -655 773947 804074 -656 152528 215693 -657 194340 394839 -658 311508 870207 -659 156013 48298 -660 205857 963307 -661 199948 173456 -662 526818 859240 -663 605293 517740 -664 786211 767131 -665 511079 651021 -666 962903 171593 -667 994488 40108 -668 570127 379549 -669 634378 891422 -670 585009 407688 -671 881661 47749 -672 538986 150751 -673 858693 859524 -674 982034 764398 -675 99607 269206 -676 418642 395440 -677 241444 57213 -678 513468 593029 -679 773467 831035 -680 626796 794349 -681 615412 760313 -682 639846 366259 -683 922412 104779 -684 676707 43589 -685 536145 451984 -686 888173 391615 -687 948690 936650 -688 544665 804507 -689 932012 469604 -690 345771 12263 -691 744272 955176 -692 140335 716730 -693 35609 202590 -694 531746 628229 -695 117695 465347 -696 702841 590788 -697 355276 133024 -698 519515 490046 -699 99058 602335 -700 606086 54185 -701 927511 512429 -702 418626 87343 -703 524933 463465 -704 255105 524713 -705 21603 310878 -706 61282 145237 -707 713339 161448 -708 91507 24623 -709 405036 506821 -710 846743 432366 -711 5721 74372 -712 437503 481959 -713 524473 375743 -714 972988 861346 -715 411716 81199 -716 549402 407299 -717 448000 34893 -718 950981 599035 -719 241836 978886 -720 625223 10986 -721 797554 24925 -722 781485 685329 -723 459626 628469 -724 153285 873316 -725 608550 143771 -726 517057 237711 -727 190618 524886 -728 504783 520211 -729 64428 383733 -730 136362 925898 -731 221629 276218 -732 899490 10617 -733 513324 36356 -734 363752 28010 -735 701821 463037 -736 871337 531719 -737 898270 702971 -738 915309 768009 -739 883753 912616 -740 977176 4262 -741 311315 589255 -742 724984 485818 -743 859570 273184 -744 507808 437383 -745 521569 914624 -746 235282 213826 -747 277064 162185 -748 139648 265834 -749 126655 78514 -750 770020 691616 -751 744716 240669 -752 896139 604288 -753 832456 927801 -754 512727 704800 -755 665316 231598 -756 12402 627044 -757 862163 221737 -758 690616 7802 -759 999153 737305 -760 870968 247489 -761 909701 285237 -762 257990 10204 -763 718321 630668 -764 635580 293981 -765 870514 987507 -766 255282 178953 -767 247299 745578 -768 991859 684270 -769 727092 503246 -770 795201 851767 -771 274031 770503 -772 566414 274079 -773 4923 950044 -774 955835 266860 -775 443864 508979 -776 630253 832673 -777 207999 782512 -778 436334 565763 -779 993369 150561 -780 612429 148186 -781 200709 9480 -782 909598 813549 -783 957567 241898 -784 60630 588083 -785 216814 740571 -786 51967 732293 -787 293441 361988 -788 617236 77027 -789 77238 475477 -790 573870 152558 -791 637299 485018 -792 681552 722327 -793 786797 245802 -794 269354 433750 -795 788011 749960 -796 623640 139008 -797 286432 54630 -798 799799 541738 -799 477062 204426 -800 656843 927895 -801 872806 480357 -802 692989 291305 -803 871680 145235 -804 151121 485671 -805 995715 190532 -806 296408 559619 -807 362550 862468 -808 524546 61701 -809 723048 854967 -810 13749 415829 -811 764835 856203 -812 931240 288919 -813 867764 571609 -814 39303 2135 -815 490308 472000 -816 925906 591566 -817 479762 383338 -818 856150 77680 -819 122468 819084 -820 197781 725095 -821 522754 414387 -822 420000 372181 -823 985124 767436 -824 207767 997512 -825 186865 228190 -826 502434 474927 -827 714118 184842 -828 1988 281239 -829 595 309651 -830 435155 794000 -831 22766 332037 -832 287890 270619 -833 667777 882021 -834 139618 990368 -835 877343 757110 -836 853933 725535 -837 668101 785559 -838 913394 289908 -839 142085 746397 -840 286931 586524 -841 571013 729651 -842 566979 696308 -843 449234 593869 -844 303676 209142 -845 715560 974129 -846 938061 132100 -847 941740 440671 -848 871161 797218 -849 746285 634440 -850 458787 695216 -851 625350 461370 -852 710581 600341 -853 657176 772782 -854 907947 17809 -855 735608 408119 -856 977563 791453 -857 594090 461026 -858 855938 890666 -859 390296 847219 -860 399458 870559 -861 135735 505178 -862 531580 182126 -863 131759 392563 -864 14954 67760 -865 128383 140611 -866 381961 124275 -867 10789 878811 -868 608960 779560 -869 135560 105953 -870 840369 558567 -871 203572 456456 -872 809684 845001 -873 838393 432883 -874 600519 759613 -875 308911 478598 -876 782264 678525 -877 506057 318175 -878 313255 501075 -879 450580 831769 -880 991380 877615 -881 637221 801993 -882 177440 177040 -883 204546 521106 -884 981768 749089 -885 622632 423055 -886 290147 630682 -887 538307 920859 -888 88295 457210 -889 999120 213404 -890 868871 630684 -891 941982 183184 -892 76380 250767 -893 503389 579967 -894 946835 833748 -895 674265 87854 -896 797791 386471 -897 483320 417314 -898 268420 572889 -899 918149 888824 -900 756706 381183 -901 433762 387114 -902 890742 678227 -903 536615 599079 -904 602144 255290 -905 814786 733288 -906 554239 616832 -907 157504 903526 -908 79824 250750 -909 318673 760126 -910 908620 631656 -911 602957 92944 -912 992383 241449 -913 989292 903032 -914 739940 860079 -915 384244 489215 -916 855065 743830 -917 398618 135583 -918 76444 325150 -919 25680 172853 -920 170943 252638 -921 261293 186709 -922 925128 42608 -923 484897 704847 -924 394087 241871 -925 428021 365388 -926 832949 58070 -927 504580 267490 -928 526117 689932 -929 988496 755159 -930 601783 211465 -931 573398 429290 -932 84346 562147 -933 471994 367530 -934 691704 114895 -935 418629 374976 -936 856663 76802 -937 332105 543231 -938 798707 544887 -939 588364 724893 -940 715332 299360 -941 914536 984781 -942 258296 172779 -943 915817 425774 -944 659483 836471 -945 663588 708913 -946 633446 910785 -947 754224 795966 -948 395975 184491 -949 854603 381896 -950 243900 940009 -951 116483 669194 -952 526539 128660 -953 450851 848167 -954 799773 331800 -955 351672 100342 -956 30448 152024 -957 91570 892869 -958 578019 662613 -959 675066 288380 -960 166171 287502 -961 512926 309807 -962 870994 478619 -963 258493 187469 -964 550263 203444 -965 92380 950540 -966 745113 256563 -967 373220 614989 -968 268912 762511 -969 893211 365426 -970 594760 253203 -971 750708 371103 -972 670299 376086 -973 195986 323638 -974 39790 275605 -975 537473 208506 -976 634228 651159 -977 289516 293360 -978 194896 859919 -979 295973 911549 -980 859748 688339 -981 954956 434346 -982 737040 98469 -983 349634 659345 -984 481724 715484 -985 696882 254380 -986 622823 399460 -987 750906 957664 -988 653273 861767 -989 20291 577634 -990 283722 398573 -991 891495 638713 -992 343695 404040 -993 995584 310766 -994 519522 274751 -995 265355 609566 -996 280787 177727 -997 668544 340379 -998 630380 351248 -999 620196 712635 -1000 722812 334219 diff --git a/devel-examples/vignette-example/instances/1000-12.tsp b/devel-examples/vignette-example/instances/1000-12.tsp deleted file mode 100644 index 57df028e..00000000 --- a/devel-examples/vignette-example/instances/1000-12.tsp +++ /dev/null @@ -1,1006 +0,0 @@ -NAME : portgen-1000-12 -COMMENT : portgen N=1000, seed=12 -TYPE : TSP -DIMENSION : 1000 -EDGE_WEIGHT_TYPE : EUC_2D -NODE_COORD_SECTION -1 992149 823700 -2 658503 412865 -3 995271 541325 -4 979486 328279 -5 154031 473760 -6 775588 649034 -7 674425 372571 -8 335733 480068 -9 930498 336194 -10 464482 6117 -11 935333 999935 -12 792896 686725 -13 493620 406208 -14 628704 732102 -15 226211 760426 -16 691774 346851 -17 457957 960362 -18 887792 478162 -19 840620 595066 -20 21976 652734 -21 4491 867375 -22 622651 648120 -23 157670 49185 -24 73941 196136 -25 686115 456802 -26 385620 961709 -27 597616 321421 -28 19857 645298 -29 365742 698140 -30 525073 517108 -31 700705 384419 -32 306303 501297 -33 469268 908213 -34 26383 26304 -35 214901 286547 -36 406126 734361 -37 650078 7679 -38 620496 973623 -39 402318 471474 -40 666867 848322 -41 40465 930563 -42 207029 709102 -43 59721 307354 -44 40548 956660 -45 491093 979579 -46 451778 814315 -47 380165 735428 -48 246607 270130 -49 217296 614972 -50 27624 184047 -51 646866 602467 -52 529268 837792 -53 416337 455056 -54 754680 888513 -55 261700 712503 -56 604749 409082 -57 207046 545493 -58 65329 886389 -59 4254 570874 -60 254690 199138 -61 690916 411410 -62 998680 30853 -63 639681 803659 -64 205092 812286 -65 591342 165439 -66 218942 513804 -67 209773 954364 -68 243573 631382 -69 723516 661535 -70 643772 173332 -71 303099 469673 -72 701970 291955 -73 288662 40368 -74 815635 349311 -75 95746 442948 -76 65037 405010 -77 23630 862184 -78 965104 133061 -79 392693 574904 -80 594219 784954 -81 731539 93145 -82 244741 88367 -83 409403 135075 -84 707112 915091 -85 256831 24961 -86 70753 654943 -87 475127 811741 -88 134100 285906 -89 387780 136496 -90 65749 506619 -91 410965 630188 -92 218066 806387 -93 433899 125797 -94 269063 121406 -95 544960 108497 -96 924270 808425 -97 404703 618811 -98 102578 648156 -99 994545 890228 -100 157854 2756 -101 652587 679139 -102 283562 589127 -103 31983 434849 -104 186944 217242 -105 428284 839307 -106 863998 271287 -107 29943 485722 -108 860683 923114 -109 688441 625930 -110 985788 761247 -111 140530 816883 -112 757237 254075 -113 372374 391614 -114 371381 886000 -115 779758 699251 -116 98961 170537 -117 708211 226442 -118 642621 139677 -119 600244 732343 -120 945703 510784 -121 437356 643133 -122 135617 783713 -123 967966 107386 -124 51187 150628 -125 246436 710963 -126 276774 108545 -127 110469 458603 -128 903794 482049 -129 970927 57120 -130 946506 892305 -131 834604 454600 -132 271538 917499 -133 401950 220864 -134 135669 246230 -135 517755 753297 -136 871926 537812 -137 379493 274824 -138 484472 31984 -139 706414 298634 -140 350281 890324 -141 420687 314261 -142 939493 887453 -143 864646 644361 -144 898999 790711 -145 824491 421756 -146 4008 354014 -147 214588 192406 -148 638857 899543 -149 263640 860792 -150 299240 935982 -151 400972 752553 -152 800347 356112 -153 290276 962513 -154 169051 223016 -155 593956 259432 -156 583050 180216 -157 232628 524749 -158 888296 480590 -159 240012 79132 -160 278642 502407 -161 957224 274876 -162 946989 581773 -163 352324 119373 -164 737465 23380 -165 984548 521959 -166 862932 483397 -167 704677 90848 -168 307273 240470 -169 81632 414743 -170 999156 384056 -171 404349 819866 -172 512069 322084 -173 464532 729131 -174 407024 632815 -175 840082 519484 -176 162078 240259 -177 876244 777281 -178 73049 917575 -179 47876 709498 -180 48838 49805 -181 880881 754307 -182 223859 209900 -183 855083 763183 -184 668146 910544 -185 60217 159165 -186 73565 607197 -187 239049 759157 -188 340328 716964 -189 398632 169708 -190 508723 434749 -191 71497 27966 -192 974542 934742 -193 641078 108624 -194 259538 494776 -195 235765 544090 -196 572324 171087 -197 354526 839991 -198 310490 797152 -199 580817 752911 -200 981755 747567 -201 330498 237316 -202 124091 405333 -203 447987 134111 -204 265716 941501 -205 136203 964424 -206 658037 553099 -207 473732 504747 -208 477481 709793 -209 399781 383868 -210 899409 57930 -211 182366 915234 -212 928789 312650 -213 828666 836248 -214 483105 833716 -215 311170 206217 -216 451248 457130 -217 33505 544298 -218 776712 518398 -219 554234 469794 -220 457260 931284 -221 708843 875669 -222 595367 177834 -223 361724 657089 -224 242297 41875 -225 11324 474241 -226 314046 747101 -227 441741 775537 -228 296318 873368 -229 203811 579793 -230 628621 929589 -231 579877 795921 -232 484240 204918 -233 255581 782367 -234 957732 295898 -235 143023 820392 -236 467495 357905 -237 372544 425168 -238 743884 435265 -239 473493 153252 -240 16331 955298 -241 632437 903312 -242 205095 381581 -243 626340 655326 -244 972889 828586 -245 288716 994344 -246 560665 258335 -247 326770 636868 -248 463788 350937 -249 503125 170198 -250 433950 926458 -251 183595 89045 -252 25544 56025 -253 841804 410734 -254 542006 60160 -255 149197 640992 -256 900478 375225 -257 291076 634276 -258 368923 321541 -259 469151 847372 -260 741130 904643 -261 279242 787533 -262 861947 216564 -263 636796 378449 -264 332361 316518 -265 583363 333149 -266 893258 413332 -267 4054 375338 -268 54820 257212 -269 612235 570818 -270 12657 304798 -271 186175 125517 -272 87455 384073 -273 715101 773132 -274 396388 110205 -275 71 85338 -276 18576 186606 -277 586835 100801 -278 33200 770262 -279 84991 650205 -280 1205 584592 -281 798499 971187 -282 47502 844398 -283 454817 774961 -284 287769 907003 -285 919174 595790 -286 925153 358945 -287 847300 655791 -288 886066 92636 -289 200698 761146 -290 183364 866533 -291 293458 682155 -292 315313 998771 -293 534650 922071 -294 365829 159656 -295 920521 279859 -296 969443 705231 -297 651643 416867 -298 379645 827229 -299 278216 431664 -300 498006 622465 -301 572434 635241 -302 926840 133538 -303 791879 336421 -304 871292 588063 -305 566151 111129 -306 404433 925335 -307 729684 721346 -308 615149 93267 -309 319543 630635 -310 464753 627587 -311 496744 856104 -312 408997 296708 -313 23355 289911 -314 432104 713762 -315 863911 549645 -316 221343 612634 -317 194995 72235 -318 462100 368123 -319 952471 593966 -320 383622 441382 -321 602527 735194 -322 694902 292934 -323 783114 113338 -324 296234 354934 -325 393511 89733 -326 395125 564454 -327 567752 948360 -328 401122 959800 -329 440246 854605 -330 671437 423756 -331 383950 277326 -332 204440 124768 -333 508602 669238 -334 230432 436750 -335 938231 501810 -336 797032 964609 -337 237124 375019 -338 232461 932289 -339 288351 460637 -340 895586 63555 -341 849665 577499 -342 42324 440155 -343 165695 944017 -344 408193 70227 -345 563633 792861 -346 137691 515721 -347 655735 881811 -348 644349 637918 -349 498069 319883 -350 60472 850825 -351 824986 835597 -352 459348 329955 -353 240068 817626 -354 522130 127597 -355 782665 457105 -356 551606 370018 -357 290971 878576 -358 286064 868228 -359 621591 322628 -360 480418 870684 -361 171169 910548 -362 376277 87406 -363 676824 961435 -364 505260 907168 -365 134951 414835 -366 410159 160754 -367 677971 438481 -368 511949 479646 -369 286527 163748 -370 154090 297466 -371 322426 85565 -372 589808 692949 -373 621692 227142 -374 139443 568328 -375 204987 682914 -376 132657 590900 -377 184932 645637 -378 440665 664231 -379 157625 20867 -380 818005 760421 -381 531099 358381 -382 973506 485198 -383 134679 466041 -384 780209 598021 -385 256883 58921 -386 728785 53262 -387 117641 797503 -388 738026 580268 -389 725616 730640 -390 646741 12592 -391 803809 484393 -392 89163 374529 -393 883736 51778 -394 187247 192773 -395 303802 45908 -396 699436 688505 -397 906865 95168 -398 568680 269163 -399 967923 792305 -400 954922 41424 -401 501525 408802 -402 921587 192394 -403 879104 648263 -404 501737 810402 -405 761900 388887 -406 476983 964852 -407 717064 772097 -408 60985 842593 -409 451516 878337 -410 916517 865515 -411 498117 987904 -412 643099 215459 -413 557396 319982 -414 131674 925246 -415 918398 89763 -416 78530 915214 -417 968739 257853 -418 535609 838957 -419 767328 317065 -420 313544 41142 -421 600261 308910 -422 276255 438286 -423 547791 711531 -424 45406 691405 -425 537772 248698 -426 137488 42676 -427 873906 865159 -428 962893 586311 -429 440063 663733 -430 656785 40147 -431 880934 184671 -432 496858 720758 -433 788625 168073 -434 688596 278777 -435 224306 349453 -436 797187 760110 -437 340564 667819 -438 728026 455440 -439 113998 777939 -440 252566 971084 -441 879919 467941 -442 268461 878251 -443 208828 893858 -444 418355 247981 -445 469227 367535 -446 150361 488550 -447 92759 964090 -448 243955 840150 -449 968345 608436 -450 710260 92350 -451 597533 267467 -452 438839 566688 -453 368779 669547 -454 774215 995654 -455 656331 69034 -456 167955 192082 -457 194506 289250 -458 889785 392383 -459 91912 532767 -460 476803 948474 -461 199728 80159 -462 568517 131955 -463 751919 529719 -464 321271 773876 -465 299040 58479 -466 681224 118343 -467 121607 183531 -468 803128 687836 -469 273434 979211 -470 988466 816444 -471 801946 885587 -472 771178 520752 -473 167806 70201 -474 920032 960804 -475 212170 714235 -476 518879 194468 -477 309396 651780 -478 411125 479190 -479 145859 255308 -480 763559 680942 -481 396112 795004 -482 7188 839887 -483 267087 282367 -484 420904 673753 -485 121443 819584 -486 472350 131107 -487 320597 762567 -488 429595 5260 -489 770763 916737 -490 720829 272729 -491 383859 65962 -492 10317 618097 -493 662366 886220 -494 111154 281719 -495 916443 520760 -496 266932 599681 -497 857767 168881 -498 344093 670838 -499 564990 8610 -500 91157 162546 -501 299437 3295 -502 239974 939441 -503 330375 452916 -504 184150 691298 -505 989414 524905 -506 368035 864140 -507 338864 242798 -508 414009 796430 -509 937236 838306 -510 495794 596248 -511 717377 412293 -512 582595 958896 -513 520146 469055 -514 891133 381155 -515 432191 976678 -516 821109 79464 -517 927322 195924 -518 904693 519719 -519 727098 767519 -520 204088 865935 -521 948983 272847 -522 785925 320195 -523 803383 854638 -524 17086 898871 -525 648735 875038 -526 779705 183835 -527 576418 114479 -528 341437 219972 -529 75973 44050 -530 34748 810655 -531 725818 416631 -532 487210 123479 -533 575921 95187 -534 78214 18669 -535 439414 559370 -536 779344 38365 -537 189571 620755 -538 816543 533541 -539 835874 468115 -540 617458 300173 -541 393081 847082 -542 346407 621535 -543 250859 404477 -544 592254 803843 -545 620002 809505 -546 441505 708429 -547 328104 644717 -548 86590 910618 -549 83276 165170 -550 503651 269842 -551 18764 548970 -552 281412 348562 -553 481956 932623 -554 837428 954882 -555 863619 936960 -556 627718 272130 -557 424048 225242 -558 369150 17388 -559 88527 842493 -560 36888 665303 -561 11911 913043 -562 515017 169572 -563 540606 230374 -564 756952 841008 -565 138798 883920 -566 696113 880992 -567 604495 680498 -568 672454 120951 -569 423034 121164 -570 252385 233471 -571 315950 749761 -572 766954 954698 -573 797594 528461 -574 193411 158532 -575 104110 856216 -576 153665 242267 -577 26371 619731 -578 573728 137771 -579 944474 600914 -580 676107 361005 -581 509589 716264 -582 702496 630148 -583 621010 877957 -584 505175 469350 -585 427648 840688 -586 823976 929994 -587 738382 180671 -588 511638 769643 -589 886672 895285 -590 595843 402834 -591 285899 156038 -592 164901 777793 -593 374331 979849 -594 178495 974347 -595 59488 794497 -596 615776 953683 -597 693515 411697 -598 409494 385955 -599 11378 586283 -600 443059 27951 -601 641789 298126 -602 562689 701275 -603 570317 997626 -604 77366 248577 -605 245400 593879 -606 959276 970127 -607 541425 881609 -608 745228 555905 -609 22748 290799 -610 220654 235054 -611 866578 918891 -612 26290 399697 -613 198898 525850 -614 367305 37106 -615 610354 514012 -616 692277 638094 -617 649884 1963 -618 443558 315771 -619 614613 283292 -620 32564 818425 -621 957100 887695 -622 753692 824433 -623 927919 696884 -624 927393 293818 -625 212798 883643 -626 18649 974272 -627 975928 929047 -628 335674 3694 -629 648241 560725 -630 257717 254545 -631 383013 794073 -632 216012 426974 -633 636778 71580 -634 216434 716991 -635 953689 48344 -636 628512 728930 -637 78000 337011 -638 216404 892305 -639 942963 97243 -640 64023 195204 -641 877609 806580 -642 779389 355808 -643 130998 898203 -644 422082 222909 -645 365184 371977 -646 99337 897621 -647 329602 984220 -648 189913 228170 -649 809694 416681 -650 608029 35613 -651 753921 830149 -652 229794 17593 -653 6034 212069 -654 194883 620120 -655 798048 437471 -656 581612 425331 -657 195540 885740 -658 155208 485392 -659 464471 231792 -660 237061 408608 -661 261886 799753 -662 108962 918075 -663 294423 798362 -664 499135 60407 -665 330976 4334 -666 697422 322842 -667 299194 626552 -668 613592 452277 -669 611039 893648 -670 200600 645606 -671 433731 190289 -672 985848 956576 -673 110091 299583 -674 788658 411526 -675 689797 391550 -676 729035 749287 -677 85704 603694 -678 338190 431078 -679 530954 603242 -680 404001 553756 -681 601030 301234 -682 419519 152441 -683 3739 391322 -684 439482 238964 -685 775649 855624 -686 696733 52944 -687 541995 845510 -688 679572 512598 -689 714048 505267 -690 579885 863344 -691 267407 895892 -692 656405 777220 -693 403303 396188 -694 903323 146753 -695 622813 222270 -696 12794 372074 -697 117999 344975 -698 948873 380786 -699 648294 140337 -700 277004 597492 -701 585534 283391 -702 831641 826452 -703 124143 833143 -704 92882 308483 -705 200390 942002 -706 527755 384201 -707 980429 181731 -708 540961 228955 -709 183234 74544 -710 203567 622952 -711 743027 299145 -712 961959 178156 -713 270089 413341 -714 221303 715542 -715 721367 846429 -716 419716 405564 -717 304876 637883 -718 335589 883205 -719 915463 474674 -720 236259 174348 -721 212954 828779 -722 943185 999860 -723 479242 713649 -724 410115 939843 -725 74885 535531 -726 159483 932752 -727 418970 430575 -728 177776 179969 -729 978514 193758 -730 490863 240937 -731 917680 618207 -732 72224 26042 -733 729047 698976 -734 441016 980568 -735 702488 827311 -736 818840 243390 -737 999658 668036 -738 463468 810275 -739 880174 531384 -740 379 90119 -741 434827 27544 -742 224679 480429 -743 928748 801508 -744 333340 278834 -745 908835 636613 -746 442188 934894 -747 772186 408947 -748 355507 969563 -749 829120 275149 -750 536392 668966 -751 833475 878730 -752 939464 984765 -753 100704 131938 -754 708072 938541 -755 501826 376267 -756 846629 699680 -757 284923 854249 -758 798748 982785 -759 846021 663276 -760 670535 759484 -761 869855 165867 -762 444176 33521 -763 993835 940109 -764 303926 14892 -765 567331 331529 -766 102202 941632 -767 29557 624111 -768 243490 735146 -769 742621 370429 -770 681680 945963 -771 955487 670063 -772 608298 149350 -773 766757 276321 -774 490717 738664 -775 415112 415397 -776 665636 814227 -777 707817 204862 -778 566764 891842 -779 849172 315352 -780 741275 365557 -781 389317 337643 -782 256860 555863 -783 420780 176566 -784 91381 135572 -785 87491 522427 -786 492068 107356 -787 248164 255137 -788 93847 55627 -789 458050 239314 -790 466757 101992 -791 90936 988574 -792 273617 201773 -793 942212 764559 -794 684771 473694 -795 203331 66923 -796 643765 607049 -797 282937 159253 -798 453895 848130 -799 421898 353161 -800 55502 711129 -801 818271 251403 -802 271907 313119 -803 324460 677062 -804 540610 506043 -805 262649 802204 -806 207071 375478 -807 112021 674351 -808 721792 782268 -809 54705 97607 -810 101967 572649 -811 754667 738220 -812 80069 376361 -813 704156 240665 -814 835449 935045 -815 930676 416785 -816 515017 952006 -817 976665 664553 -818 894921 715458 -819 876552 599265 -820 479981 159944 -821 709854 587164 -822 371726 630681 -823 312256 905544 -824 526979 906575 -825 455096 213230 -826 12681 486852 -827 422484 638717 -828 196112 866265 -829 274738 607353 -830 418197 609001 -831 800510 941344 -832 26062 102705 -833 92349 619906 -834 3751 481340 -835 362095 816248 -836 255288 148129 -837 642510 888737 -838 559968 267814 -839 315736 441352 -840 180249 837891 -841 965927 228095 -842 516848 321675 -843 616274 573672 -844 925943 873959 -845 572203 275014 -846 711707 395212 -847 237169 663733 -848 904655 561725 -849 944653 482988 -850 70712 44441 -851 589808 85627 -852 726326 617205 -853 247303 784585 -854 970004 100809 -855 22442 622439 -856 940321 400778 -857 35150 143182 -858 897294 405298 -859 704174 362329 -860 198049 530624 -861 675253 520762 -862 410627 317654 -863 226440 169661 -864 421802 25304 -865 641434 775382 -866 297809 214927 -867 418909 557810 -868 897569 565148 -869 192945 373665 -870 424380 210976 -871 869498 563613 -872 675910 41579 -873 599761 190945 -874 984584 919515 -875 437293 734994 -876 139922 919348 -877 841554 295330 -878 746631 374880 -879 666718 168516 -880 719636 682154 -881 591640 596339 -882 676428 811466 -883 752941 376708 -884 724867 993571 -885 543421 706348 -886 420713 784659 -887 925036 463055 -888 390701 755904 -889 679208 115297 -890 571023 851559 -891 502943 253286 -892 305668 959280 -893 183742 701470 -894 538499 607443 -895 804869 520860 -896 840281 199374 -897 830244 718031 -898 790263 84839 -899 638576 212854 -900 650877 843856 -901 511737 869287 -902 348492 585733 -903 232050 886635 -904 613680 882273 -905 111587 45160 -906 836381 59274 -907 363647 198776 -908 841872 392266 -909 766095 958396 -910 21202 668102 -911 738131 512013 -912 342693 699564 -913 194611 551425 -914 436166 339302 -915 231004 504066 -916 142224 796934 -917 3709 525862 -918 15177 443668 -919 889639 106891 -920 117407 791476 -921 935375 580102 -922 586240 136767 -923 782729 328268 -924 856680 130679 -925 523420 238837 -926 648672 299273 -927 981849 146811 -928 701632 714802 -929 865577 822630 -930 570556 788382 -931 996996 506788 -932 764865 320111 -933 109785 256279 -934 473034 226880 -935 416046 513604 -936 535586 635415 -937 434976 782365 -938 19429 438857 -939 530163 195882 -940 997932 479809 -941 685848 613536 -942 768746 442622 -943 507069 635435 -944 32068 683598 -945 416076 758898 -946 970634 662759 -947 690844 603802 -948 255889 299959 -949 145126 803875 -950 117337 343871 -951 798104 660797 -952 132747 43611 -953 552989 35135 -954 530526 539227 -955 639741 66196 -956 682733 181979 -957 406553 811657 -958 817748 334236 -959 815943 161062 -960 64221 809825 -961 111152 669159 -962 109542 72175 -963 715499 874788 -964 502668 391365 -965 229375 984294 -966 908330 990935 -967 556141 931735 -968 797075 503868 -969 206983 957089 -970 624874 172832 -971 819491 871006 -972 619376 606251 -973 647745 301474 -974 553216 618669 -975 888302 381101 -976 797291 753760 -977 574499 133042 -978 435540 807169 -979 104656 201011 -980 246535 49120 -981 828151 573437 -982 914352 466908 -983 246705 811727 -984 562602 800302 -985 163911 516273 -986 781020 197274 -987 272760 683120 -988 12534 357392 -989 94659 976499 -990 636634 908330 -991 770132 301656 -992 144829 180255 -993 156142 334893 -994 76582 89232 -995 685030 985347 -996 941265 406681 -997 793177 108601 -998 391812 622217 -999 598246 936256 -1000 593717 290353 diff --git a/devel-examples/vignette-example/instances/1000-13.tsp b/devel-examples/vignette-example/instances/1000-13.tsp deleted file mode 100644 index f5e60e41..00000000 --- a/devel-examples/vignette-example/instances/1000-13.tsp +++ /dev/null @@ -1,1006 +0,0 @@ -NAME : portgen-1000-13 -COMMENT : portgen N=1000, seed=13 -TYPE : TSP -DIMENSION : 1000 -EDGE_WEIGHT_TYPE : EUC_2D -NODE_COORD_SECTION -1 89465 148576 -2 47684 278031 -3 797589 479775 -4 625102 102360 -5 65912 298645 -6 752407 306362 -7 816286 660031 -8 736767 708215 -9 88339 91915 -10 502586 500313 -11 905398 708432 -12 754793 538274 -13 318183 208483 -14 757476 342116 -15 725504 766724 -16 185180 377726 -17 158161 925121 -18 168596 132001 -19 807898 811964 -20 43997 899562 -21 216986 150682 -22 501203 873816 -23 956694 684443 -24 87214 636320 -25 387008 241996 -26 181328 284330 -27 133466 367217 -28 98555 711739 -29 990415 122563 -30 109435 665587 -31 671876 813137 -32 58362 166349 -33 81658 601724 -34 805158 942469 -35 703336 52323 -36 621000 452018 -37 704907 260590 -38 318985 621067 -39 574965 387576 -40 439718 606444 -41 218068 634913 -42 232681 59916 -43 94847 372042 -44 319363 991811 -45 843462 566871 -46 326843 865428 -47 108628 991673 -48 278562 764967 -49 445775 240613 -50 554831 335626 -51 109477 699638 -52 196601 780564 -53 23928 546415 -54 51649 73549 -55 272370 726513 -56 392375 998604 -57 279101 542563 -58 338744 806448 -59 704509 66688 -60 887787 316690 -61 155949 564544 -62 387638 367710 -63 942845 921361 -64 255416 924342 -65 236662 772570 -66 569418 501415 -67 115205 713205 -68 214068 219464 -69 355811 690117 -70 721172 288399 -71 667533 252675 -72 104023 526771 -73 410921 762298 -74 477790 740917 -75 48827 357200 -76 509550 521432 -77 3951 782261 -78 766207 608062 -79 584432 483396 -80 566495 804464 -81 190603 361531 -82 352377 983970 -83 58979 139700 -84 894580 752330 -85 131641 576445 -86 328658 963591 -87 17860 530587 -88 807139 634516 -89 560593 605377 -90 601502 334783 -91 336929 772020 -92 357847 432198 -93 581966 207886 -94 149038 131235 -95 654225 74368 -96 324883 603481 -97 558475 144726 -98 959741 703941 -99 234814 573436 -100 719631 776405 -101 201704 872413 -102 139415 714044 -103 20271 737530 -104 163585 571753 -105 200294 558321 -106 459023 453197 -107 829171 492126 -108 479580 587121 -109 803056 207650 -110 24229 355038 -111 904886 321143 -112 32698 355236 -113 374740 456244 -114 824175 303816 -115 510316 69609 -116 470931 988840 -117 405082 43181 -118 875760 883731 -119 942848 865720 -120 952617 994844 -121 404830 941387 -122 107005 299186 -123 169482 3739 -124 570783 203239 -125 769985 503496 -126 879765 930997 -127 63120 650022 -128 305473 212864 -129 467330 96234 -130 838283 136539 -131 794682 297864 -132 619135 205450 -133 153490 517636 -134 346192 529984 -135 322644 475840 -136 16338 599816 -137 437665 520733 -138 475272 973888 -139 258023 382676 -140 49762 161876 -141 988914 727941 -142 465532 373776 -143 274926 173066 -144 369705 199631 -145 889690 358124 -146 537538 412863 -147 543075 476777 -148 978505 805013 -149 503722 586272 -150 823914 195593 -151 745716 188106 -152 153476 608109 -153 514582 151824 -154 465464 689343 -155 375095 132407 -156 843159 267698 -157 206543 480159 -158 599000 381819 -159 754788 142357 -160 226944 348476 -161 13913 759920 -162 706070 127050 -163 730124 828232 -164 446339 829555 -165 6150 323447 -166 508423 568680 -167 7580 917355 -168 318717 721215 -169 521397 985373 -170 774775 893107 -171 418278 227347 -172 972687 541214 -173 344210 777618 -174 706793 416025 -175 746653 150273 -176 358674 674166 -177 580121 500467 -178 687170 177035 -179 180525 236121 -180 289392 793367 -181 630427 480091 -182 914568 481988 -183 714129 615811 -184 295011 665329 -185 135948 821381 -186 675026 338763 -187 395704 76670 -188 989802 339746 -189 179798 205603 -190 439880 553088 -191 647706 210218 -192 540163 212783 -193 693020 28332 -194 654111 525592 -195 203226 702905 -196 426203 856067 -197 849424 953393 -198 218081 79514 -199 831642 896016 -200 551411 4463 -201 597820 501189 -202 976144 193564 -203 502567 148455 -204 134003 367337 -205 807446 658837 -206 522924 654933 -207 32895 586486 -208 367163 774359 -209 630666 961174 -210 263907 634614 -211 784168 398994 -212 113918 131484 -213 223561 173836 -214 362618 202140 -215 574103 841346 -216 205742 812461 -217 398156 781042 -218 30164 992773 -219 177322 953676 -220 845620 918660 -221 397665 692936 -222 261684 568611 -223 918736 27208 -224 742148 717939 -225 729831 44244 -226 716895 629502 -227 321912 710064 -228 798721 785358 -229 103033 195101 -230 163400 509793 -231 971132 180327 -232 521716 888786 -233 261172 829987 -234 393248 464284 -235 667750 339954 -236 32210 912727 -237 231343 219662 -238 917718 154666 -239 77082 403853 -240 332763 438203 -241 70802 167516 -242 38740 64309 -243 870213 25415 -244 290744 509370 -245 519870 200177 -246 599524 713038 -247 285925 505666 -248 886449 484937 -249 461593 42022 -250 650892 764069 -251 950126 338295 -252 385175 291628 -253 973441 549379 -254 590762 257602 -255 839850 773305 -256 494614 593663 -257 675231 963222 -258 910269 258094 -259 894401 16050 -260 2336 776234 -261 368393 351226 -262 813391 903680 -263 389827 693915 -264 527552 939714 -265 246220 368339 -266 563904 819479 -267 564002 559458 -268 943589 477139 -269 492285 75517 -270 154040 612119 -271 131014 274694 -272 507033 743635 -273 831784 248298 -274 899647 382245 -275 115838 192533 -276 957385 521879 -277 795801 282553 -278 200165 130647 -279 774292 825716 -280 348039 496302 -281 57094 515244 -282 103562 227731 -283 642290 219919 -284 86629 931596 -285 131438 661970 -286 358446 512155 -287 900212 809802 -288 818849 846514 -289 555424 530838 -290 703514 259180 -291 919622 701835 -292 591675 749918 -293 311245 48659 -294 715917 336271 -295 917168 723669 -296 390510 560688 -297 944078 492069 -298 253673 618858 -299 374481 697230 -300 924786 985269 -301 692874 368809 -302 678730 856658 -303 272910 255550 -304 930203 45883 -305 971307 151506 -306 414730 438021 -307 908548 624370 -308 105791 496405 -309 571165 611492 -310 974058 23431 -311 845438 389398 -312 6809 146168 -313 969096 989637 -314 833425 43553 -315 536891 563298 -316 916310 509541 -317 559530 552008 -318 844450 481601 -319 793286 967304 -320 644126 814840 -321 477494 104424 -322 362213 893736 -323 878231 1111 -324 553879 797910 -325 522972 264035 -326 785433 330928 -327 160338 361487 -328 68959 183333 -329 809278 126721 -330 12207 791309 -331 462263 962899 -332 401756 156466 -333 674012 310305 -334 75808 14812 -335 746139 104680 -336 942526 773255 -337 88519 710022 -338 237998 514510 -339 229060 645322 -340 77209 785763 -341 180358 706703 -342 31346 745582 -343 101035 953411 -344 107784 403063 -345 877996 534144 -346 405793 778474 -347 221165 539446 -348 872314 704238 -349 15904 652190 -350 655737 363721 -351 772051 908557 -352 720700 737208 -353 83677 78729 -354 299581 414755 -355 260452 115548 -356 75548 406214 -357 248724 478063 -358 385515 683789 -359 741734 862310 -360 284152 969773 -361 294400 423617 -362 359075 382418 -363 332628 33969 -364 52554 351310 -365 626345 159269 -366 214928 814304 -367 384869 961660 -368 710215 774143 -369 457979 553282 -370 360067 417245 -371 211676 245474 -372 118911 908223 -373 239743 982175 -374 419398 838746 -375 206817 838345 -376 651684 664593 -377 25845 496468 -378 148792 957747 -379 523687 759040 -380 26993 309533 -381 620750 746299 -382 54688 843207 -383 903871 830073 -384 287303 340501 -385 238319 403339 -386 264390 902987 -387 655492 445807 -388 318089 629807 -389 397772 862606 -390 233625 374881 -391 510281 293514 -392 324317 316811 -393 538518 468628 -394 759615 541661 -395 57789 880141 -396 486840 117477 -397 314962 956727 -398 152854 308689 -399 589982 673104 -400 590134 609936 -401 584403 556792 -402 605121 831935 -403 328064 358169 -404 340276 709033 -405 957950 680164 -406 198131 982025 -407 701250 146852 -408 822693 503272 -409 431336 97961 -410 690352 595181 -411 240091 614199 -412 750367 628382 -413 818935 707598 -414 297866 823556 -415 117742 959919 -416 289530 688738 -417 904656 553461 -418 176750 528255 -419 592263 177465 -420 494118 35245 -421 37292 661653 -422 851309 462607 -423 640049 872640 -424 367110 686579 -425 137791 445256 -426 10823 766425 -427 555361 630214 -428 320405 895665 -429 652135 51659 -430 655185 799808 -431 765905 162810 -432 214914 922704 -433 642871 536477 -434 130716 238643 -435 506802 950052 -436 136162 744756 -437 960170 245096 -438 584358 473666 -439 58838 120153 -440 307977 923270 -441 55462 246206 -442 168371 317934 -443 194014 126719 -444 473823 981952 -445 910589 640272 -446 397538 353620 -447 670662 544066 -448 899083 292535 -449 701483 606213 -450 878248 166382 -451 813802 246957 -452 378602 214520 -453 389793 764617 -454 598053 237427 -455 436199 193685 -456 421841 670183 -457 141070 14912 -458 402269 412284 -459 492147 670848 -460 23621 350336 -461 834993 524503 -462 360394 340419 -463 136249 889204 -464 366153 745649 -465 855302 819741 -466 875613 821410 -467 683953 114291 -468 501429 385279 -469 105135 153458 -470 915664 781729 -471 634572 802975 -472 958331 560253 -473 805278 873035 -474 993226 330243 -475 407817 9878 -476 926382 955834 -477 750911 58507 -478 290769 992392 -479 563004 264311 -480 713091 4514 -481 659481 444595 -482 321762 654470 -483 559113 618865 -484 711852 580817 -485 209634 529234 -486 419058 161904 -487 263031 13742 -488 423953 879159 -489 773591 301886 -490 49650 143857 -491 326200 101841 -492 32557 850787 -493 160259 431018 -494 499647 29482 -495 555177 882563 -496 673426 524317 -497 943824 386430 -498 362670 472667 -499 539944 944588 -500 136299 926119 -501 99443 691339 -502 280592 263960 -503 683677 824540 -504 923276 900123 -505 898247 859751 -506 492744 533521 -507 709134 830527 -508 331087 135163 -509 500770 935332 -510 291799 86446 -511 78921 767264 -512 444518 283514 -513 429790 727718 -514 881311 999070 -515 330064 599413 -516 955882 873468 -517 403639 189898 -518 651112 792678 -519 392707 202029 -520 519699 25096 -521 930247 564315 -522 737682 468731 -523 803642 906162 -524 79799 660310 -525 956639 634952 -526 591355 540873 -527 614523 536886 -528 970237 225975 -529 287700 90694 -530 612847 890998 -531 431832 721246 -532 380423 873151 -533 929503 928429 -534 795838 240403 -535 26885 424925 -536 55364 840460 -537 978693 656847 -538 495090 538047 -539 152740 907631 -540 313277 203814 -541 440017 790617 -542 386223 439065 -543 167580 234635 -544 493044 530487 -545 260394 722683 -546 996840 152304 -547 175144 94774 -548 969732 89787 -549 585622 80835 -550 973640 265594 -551 753422 172168 -552 347032 752825 -553 194934 800737 -554 154649 175457 -555 369306 735601 -556 732931 757213 -557 830299 890163 -558 894158 279528 -559 546102 285649 -560 903419 839716 -561 342807 715003 -562 266762 761291 -563 671502 883195 -564 493427 225867 -565 461913 694352 -566 383397 977282 -567 538325 577675 -568 470883 682804 -569 960317 496060 -570 544906 888052 -571 688532 207394 -572 627067 420677 -573 379876 281836 -574 885541 413852 -575 423271 86536 -576 596359 359754 -577 618922 279287 -578 882196 776139 -579 633842 769356 -580 281941 512129 -581 840419 658589 -582 630550 481254 -583 47069 525536 -584 130145 409621 -585 510286 612321 -586 393986 132250 -587 862378 816883 -588 243357 983052 -589 96081 987475 -590 879094 895363 -591 249353 724070 -592 943926 949783 -593 853932 724808 -594 346731 57071 -595 530606 945347 -596 552658 550696 -597 985773 932584 -598 494065 556282 -599 345016 810184 -600 177320 396823 -601 185755 898066 -602 534757 527908 -603 837182 872288 -604 415827 669138 -605 425354 157388 -606 429408 576771 -607 238750 336594 -608 959470 289723 -609 672815 697965 -610 987188 490786 -611 180519 319961 -612 232301 113462 -613 426566 495920 -614 597492 334469 -615 979700 371069 -616 567225 426942 -617 562121 721706 -618 465954 672581 -619 485320 607332 -620 990312 564209 -621 51992 648765 -622 69882 39820 -623 764827 232697 -624 318394 872310 -625 506018 998145 -626 958790 10546 -627 830483 806250 -628 829598 758812 -629 335944 813051 -630 61953 164601 -631 386967 808494 -632 678825 861145 -633 105395 780642 -634 506888 198930 -635 571766 726773 -636 971329 800504 -637 191947 989043 -638 531996 169972 -639 489477 426050 -640 283864 667753 -641 159975 784440 -642 272516 815099 -643 984101 758730 -644 748116 700975 -645 616310 685312 -646 165693 286390 -647 35566 263539 -648 592880 251487 -649 456818 80839 -650 507823 594854 -651 743220 892344 -652 588446 838264 -653 838169 174349 -654 738030 15384 -655 822149 70867 -656 10695 634969 -657 196740 376640 -658 998908 100576 -659 772928 415286 -660 268265 853908 -661 323824 426049 -662 691106 976911 -663 983552 78984 -664 212058 353682 -665 150873 357647 -666 431942 474092 -667 603901 212996 -668 657058 525006 -669 587699 895875 -670 816191 883524 -671 985801 332830 -672 432710 762402 -673 361488 739643 -674 595283 58654 -675 279986 513895 -676 39429 103135 -677 929965 150176 -678 162912 269127 -679 288442 375449 -680 181206 313163 -681 586649 842155 -682 199193 938624 -683 85066 677865 -684 202258 434338 -685 15152 259264 -686 505293 714273 -687 135299 754370 -688 814479 220688 -689 496084 540930 -690 813998 714425 -691 790542 836608 -692 172476 837710 -693 770998 589786 -694 274899 665277 -695 127930 979192 -696 322747 153360 -697 880723 556926 -698 378231 271527 -699 197530 678340 -700 947922 140799 -701 243558 54353 -702 244656 565561 -703 723352 202820 -704 930659 92254 -705 379177 573126 -706 994227 623165 -707 247519 202013 -708 990415 433288 -709 961432 642266 -710 560392 813538 -711 480334 523918 -712 486416 874352 -713 15706 450939 -714 469617 569738 -715 31017 611658 -716 290029 403830 -717 161752 240872 -718 720197 167376 -719 589089 970462 -720 847295 337709 -721 628354 632632 -722 104884 314392 -723 498858 798829 -724 666944 6370 -725 541220 927291 -726 801909 627792 -727 647323 336263 -728 850770 839728 -729 892600 3783 -730 845363 555976 -731 613731 960196 -732 244958 41468 -733 944771 361595 -734 518281 933126 -735 703155 191585 -736 766343 955061 -737 101046 633100 -738 11628 852541 -739 876594 150152 -740 23582 175977 -741 558339 465834 -742 724374 475040 -743 997927 329832 -744 158871 120284 -745 296100 358602 -746 649095 655962 -747 267307 655710 -748 571366 673292 -749 531585 471783 -750 302763 646316 -751 922235 516792 -752 982788 365242 -753 368952 336075 -754 903418 172282 -755 338336 520937 -756 680856 772315 -757 707682 486761 -758 906881 957768 -759 692889 589248 -760 470102 271479 -761 830009 46497 -762 630362 56838 -763 269350 249550 -764 972273 735804 -765 264148 675552 -766 949123 704312 -767 811816 502644 -768 495120 786023 -769 758151 237612 -770 568159 40159 -771 636943 569622 -772 650182 24621 -773 528592 602597 -774 25599 210468 -775 386360 321815 -776 701019 795781 -777 207635 627211 -778 697193 217922 -779 704976 480143 -780 870121 582928 -781 577924 665805 -782 604123 298176 -783 883993 111234 -784 122133 683061 -785 958169 304283 -786 932168 482420 -787 202888 148286 -788 570459 34227 -789 838861 3151 -790 359644 51427 -791 544574 492129 -792 865682 681219 -793 97627 283317 -794 100188 513639 -795 618650 383886 -796 663890 75089 -797 279443 263875 -798 107990 154523 -799 366734 501895 -800 454162 494364 -801 763735 22448 -802 850824 334932 -803 777241 208890 -804 930099 526416 -805 529583 413876 -806 117733 191336 -807 861493 486235 -808 919038 502834 -809 386362 340247 -810 190186 729470 -811 744499 620237 -812 228899 463804 -813 540547 909720 -814 631595 867955 -815 371045 361569 -816 104128 312445 -817 473567 945768 -818 933693 353237 -819 630636 379447 -820 762180 594793 -821 896955 759941 -822 323453 889180 -823 639387 43652 -824 846190 815639 -825 723327 198269 -826 522927 498778 -827 130850 92592 -828 390236 451290 -829 548880 905056 -830 401239 424003 -831 578253 550651 -832 764235 934790 -833 516921 357792 -834 867883 972312 -835 846848 875385 -836 656643 570722 -837 616919 991916 -838 206542 245721 -839 489387 136306 -840 73567 89257 -841 360840 726539 -842 466716 947041 -843 783315 553476 -844 548210 538777 -845 428846 575900 -846 485353 658323 -847 532599 886794 -848 938149 326232 -849 143021 331537 -850 682638 393666 -851 554265 709884 -852 742071 634070 -853 837429 796388 -854 32062 183808 -855 309277 836759 -856 903080 10102 -857 476209 825338 -858 938649 919929 -859 18052 877440 -860 996641 190689 -861 214771 536346 -862 289673 453182 -863 321120 946759 -864 828651 982849 -865 154486 410154 -866 213658 305579 -867 827029 236808 -868 186177 350737 -869 250329 641377 -870 8392 863385 -871 535423 670770 -872 542136 238157 -873 361129 949007 -874 368650 79417 -875 565674 991390 -876 497580 160172 -877 177050 272484 -878 180007 248686 -879 882855 505263 -880 447893 486692 -881 546058 390684 -882 175416 445891 -883 301335 232310 -884 467966 238052 -885 464209 989642 -886 551279 938635 -887 311766 5251 -888 693108 54599 -889 359295 17189 -890 273174 72434 -891 63904 323388 -892 534955 667794 -893 864095 822973 -894 130162 381137 -895 935472 953867 -896 882770 12276 -897 177167 18749 -898 312105 596788 -899 359003 536885 -900 545048 306530 -901 589711 351460 -902 806242 493239 -903 927485 174192 -904 625216 509256 -905 408388 357033 -906 118523 501717 -907 569790 494026 -908 603921 533781 -909 213517 156666 -910 133785 704547 -911 873306 931081 -912 693004 157679 -913 399930 199819 -914 132393 818526 -915 77765 518916 -916 429382 850038 -917 608800 916141 -918 953911 562186 -919 753598 40929 -920 63872 330313 -921 609456 973495 -922 247352 230925 -923 80561 951688 -924 319272 19488 -925 618819 112286 -926 464395 540477 -927 459119 26132 -928 877147 739673 -929 742659 890101 -930 539328 365299 -931 420594 584286 -932 445384 78075 -933 747576 145027 -934 254364 338864 -935 413464 652233 -936 214509 194029 -937 537847 21498 -938 240151 332828 -939 471961 666871 -940 280532 660257 -941 457159 242292 -942 279197 712465 -943 98322 845096 -944 404654 530725 -945 168564 808883 -946 307822 414733 -947 627464 411638 -948 115804 415427 -949 435648 225853 -950 990773 747732 -951 479726 652400 -952 738955 958561 -953 655126 222103 -954 261280 746653 -955 927810 32050 -956 335018 211934 -957 721536 730445 -958 57477 5860 -959 956821 33745 -960 962271 332148 -961 709379 28510 -962 348090 665731 -963 172506 562108 -964 455073 579285 -965 366371 18048 -966 71548 725308 -967 739061 248481 -968 325238 245225 -969 520755 548752 -970 654988 92461 -971 888274 370908 -972 568454 836416 -973 99504 279311 -974 66643 961732 -975 239132 553695 -976 960354 856362 -977 859482 972725 -978 676184 754418 -979 913338 490474 -980 633322 409901 -981 701347 712528 -982 91665 835348 -983 143776 964110 -984 643480 885119 -985 630940 778165 -986 939217 995088 -987 794613 408575 -988 371794 853016 -989 169028 375365 -990 989547 418087 -991 648769 964599 -992 945963 956469 -993 316701 359019 -994 633643 903713 -995 104705 361128 -996 601744 635636 -997 917811 876823 -998 153244 893185 -999 576295 159878 -1000 464621 246487 diff --git a/devel-examples/vignette-example/instances/1000-14.tsp b/devel-examples/vignette-example/instances/1000-14.tsp deleted file mode 100644 index e33c2f96..00000000 --- a/devel-examples/vignette-example/instances/1000-14.tsp +++ /dev/null @@ -1,1006 +0,0 @@ -NAME : portgen-1000-14 -COMMENT : portgen N=1000, seed=14 -TYPE : TSP -DIMENSION : 1000 -EDGE_WEIGHT_TYPE : EUC_2D -NODE_COORD_SECTION -1 186781 473452 -2 436866 143196 -3 599907 418224 -4 270718 876441 -5 977793 123530 -6 729225 963689 -7 958147 947490 -8 137801 936362 -9 246180 847637 -10 540691 994509 -11 875463 416929 -12 716691 389823 -13 142746 10758 -14 886248 952129 -15 224797 773021 -16 678586 408600 -17 858364 889879 -18 449399 785841 -19 775176 28863 -20 66019 146391 -21 429482 433989 -22 379756 99512 -23 755718 319701 -24 100488 76503 -25 87902 27189 -26 977035 606950 -27 669316 413013 -28 177254 778180 -29 615088 546986 -30 693796 814066 -31 643048 241855 -32 810421 831402 -33 694048 295235 -34 583933 858635 -35 191772 818099 -36 835874 169676 -37 759735 513501 -38 17474 268512 -39 747612 303678 -40 212568 364566 -41 395670 339262 -42 258332 410730 -43 129973 436731 -44 598178 26962 -45 195830 154163 -46 201907 916541 -47 837091 247919 -48 310516 259805 -49 674254 866254 -50 82038 487205 -51 572089 796809 -52 863935 723335 -53 631518 637773 -54 348617 258585 -55 283040 740523 -56 180001 588125 -57 351155 539633 -58 612158 726507 -59 404764 562502 -60 520885 434243 -61 620981 717678 -62 776596 704566 -63 246010 39064 -64 305741 36399 -65 881982 379700 -66 919894 489026 -67 20637 472045 -68 184564 807545 -69 988106 718699 -70 798572 403466 -71 31967 35676 -72 506077 761587 -73 533181 484228 -74 139944 132524 -75 1908 271452 -76 954064 637854 -77 984272 702338 -78 567311 83062 -79 776171 391889 -80 538770 823973 -81 649667 629918 -82 460013 879574 -83 708556 144325 -84 82048 589568 -85 6451 127929 -86 586562 272239 -87 560593 249433 -88 480179 983127 -89 733406 74257 -90 137255 162948 -91 262892 913851 -92 497628 58008 -93 730033 289976 -94 29013 141063 -95 763489 40239 -96 725496 398538 -97 712247 670642 -98 816903 759727 -99 475082 256644 -100 281408 550054 -101 750822 65686 -102 995269 838960 -103 8559 40212 -104 140226 926263 -105 972305 277335 -106 54049 635108 -107 628400 498531 -108 98476 251128 -109 917670 789371 -110 62670 948828 -111 669242 825404 -112 308160 456397 -113 377107 520875 -114 276970 721633 -115 240873 439967 -116 842900 807143 -117 101952 859920 -118 108899 627784 -119 285451 999097 -120 959531 478904 -121 372305 239641 -122 78393 814660 -123 370997 900092 -124 90378 255850 -125 293534 296028 -126 482757 753449 -127 15770 841441 -128 707153 943679 -129 963734 135349 -130 730061 380774 -131 754760 141128 -132 966731 493400 -133 905030 814407 -134 556714 813739 -135 127533 198383 -136 160750 661820 -137 495836 766641 -138 466071 915792 -139 809633 466718 -140 749243 433428 -141 557141 141621 -142 991572 860099 -143 685206 701771 -144 840411 608552 -145 954889 294491 -146 71069 471712 -147 871563 761148 -148 318153 710484 -149 743804 311751 -150 348588 455205 -151 90459 623659 -152 506606 860106 -153 738887 341135 -154 761877 155671 -155 156234 5381 -156 103267 355181 -157 180459 435569 -158 309704 283048 -159 269565 205583 -160 175246 194545 -161 70602 244963 -162 465150 672328 -163 107924 537091 -164 155214 635730 -165 27753 124936 -166 153914 653962 -167 310484 743861 -168 330160 201959 -169 961162 556002 -170 550394 402158 -171 432206 634828 -172 433305 760344 -173 223889 826106 -174 6561 199235 -175 653223 781062 -176 555269 108074 -177 283998 223652 -178 301290 436496 -179 313175 762744 -180 529946 536928 -181 379973 205874 -182 605276 754076 -183 573175 468439 -184 921876 420115 -185 211680 483598 -186 276486 70329 -187 552359 394184 -188 639276 962528 -189 960965 241498 -190 371037 671427 -191 223915 392469 -192 105783 490825 -193 744962 948039 -194 48685 556407 -195 170686 861720 -196 280083 541047 -197 344322 66795 -198 125671 361876 -199 82468 39121 -200 121067 261360 -201 865141 765063 -202 828197 981795 -203 557146 162800 -204 2290 793172 -205 478690 353250 -206 387811 756767 -207 592058 668225 -208 256845 838926 -209 861552 538481 -210 628404 211298 -211 385971 882754 -212 299047 950319 -213 618456 511423 -214 242132 570564 -215 837037 476475 -216 960237 167792 -217 762807 17787 -218 283616 467148 -219 800411 437558 -220 233980 906036 -221 86487 510204 -222 928002 959388 -223 475748 397328 -224 242000 394002 -225 448338 614247 -226 119744 511903 -227 202083 644591 -228 301123 697349 -229 2255 810410 -230 698178 89998 -231 362388 564732 -232 559192 572653 -233 266763 877606 -234 828764 632669 -235 192476 859516 -236 596926 467549 -237 90142 14156 -238 91553 874067 -239 680670 654455 -240 649196 921107 -241 509168 431721 -242 872385 747038 -243 114087 395505 -244 608599 190154 -245 751023 406010 -246 638383 167741 -247 245081 374463 -248 309110 618937 -249 420062 913845 -250 867834 601680 -251 716657 587544 -252 744806 527231 -253 105079 688023 -254 639518 455044 -255 530504 905617 -256 88749 812100 -257 59386 292168 -258 451615 194646 -259 319651 184728 -260 263543 647825 -261 457544 914919 -262 764834 590796 -263 142858 9381 -264 722743 562910 -265 909077 403529 -266 234549 225625 -267 123950 743578 -268 832357 697067 -269 372334 580216 -270 295423 919440 -271 75853 423870 -272 926611 103197 -273 948466 723464 -274 402906 654285 -275 231605 299728 -276 896194 857151 -277 4768 464305 -278 367131 491031 -279 463594 1227 -280 694873 408012 -281 315688 59301 -282 159621 611063 -283 829764 664878 -284 885489 956188 -285 343702 728151 -286 791739 665366 -287 953123 963814 -288 751631 600392 -289 910150 300529 -290 223664 651827 -291 545787 721515 -292 868037 501065 -293 87841 175248 -294 66004 512886 -295 913814 167479 -296 811577 416145 -297 236514 567272 -298 127700 410487 -299 470747 962796 -300 351566 348073 -301 813313 102377 -302 430620 579778 -303 753941 174679 -304 989114 503703 -305 376463 191882 -306 425027 950707 -307 87413 527394 -308 596434 899543 -309 822787 592349 -310 483363 419276 -311 194131 922692 -312 604622 995628 -313 914837 689362 -314 234746 373345 -315 209872 576951 -316 611277 406447 -317 924066 31782 -318 226799 595079 -319 634102 340642 -320 904630 188298 -321 352460 473654 -322 29523 494538 -323 973347 888884 -324 811523 240885 -325 652434 438337 -326 175741 97401 -327 752924 774615 -328 736796 406865 -329 178311 398837 -330 352978 158862 -331 540577 648472 -332 599072 188165 -333 839421 951372 -334 921184 592875 -335 554047 707549 -336 88019 581901 -337 939915 45025 -338 243535 96730 -339 169768 830007 -340 258832 507971 -341 511051 835908 -342 20367 51010 -343 36374 962804 -344 807375 735900 -345 192360 275427 -346 673895 41227 -347 786595 197081 -348 100278 770559 -349 533739 984497 -350 251002 876616 -351 719116 981516 -352 982053 144462 -353 927286 339832 -354 77033 701914 -355 738240 773991 -356 599490 442410 -357 206477 77550 -358 484966 499350 -359 861877 401991 -360 87886 68862 -361 417632 936686 -362 341872 677430 -363 988433 106502 -364 599848 795452 -365 117739 903702 -366 19696 467854 -367 91766 484840 -368 908481 68641 -369 629430 942816 -370 566043 537024 -371 100927 405383 -372 648013 123497 -373 857794 737209 -374 699354 109164 -375 208647 993776 -376 170710 738286 -377 866758 347299 -378 856920 251262 -379 889750 497213 -380 235981 858645 -381 710402 134217 -382 135871 201216 -383 673063 194106 -384 794397 82980 -385 219755 747757 -386 799995 752712 -387 193343 94110 -388 898152 679345 -389 69927 994572 -390 820509 737170 -391 216752 102635 -392 559471 259094 -393 193300 885479 -394 331982 890550 -395 811776 714374 -396 274244 546450 -397 723060 818285 -398 737028 348215 -399 212040 553903 -400 225345 178449 -401 667281 704781 -402 288654 471476 -403 777023 68075 -404 178814 607664 -405 153999 971441 -406 919279 999199 -407 685436 521606 -408 584400 163952 -409 411156 317585 -410 464187 324848 -411 982066 240493 -412 857635 41306 -413 80475 95214 -414 464057 721866 -415 317087 830076 -416 500530 462263 -417 840573 849068 -418 817891 217552 -419 417198 37865 -420 674693 29348 -421 474322 14397 -422 426363 486928 -423 732307 33750 -424 688814 681754 -425 737810 641814 -426 884157 490173 -427 236816 395268 -428 677918 205018 -429 864207 439586 -430 653585 559470 -431 650876 140949 -432 932971 124650 -433 497118 904881 -434 572836 198508 -435 789298 550650 -436 475137 729402 -437 579775 822372 -438 440690 491893 -439 3677 462367 -440 363388 875456 -441 231006 24471 -442 68281 757616 -443 179200 359581 -444 529291 715923 -445 351950 913009 -446 644716 218690 -447 248566 124042 -448 554211 744920 -449 434622 603990 -450 46237 240414 -451 30072 226447 -452 318366 862353 -453 410808 859686 -454 421891 479199 -455 216068 318337 -456 675726 148284 -457 87635 740575 -458 914754 432186 -459 892383 808928 -460 570439 752197 -461 470259 968846 -462 152270 548884 -463 520578 248689 -464 411035 717421 -465 411564 581003 -466 70001 524478 -467 246299 45050 -468 199729 82721 -469 936836 327706 -470 842862 747014 -471 467197 720363 -472 145484 599753 -473 442750 675870 -474 66420 699681 -475 603463 305521 -476 333885 717200 -477 192425 465234 -478 170413 505593 -479 980148 273315 -480 662623 328087 -481 922850 94185 -482 636337 469053 -483 851139 955363 -484 2800 487882 -485 297824 238884 -486 365765 192701 -487 205465 264917 -488 418312 753058 -489 776420 687036 -490 378470 14985 -491 268541 137720 -492 54798 83477 -493 658153 975815 -494 888141 777245 -495 193911 244366 -496 79920 448953 -497 29881 603978 -498 381248 274495 -499 514898 880566 -500 181441 689692 -501 899449 379383 -502 321211 588478 -503 36980 196164 -504 662402 108948 -505 807080 194597 -506 617452 202903 -507 79404 418257 -508 248166 473896 -509 64304 32358 -510 87805 576643 -511 440465 122234 -512 306440 608132 -513 339434 986382 -514 871490 616986 -515 227937 222148 -516 90655 667471 -517 879955 183873 -518 397532 65638 -519 58316 636540 -520 835311 184256 -521 911511 855782 -522 689439 617267 -523 803900 957686 -524 142513 421748 -525 264544 394866 -526 403005 897911 -527 652628 959293 -528 599036 231978 -529 499428 137337 -530 190946 971342 -531 137847 25862 -532 273637 622824 -533 283085 761670 -534 513463 462136 -535 614357 290479 -536 331383 642555 -537 767814 692939 -538 173638 542553 -539 469605 347147 -540 9096 107456 -541 486954 734152 -542 426040 256595 -543 84300 64793 -544 393834 257131 -545 900787 635862 -546 552174 596179 -547 22183 544831 -548 852873 268956 -549 87967 996500 -550 443628 261346 -551 488080 795365 -552 412652 157087 -553 907911 668852 -554 471871 396032 -555 874992 534243 -556 838143 242296 -557 236550 555083 -558 419167 541667 -559 3678 728806 -560 769950 14129 -561 673703 516962 -562 18508 353010 -563 802399 536017 -564 229902 610727 -565 785027 504785 -566 70682 73572 -567 472154 474853 -568 269312 244657 -569 497601 870956 -570 837428 542633 -571 61114 665028 -572 487180 886657 -573 962159 35211 -574 577671 669172 -575 742432 316855 -576 39053 477240 -577 211472 938842 -578 190664 414507 -579 323210 937799 -580 887775 663254 -581 171250 600915 -582 558604 332359 -583 473128 173115 -584 755116 349893 -585 592924 383955 -586 963996 334505 -587 986373 453095 -588 975076 196462 -589 305490 79665 -590 162346 387891 -591 212807 292103 -592 722951 121773 -593 333534 469767 -594 514968 139795 -595 1724 96196 -596 489541 147708 -597 278031 453472 -598 578636 726609 -599 678654 34085 -600 911580 765696 -601 729721 498005 -602 506826 354541 -603 104048 746950 -604 754288 89699 -605 605308 720896 -606 899539 183415 -607 936074 791578 -608 173712 23542 -609 322883 105132 -610 753722 746518 -611 494461 721030 -612 438312 827228 -613 654234 465990 -614 827679 631832 -615 349046 228126 -616 442173 215790 -617 474357 441449 -618 488351 29391 -619 356028 931372 -620 948060 309992 -621 146883 409835 -622 386072 255206 -623 601735 768510 -624 709396 450803 -625 799238 112646 -626 898930 46821 -627 685038 683454 -628 323522 513931 -629 23648 65377 -630 866189 74656 -631 390921 822915 -632 141639 295315 -633 574012 489703 -634 797342 680868 -635 189843 405201 -636 314145 872079 -637 305893 641075 -638 847588 447639 -639 35991 754857 -640 503705 140303 -641 442341 762301 -642 765643 274389 -643 837204 619257 -644 74151 179041 -645 867436 998647 -646 232049 675159 -647 741529 542858 -648 995846 274804 -649 103941 744996 -650 407617 154096 -651 732519 954538 -652 947097 658935 -653 670305 136628 -654 281178 410648 -655 846249 704264 -656 439779 844606 -657 197941 867541 -658 842607 715761 -659 81386 598780 -660 299469 299208 -661 385762 52346 -662 273251 35746 -663 672682 359607 -664 924981 646958 -665 970770 710960 -666 166461 625342 -667 908607 799440 -668 700523 597734 -669 564360 898101 -670 431782 121442 -671 537871 475371 -672 879572 568227 -673 612885 179702 -674 401908 705782 -675 870176 636239 -676 349822 456983 -677 774225 696657 -678 987634 107176 -679 45930 147657 -680 958411 72570 -681 572267 383076 -682 978866 724806 -683 166392 964408 -684 965033 629713 -685 254655 662904 -686 313852 375603 -687 728604 663229 -688 949386 928778 -689 278120 576593 -690 48112 565506 -691 313676 777324 -692 688546 898200 -693 138692 783384 -694 646475 183800 -695 633048 736115 -696 632700 934647 -697 643446 768877 -698 807589 162268 -699 746767 216342 -700 618841 684106 -701 901582 825314 -702 657670 304670 -703 322562 572497 -704 768436 876024 -705 557965 204250 -706 460700 862129 -707 514608 222296 -708 439869 637620 -709 739630 209989 -710 917217 4124 -711 217640 748691 -712 10872 570548 -713 761322 488537 -714 717932 423933 -715 340667 376888 -716 160342 402095 -717 18628 843861 -718 104805 451547 -719 262715 466250 -720 458331 501071 -721 43754 436486 -722 266583 628923 -723 518474 884009 -724 923774 72897 -725 7554 319051 -726 444336 322833 -727 875675 241952 -728 523764 499486 -729 806686 813809 -730 199864 871015 -731 309781 302185 -732 417692 56893 -733 160495 24214 -734 595545 885685 -735 703822 555859 -736 713846 666732 -737 202434 598165 -738 559787 894807 -739 873015 768920 -740 46784 261835 -741 681851 904123 -742 224069 469651 -743 67106 858157 -744 984402 961734 -745 683366 80590 -746 856001 377030 -747 762428 902472 -748 787224 377021 -749 234051 668418 -750 69135 623666 -751 10994 154854 -752 26112 745719 -753 637200 540212 -754 98764 406023 -755 174845 665606 -756 515084 844951 -757 130442 119273 -758 15014 932751 -759 539757 515220 -760 269668 783473 -761 790163 927127 -762 816549 80155 -763 544864 558991 -764 640619 456715 -765 960965 19575 -766 796043 466991 -767 594074 381177 -768 746751 836900 -769 773680 104795 -770 454637 134355 -771 318400 469182 -772 692066 899892 -773 290427 928874 -774 560481 682273 -775 357608 228233 -776 736402 777335 -777 707453 49560 -778 827623 544002 -779 560780 644935 -780 998968 800299 -781 766532 993968 -782 951385 40490 -783 347206 45901 -784 152885 230549 -785 828846 86139 -786 372269 857483 -787 157612 41435 -788 47071 12827 -789 219673 766988 -790 252531 861 -791 998211 995684 -792 457747 160665 -793 253043 802074 -794 515605 553584 -795 33970 700849 -796 684015 543130 -797 275949 368498 -798 762085 460916 -799 311570 650630 -800 852821 277599 -801 709200 793493 -802 429741 356746 -803 230021 740717 -804 319588 546788 -805 796517 25548 -806 28396 7195 -807 610964 298118 -808 116284 223401 -809 718019 582887 -810 278404 886290 -811 734331 502254 -812 377728 551246 -813 376939 578776 -814 427742 800865 -815 811414 306353 -816 693239 672885 -817 970470 226982 -818 972464 991015 -819 384720 159628 -820 44380 29641 -821 84055 932718 -822 275179 147680 -823 966518 181760 -824 165402 724702 -825 991558 183309 -826 33174 510704 -827 839216 546467 -828 584360 36315 -829 823022 202758 -830 384281 239005 -831 355997 159959 -832 502407 766875 -833 941493 95678 -834 732015 463284 -835 331600 934523 -836 57998 993316 -837 591328 95094 -838 853116 223627 -839 663038 831260 -840 966886 340623 -841 755753 224983 -842 416584 572408 -843 950356 533280 -844 170477 203594 -845 285489 876786 -846 258999 921435 -847 828028 109856 -848 971643 90738 -849 341389 180085 -850 294563 742890 -851 518722 334141 -852 757816 650935 -853 427556 808191 -854 94119 266807 -855 596111 51079 -856 865838 619427 -857 917269 507495 -858 980005 434561 -859 331930 392550 -860 795232 850754 -861 754289 551930 -862 168720 588709 -863 415801 723857 -864 235499 940393 -865 667538 44925 -866 129507 396230 -867 235149 915806 -868 474785 136325 -869 307713 909089 -870 592403 515794 -871 201349 777926 -872 408362 434735 -873 122496 707069 -874 752715 239318 -875 694055 247785 -876 855239 400995 -877 512547 249638 -878 613383 122491 -879 98992 842009 -880 176149 291230 -881 500477 185030 -882 674404 80316 -883 849730 87911 -884 211065 482534 -885 384998 272935 -886 681846 92611 -887 698495 547446 -888 995515 353293 -889 39383 919082 -890 975326 293309 -891 624864 393490 -892 764243 376307 -893 544448 944477 -894 721826 154832 -895 66076 386873 -896 925259 825179 -897 524091 319467 -898 833948 108737 -899 79431 860915 -900 439219 769203 -901 667686 833633 -902 263992 400745 -903 622921 461748 -904 636752 136239 -905 705190 668905 -906 400665 944159 -907 775933 789276 -908 365970 675297 -909 660939 354936 -910 246367 740992 -911 8480 350150 -912 43314 615794 -913 605249 848212 -914 828619 297749 -915 924525 533766 -916 716541 903143 -917 213891 306420 -918 892644 680705 -919 617556 974967 -920 10336 869150 -921 283538 366889 -922 908464 325084 -923 378393 575109 -924 781865 908296 -925 714218 985735 -926 280118 781681 -927 936389 905453 -928 52662 764543 -929 619741 957572 -930 508101 942216 -931 844191 661784 -932 125903 836039 -933 385367 33775 -934 35694 450848 -935 410882 790861 -936 893432 752642 -937 640718 260632 -938 460873 226798 -939 413760 137861 -940 563132 840705 -941 228471 871047 -942 789647 982309 -943 689574 54757 -944 777239 377852 -945 921053 858868 -946 645010 166707 -947 564084 219474 -948 975718 530895 -949 726170 647832 -950 864210 151594 -951 161348 644003 -952 345163 873512 -953 757264 409071 -954 992033 954080 -955 215878 997905 -956 987303 241889 -957 36519 649233 -958 297206 677483 -959 97699 906428 -960 860321 854471 -961 307605 387862 -962 586638 259288 -963 629513 249428 -964 407478 767206 -965 503367 51802 -966 234765 459680 -967 921982 565227 -968 853402 986581 -969 834528 140414 -970 685103 12090 -971 957057 870811 -972 517531 66581 -973 551263 257148 -974 580069 304796 -975 589961 726289 -976 123417 958963 -977 144465 812407 -978 916829 701668 -979 722021 779936 -980 20110 770682 -981 574542 851618 -982 268977 203788 -983 40847 116492 -984 724358 969937 -985 97969 40057 -986 97414 792902 -987 316466 134031 -988 731054 348641 -989 243396 774230 -990 342459 927844 -991 527406 627542 -992 747096 732684 -993 477259 383146 -994 190703 718194 -995 524380 736909 -996 262223 864590 -997 42445 645045 -998 914676 164154 -999 554344 383499 -1000 335526 202621 diff --git a/devel-examples/vignette-example/instances/1000-15.tsp b/devel-examples/vignette-example/instances/1000-15.tsp deleted file mode 100644 index e5353d75..00000000 --- a/devel-examples/vignette-example/instances/1000-15.tsp +++ /dev/null @@ -1,1006 +0,0 @@ -NAME : portgen-1000-15 -COMMENT : portgen N=1000, seed=15 -TYPE : TSP -DIMENSION : 1000 -EDGE_WEIGHT_TYPE : EUC_2D -NODE_COORD_SECTION -1 284097 798328 -2 826047 8361 -3 402225 356674 -4 916334 650522 -5 889673 948416 -6 706043 621016 -7 100009 234950 -8 538835 164509 -9 404021 603359 -10 578795 488706 -11 845528 125426 -12 678589 241372 -13 967309 813034 -14 15021 562143 -15 724089 779318 -16 171992 439475 -17 558567 854638 -18 730202 439680 -19 742454 245762 -20 88040 393219 -21 641977 717296 -22 258308 325208 -23 554742 954960 -24 113761 516687 -25 788795 812383 -26 772743 929571 -27 205166 458809 -28 255952 844621 -29 239760 971409 -30 278158 962544 -31 614219 670572 -32 562481 496454 -33 306438 988747 -34 362708 774800 -35 680208 583875 -36 50748 887334 -37 814563 766412 -38 715962 915956 -39 920259 219780 -40 985419 122688 -41 573273 43611 -42 283984 761544 -43 165098 501419 -44 876994 62113 -45 548199 741455 -46 76971 967653 -47 565554 504165 -48 342471 754643 -49 902732 491895 -50 609246 638785 -51 34700 893980 -52 531268 666107 -53 239109 729131 -54 645586 443621 -55 293710 754533 -56 967627 177647 -57 423210 536703 -58 885573 646565 -59 105018 58316 -60 153982 551795 -61 86014 870812 -62 165554 41423 -63 549175 156767 -64 356065 148456 -65 527302 986830 -66 270370 476637 -67 926069 230886 -68 155060 395626 -69 620401 747281 -70 875971 518532 -71 396400 818677 -72 908130 996403 -73 655441 206159 -74 802099 524131 -75 954989 185704 -76 398577 754276 -77 964593 622415 -78 368414 558062 -79 967910 300382 -80 511046 843483 -81 108730 898305 -82 567650 775177 -83 358132 148949 -84 269516 426806 -85 881262 679413 -86 844466 580887 -87 103326 968278 -88 153218 331737 -89 906219 543138 -90 673008 991112 -91 188856 55683 -92 637409 683819 -93 878100 372065 -94 908987 150892 -95 872753 6111 -96 126110 193594 -97 866019 196557 -98 674066 815513 -99 715351 939852 -100 843185 323703 -101 299939 258960 -102 851122 963876 -103 996847 342893 -104 116867 280773 -105 744315 996349 -106 649074 817018 -107 427628 504935 -108 717373 915135 -109 32285 371092 -110 101111 542619 -111 433598 329664 -112 583621 557558 -113 379474 585505 -114 729765 139449 -115 971431 810324 -116 214870 625445 -117 798823 676659 -118 342038 371838 -119 628055 132474 -120 966446 962964 -121 339779 537895 -122 49781 330134 -123 572513 796445 -124 609973 308460 -125 817083 88561 -126 85748 575901 -127 968420 32860 -128 108833 674494 -129 460137 174463 -130 621838 625009 -131 714838 984393 -132 314327 781350 -133 656569 111178 -134 767237 97494 -135 932422 920927 -136 305162 723824 -137 554008 12549 -138 456871 857696 -139 361243 550760 -140 448724 704979 -141 125368 555301 -142 517611 346422 -143 95486 230476 -144 311117 17472 -145 20089 230859 -146 604600 530560 -147 200051 45518 -148 657801 615954 -149 983886 37231 -150 873262 714816 -151 435202 59212 -152 859736 112103 -153 963193 530446 -154 58290 621998 -155 937373 878356 -156 363376 442664 -157 154374 390978 -158 20408 184277 -159 784341 268809 -160 123549 40615 -161 127292 730006 -162 224231 217605 -163 485724 245950 -164 864088 441904 -165 49356 926424 -166 799405 739244 -167 613387 570368 -168 341603 682703 -169 400927 126632 -170 326013 911209 -171 446135 42308 -172 893923 979474 -173 103567 874594 -174 306329 982445 -175 559793 411851 -176 751865 541982 -177 987874 946838 -178 915410 695957 -179 445825 289367 -180 770500 280489 -181 129519 931658 -182 295985 26164 -183 432221 321067 -184 548740 174900 -185 287411 145814 -186 877947 801895 -187 709015 711697 -188 288749 585309 -189 742131 277392 -190 302195 789767 -191 800125 574721 -192 671404 768866 -193 796905 867747 -194 443259 587223 -195 138147 20535 -196 133962 226027 -197 839220 180198 -198 33261 644239 -199 333293 182226 -200 690724 518257 -201 132462 28936 -202 680250 770026 -203 611726 177144 -204 870577 219008 -205 149933 47663 -206 252698 858602 -207 151220 749964 -208 146526 903492 -209 92438 115787 -210 992902 787981 -211 987774 366514 -212 484175 769154 -213 13351 849010 -214 121645 938988 -215 99971 111605 -216 714732 523122 -217 127459 254531 -218 537069 941523 -219 423500 921439 -220 622339 893412 -221 775309 327472 -222 594320 350165 -223 32761 767448 -224 741851 70066 -225 166846 184250 -226 522593 394304 -227 82254 579119 -228 803525 609339 -229 901477 425719 -230 232957 670203 -231 753644 949137 -232 596668 256520 -233 272354 925226 -234 264281 801055 -235 717203 379078 -236 161641 22371 -237 948940 808651 -238 265387 593469 -239 284259 905056 -240 965629 404012 -241 947533 695926 -242 706030 429767 -243 357960 765594 -244 926454 870938 -245 982177 611842 -246 677241 622444 -247 204236 243261 -248 731771 752937 -249 378531 785669 -250 84777 439291 -251 483188 836794 -252 104437 762834 -253 236717 826667 -254 688274 652487 -255 221158 37930 -256 682885 30538 -257 443541 621114 -258 992961 131199 -259 744901 353406 -260 524749 519416 -261 546694 478611 -262 716278 277911 -263 895889 324847 -264 917934 186106 -265 571934 438720 -266 905195 631772 -267 683898 927698 -268 721126 916994 -269 252384 84915 -270 436806 226761 -271 20693 573047 -272 346188 462760 -273 65148 198629 -274 906166 926324 -275 347371 406924 -276 835003 192424 -277 213735 646056 -278 534096 851416 -279 152895 176738 -280 41707 319722 -281 574283 603358 -282 215680 994396 -283 17237 109837 -284 684350 980781 -285 555966 794331 -286 225033 818576 -287 6035 117825 -288 684413 354270 -289 264876 70221 -290 743815 44473 -291 171951 741195 -292 144398 252212 -293 864436 301836 -294 416091 689501 -295 910461 611288 -296 232644 271602 -297 528949 642474 -298 1728 202116 -299 567012 228363 -300 778347 710878 -301 933752 835944 -302 182509 302897 -303 234972 93808 -304 48025 961522 -305 781620 232259 -306 435324 463393 -307 266277 430418 -308 87077 302680 -309 74408 573206 -310 992668 815121 -311 542825 455986 -312 202434 845088 -313 860578 389088 -314 636067 703137 -315 882852 590604 -316 306244 303353 -317 288601 511555 -318 609149 708557 -319 474917 713980 -320 165134 561755 -321 227427 842885 -322 696833 95340 -323 68463 776658 -324 69168 683860 -325 781895 612639 -326 566048 863874 -327 345510 187742 -328 404633 630398 -329 547343 670954 -330 693748 526415 -331 618890 334045 -332 796388 219864 -333 4831 592438 -334 766560 170937 -335 361955 310419 -336 233512 390548 -337 791310 380028 -338 249072 678950 -339 110476 14692 -340 440454 230179 -341 841744 965113 -342 9388 356437 -343 971714 972198 -344 506965 68736 -345 506723 16710 -346 941997 303979 -347 352025 854715 -348 328243 836879 -349 51574 316805 -350 846267 389512 -351 666181 54476 -352 243405 551715 -353 770895 600935 -354 854485 989072 -355 216028 432434 -356 123433 478606 -357 164230 677038 -358 584417 314910 -359 982020 941672 -360 891621 167952 -361 540864 449754 -362 324669 972442 -363 644237 179036 -364 147142 239594 -365 609133 648136 -366 824464 121403 -367 798663 8019 -368 106746 363138 -369 800882 332350 -370 772019 656803 -371 990178 565293 -372 177115 338771 -373 475846 492242 -374 979309 379582 -375 210477 149206 -376 689737 811979 -377 707672 198131 -378 565047 544777 -379 255812 235386 -380 444968 407756 -381 800053 522135 -382 217053 559224 -383 442256 558139 -384 301490 825459 -385 201191 92175 -386 335600 602437 -387 731194 742414 -388 478214 728884 -389 742082 126538 -390 407394 99460 -391 923223 911756 -392 794626 201376 -393 848082 302329 -394 904350 239439 -395 565763 548607 -396 61647 975422 -397 131158 679844 -398 321202 387741 -399 834098 434701 -400 860556 746961 -401 750159 852771 -402 972188 111017 -403 225982 777981 -404 17353 506295 -405 350048 262718 -406 640427 16373 -407 669622 896361 -408 346108 824631 -409 390976 537209 -410 238022 54514 -411 724041 866788 -412 964903 454230 -413 342015 482829 -414 630248 620176 -415 516431 700233 -416 711530 235787 -417 776490 144676 -418 459032 906850 -419 242133 898264 -420 855267 23451 -421 911353 367141 -422 1416 511248 -423 824566 194859 -424 10518 676928 -425 337828 838372 -426 757492 213921 -427 918270 160322 -428 35430 514372 -429 76280 827512 -430 651985 319132 -431 535847 119088 -432 651027 326596 -433 351364 273285 -434 14956 158373 -435 71795 151249 -436 814112 714047 -437 199380 399649 -438 297022 510119 -439 948517 804581 -440 418799 827642 -441 406549 802736 -442 968191 197298 -443 164386 592442 -444 584759 449894 -445 793312 185746 -446 891893 83760 -447 826469 704018 -448 209339 197306 -449 167761 601767 -450 214226 314446 -451 246341 205937 -452 258129 510185 -453 431823 954756 -454 245729 720971 -455 995936 442988 -456 929612 626385 -457 34200 466238 -458 427238 452087 -459 292619 947009 -460 117257 154058 -461 105524 413189 -462 944146 757348 -463 904908 608174 -464 455918 689194 -465 967826 342266 -466 264389 227545 -467 808644 975810 -468 898030 780163 -469 768536 501953 -470 770060 712299 -471 299822 637750 -472 332637 639253 -473 80222 478704 -474 139613 69120 -475 799110 601164 -476 741387 478567 -477 633940 871960 -478 50057 18795 -479 397292 282318 -480 612155 651659 -481 186219 743776 -482 950911 283636 -483 143165 291861 -484 293748 394946 -485 386015 948533 -486 312473 223498 -487 147898 516092 -488 412670 626957 -489 779248 72186 -490 707291 886112 -491 210882 173599 -492 77038 316167 -493 156046 520613 -494 276634 525007 -495 832644 606168 -496 486414 373590 -497 115938 821526 -498 399825 76324 -499 489851 816544 -500 226582 453264 -501 699455 67427 -502 361829 912997 -503 390282 567787 -504 401528 317773 -505 715913 529443 -506 742160 872284 -507 449673 5987 -508 165244 812629 -509 627837 129385 -510 883811 66841 -511 802009 477204 -512 168363 932750 -513 249078 245045 -514 861669 234901 -515 125810 844882 -516 225428 461475 -517 356272 177847 -518 143952 338597 -519 723925 71051 -520 150922 343417 -521 892775 147249 -522 641196 765803 -523 804158 9210 -524 205226 183187 -525 572448 154779 -526 214655 254949 -527 690733 381700 -528 227836 237980 -529 711155 183981 -530 769044 51685 -531 843861 330477 -532 166850 372496 -533 636667 594911 -534 231088 683870 -535 201828 156034 -536 607402 444650 -537 556936 729031 -538 852186 547059 -539 786471 786663 -540 704914 11097 -541 533890 677687 -542 465856 74125 -543 1020 894951 -544 294624 983775 -545 541179 549040 -546 107509 40055 -547 869222 994888 -548 736014 448125 -549 590312 912165 -550 913616 257098 -551 222738 418563 -552 478272 561350 -553 620889 536967 -554 789093 616608 -555 380679 332884 -556 943356 727379 -557 642801 220004 -558 944176 803806 -559 461254 171962 -560 636481 188542 -561 4598 318922 -562 770253 944729 -563 933295 188839 -564 966377 995586 -565 108141 315218 -566 757966 169863 -567 405984 372030 -568 67741 806510 -569 34885 245852 -570 129949 197214 -571 433697 122661 -572 347293 352636 -573 544441 788586 -574 269801 924493 -575 61593 547175 -576 481747 594726 -577 804023 598398 -578 499131 52875 -579 12578 106242 -580 493609 814378 -581 502081 543240 -582 486658 183465 -583 899187 820694 -584 380086 290164 -585 675562 155589 -586 534005 536761 -587 110369 89306 -588 706795 409872 -589 514898 171855 -590 445597 880419 -591 176260 860135 -592 501976 293763 -593 813136 214726 -594 683204 222519 -595 472843 247046 -596 426424 744721 -597 570289 974359 -598 663208 896936 -599 12292 257986 -600 645841 134569 -601 273687 97945 -602 478895 181173 -603 370914 621612 -604 92750 510260 -605 785261 284405 -606 369671 790059 -607 633399 246563 -608 387954 757360 -609 972950 512298 -610 520256 2251 -611 808402 122099 -612 644323 540993 -613 881901 436059 -614 57865 929195 -615 718392 85183 -616 317122 4638 -617 386593 161192 -618 510748 386201 -619 226736 255413 -620 905808 55775 -621 241775 170905 -622 702262 470592 -623 438643 304324 -624 100397 29296 -625 92458 227148 -626 839070 83096 -627 539593 560658 -628 817446 269049 -629 711352 317703 -630 670425 984712 -631 394875 837336 -632 604452 729485 -633 42630 198765 -634 87796 162806 -635 807919 83630 -636 656962 943654 -637 419840 293108 -638 163180 725306 -639 582506 83665 -640 723546 612852 -641 724707 740162 -642 258770 733680 -643 690307 479785 -644 400186 657107 -645 118562 311983 -646 298405 63929 -647 447493 822177 -648 398813 298120 -649 751065 409154 -650 307411 713337 -651 721818 16732 -652 305749 479605 -653 502440 98907 -654 824325 805913 -655 870350 337661 -656 868863 54244 -657 199141 358442 -658 686307 330946 -659 389843 782274 -660 330672 744509 -661 447699 678642 -662 855395 94582 -663 361812 640229 -664 637905 940234 -665 790668 64272 -666 900980 776592 -667 213314 385885 -668 743989 670463 -669 541021 900328 -670 47372 359360 -671 89941 617912 -672 326434 374052 -673 864283 619762 -674 208533 352910 -675 460365 758583 -676 660215 810830 -677 618486 243138 -678 812357 945225 -679 803418 919864 -680 735616 831976 -681 557886 923997 -682 758540 510989 -683 247719 250951 -684 727809 825088 -685 494158 66544 -686 122412 36932 -687 321908 572089 -688 84293 636869 -689 60156 612256 -690 282225 416586 -691 836811 718040 -692 204617 958691 -693 506386 976982 -694 18052 702324 -695 138165 493037 -696 942653 715933 -697 406169 980827 -698 236947 53009 -699 296003 754345 -700 289759 227413 -701 559606 596276 -702 70685 43778 -703 921771 942174 -704 606213 659795 -705 736752 835374 -706 927173 101093 -707 781698 242579 -708 889322 841953 -709 517827 777712 -710 274041 194710 -711 954947 973464 -712 535328 266745 -713 506938 526136 -714 966246 278129 -715 650317 142118 -716 30656 400361 -717 875504 446851 -718 489413 735717 -719 936342 962038 -720 69368 664432 -721 459154 240339 -722 428283 943455 -723 538090 969189 -724 180604 139424 -725 473889 710811 -726 86762 17874 -727 104028 147640 -728 196757 159245 -729 720772 623834 -730 554365 186054 -731 5832 644174 -732 590427 72319 -733 376219 686833 -734 672810 838243 -735 704488 920133 -736 661349 378403 -737 303823 563229 -738 107947 937073 -739 869435 387687 -740 69987 347693 -741 805363 342412 -742 723764 464263 -743 136285 386481 -744 809933 803185 -745 70631 802579 -746 62907 98098 -747 257549 149235 -748 3083 80751 -749 936516 865053 -750 835507 601016 -751 99753 792916 -752 69437 126196 -753 905447 744349 -754 294109 639764 -755 11355 810276 -756 349311 917586 -757 553202 751785 -758 123146 907733 -759 386625 441192 -760 69235 295468 -761 750317 807756 -762 2735 103471 -763 820379 868432 -764 308966 177626 -765 657781 363597 -766 642963 229671 -767 376332 259710 -768 998381 887777 -769 789210 971978 -770 341116 228551 -771 999856 368741 -772 733949 775163 -773 52262 255150 -774 95363 154077 -775 328856 134650 -776 771784 758889 -777 207271 471910 -778 958053 870082 -779 416583 809726 -780 127814 17670 -781 955139 322131 -782 298648 782803 -783 810420 980569 -784 183636 778038 -785 699523 867996 -786 812370 232547 -787 112335 934584 -788 523683 991427 -789 600485 530825 -790 145418 950296 -791 451849 499239 -792 49812 640110 -793 408458 320832 -794 931022 593528 -795 449290 17812 -796 704140 11171 -797 272455 473120 -798 416181 767309 -799 256406 799365 -800 251480 60834 -801 654665 564538 -802 8658 378559 -803 682801 272545 -804 709077 567160 -805 63451 637221 -806 939059 823054 -807 360435 110001 -808 313530 943967 -809 49676 825527 -810 366622 43111 -811 724163 384271 -812 526557 638688 -813 213330 247831 -814 223888 733776 -815 251782 251138 -816 282350 33324 -817 467373 508197 -818 11236 628794 -819 138804 939810 -820 326580 464490 -821 271155 105495 -822 226906 406179 -823 293649 319868 -824 484613 633766 -825 259789 168349 -826 543420 522629 -827 547582 342 -828 778483 621340 -829 97165 500461 -830 367323 54007 -831 133740 769266 -832 240580 598961 -833 366065 833564 -834 596148 954256 -835 816352 993661 -836 459353 415910 -837 565737 198272 -838 499690 201533 -839 836689 526215 -840 860204 591989 -841 150666 723427 -842 366452 197775 -843 117397 513084 -844 792744 868412 -845 142132 177671 -846 32646 184547 -847 123457 332918 -848 5136 855245 -849 539757 28633 -850 906489 92115 -851 483179 958398 -852 773561 667799 -853 17682 819993 -854 156177 349806 -855 882945 265399 -856 828596 228752 -857 358328 189651 -858 21361 949193 -859 645808 907661 -860 593824 510819 -861 293807 567514 -862 47767 724237 -863 510482 500954 -864 642348 897937 -865 180589 679696 -866 45356 486882 -867 643269 594804 -868 763393 921913 -869 365098 176801 -870 176414 168203 -871 867275 885083 -872 274587 631313 -873 883864 465131 -874 136780 399220 -875 822436 504181 -876 212897 641819 -877 848043 226792 -878 46759 996297 -879 315129 178756 -880 904406 95769 -881 454895 979375 -882 173392 714741 -883 398124 943513 -884 954164 727015 -885 305786 556229 -886 812412 246588 -887 85224 89642 -888 297922 651987 -889 719470 820974 -890 677477 514184 -891 185825 463591 -892 993531 84820 -893 224801 65980 -894 313490 928527 -895 196680 819880 -896 967749 638082 -897 871014 620185 -898 355791 620687 -899 799858 184945 -900 333390 231876 -901 745661 315806 -902 721742 308251 -903 318356 749305 -904 648287 763223 -905 1991 980778 -906 682807 386602 -907 982076 84526 -908 128019 816813 -909 108360 553207 -910 358950 777437 -911 143654 769219 -912 393624 73909 -913 810568 496606 -914 524845 776972 -915 771285 548616 -916 3700 956247 -917 818982 696699 -918 831377 799223 -919 481515 909005 -920 956800 407987 -921 957619 760282 -922 569576 419242 -923 676225 198529 -924 244457 797104 -925 809616 859185 -926 95841 22885 -927 413659 784774 -928 228176 789413 -929 496823 25043 -930 476874 519133 -931 267789 739282 -932 806422 594003 -933 23158 922524 -934 817025 562833 -935 408300 929489 -936 572355 311256 -937 743590 499765 -938 681595 120769 -939 355559 608850 -940 845732 21154 -941 999782 499802 -942 300098 252152 -943 280826 264417 -944 149825 224979 -945 673541 908853 -946 982198 918682 -947 500704 27310 -948 835632 646363 -949 16692 69811 -950 737646 555455 -951 842970 635607 -952 951372 788462 -953 859402 596039 -954 722786 161506 -955 503947 963759 -956 639588 271844 -957 351501 568020 -958 536935 349107 -959 238577 779111 -960 758371 376794 -961 905831 747213 -962 825186 852844 -963 86519 936748 -964 359883 955127 -965 640363 85556 -966 397982 194052 -967 104903 881973 -968 381565 727938 -969 148300 732077 -970 715217 931719 -971 25840 370713 -972 466608 296746 -973 3022 234984 -974 93495 647860 -975 940791 898884 -976 286479 61565 -977 429448 652090 -978 157473 648917 -979 530704 69398 -980 406897 131463 -981 447738 990709 -982 446289 572227 -983 937918 268875 -984 805236 54755 -985 564998 301950 -986 255611 590717 -987 838320 859486 -988 90315 844266 -989 317765 173096 -990 695371 437602 -991 406044 290485 -992 548230 508899 -993 637818 407273 -994 747763 532675 -995 944055 112690 -996 922702 93544 -997 167078 413267 -998 676107 435123 -999 532393 607121 -1000 206431 158755 diff --git a/devel-examples/vignette-example/instances/1000-16.tsp b/devel-examples/vignette-example/instances/1000-16.tsp deleted file mode 100644 index bc578f08..00000000 --- a/devel-examples/vignette-example/instances/1000-16.tsp +++ /dev/null @@ -1,1006 +0,0 @@ -NAME : portgen-1000-16 -COMMENT : portgen N=1000, seed=16 -TYPE : TSP -DIMENSION : 1000 -EDGE_WEIGHT_TYPE : EUC_2D -NODE_COORD_SECTION -1 381413 123204 -2 215229 873526 -3 204543 295123 -4 561950 424603 -5 801554 773301 -6 682861 278344 -7 241870 522409 -8 939870 392656 -9 561862 359081 -10 616900 982902 -11 815593 833922 -12 640487 92921 -13 791872 615309 -14 143793 172157 -15 223382 785616 -16 665398 470350 -17 258771 819396 -18 11005 93519 -19 709732 462660 -20 110062 640048 -21 854472 603 -22 136860 550904 -23 353765 590218 -24 127034 956871 -25 489689 597576 -26 568451 252191 -27 741016 504605 -28 334651 911062 -29 864433 395832 -30 862520 111023 -31 585391 99290 -32 314540 161506 -33 918828 682258 -34 141483 690965 -35 168643 349651 -36 265621 604991 -37 869391 19323 -38 414451 563401 -39 92906 135882 -40 758270 880810 -41 750876 747960 -42 309636 112358 -43 200224 566107 -44 155809 97264 -45 900567 328747 -46 952036 18766 -47 294016 760410 -48 374426 249480 -49 131211 117537 -50 136453 790364 -51 497311 991151 -52 198601 608879 -53 846700 820490 -54 942555 628657 -55 304380 768543 -56 755253 767168 -57 495264 533773 -58 158987 566624 -59 805273 554130 -60 787080 669348 -61 551046 23946 -62 554512 378279 -63 852339 274470 -64 406390 260512 -65 172622 593961 -66 620846 464249 -67 831501 989726 -68 125556 983707 -69 252696 775863 -70 953371 633599 -71 760834 601679 -72 310184 231219 -73 777700 928089 -74 464254 915737 -75 908070 99955 -76 843090 870698 -77 944914 542492 -78 169517 33062 -79 159649 208874 -80 483322 862993 -81 567794 166691 -82 675286 670781 -83 7708 153574 -84 456984 264044 -85 756072 230897 -86 102370 889535 -87 646059 687124 -88 826257 680347 -89 79032 12019 -90 208761 819277 -91 114820 197515 -92 777190 309629 -93 26167 454154 -94 788962 160720 -95 982017 971982 -96 526723 988651 -97 19791 722473 -98 531229 871298 -99 955619 623059 -100 404962 97352 -101 849057 452234 -102 706976 88793 -103 985135 645574 -104 93508 635284 -105 516325 715363 -106 244099 998929 -107 226856 511339 -108 336269 579142 -109 146900 952813 -110 139551 136409 -111 197954 833924 -112 859082 658719 -113 381840 650135 -114 182559 557266 -115 701988 180682 -116 586839 443748 -117 495694 493398 -118 575177 115891 -119 970658 265850 -120 973360 447024 -121 307253 836149 -122 21168 845607 -123 774028 692799 -124 129568 361071 -125 340632 881093 -126 688739 398353 -127 921070 224279 -128 510513 405309 -129 956540 213578 -130 513615 869244 -131 674915 827657 -132 661924 69301 -133 408109 407950 -134 977760 381249 -135 737311 643470 -136 449574 785829 -137 612180 258457 -138 447670 799600 -139 912853 634803 -140 148205 976531 -141 693594 968981 -142 43650 832744 -143 505766 759182 -144 781823 426393 -145 85288 167227 -146 138131 589409 -147 528539 329889 -148 997449 521424 -149 223969 762710 -150 397937 974427 -151 779945 494765 -152 212865 364101 -153 187498 719757 -154 354703 88326 -155 718512 751331 -156 623485 530147 -157 128289 346388 -158 731112 85506 -159 299118 332034 -160 71851 886684 -161 183981 215049 -162 983311 762883 -163 863524 954809 -164 572963 248079 -165 70959 727912 -166 444896 824527 -167 916290 396874 -168 353046 163447 -169 840692 697261 -170 101631 420260 -171 460063 449789 -172 354541 198603 -173 983245 923081 -174 606097 765655 -175 466364 42640 -176 948461 975889 -177 691751 670024 -178 529531 955418 -179 578474 815990 -180 11054 24051 -181 879065 657442 -182 986694 298252 -183 291267 173695 -184 175605 929685 -185 363142 808031 -186 479408 533462 -187 865670 29210 -188 938223 208091 -189 523297 313287 -190 233352 908106 -191 376334 756972 -192 237025 46908 -193 848847 787454 -194 837833 618038 -195 105607 179350 -196 987842 911006 -197 334118 293600 -198 940852 926601 -199 584118 325330 -200 260380 775154 -201 399784 292810 -202 532303 558257 -203 666306 191488 -204 738864 644843 -205 821176 742076 -206 117585 960436 -207 710383 831703 -208 36208 968058 -209 323323 693093 -210 357399 364665 -211 589577 850274 -212 669304 587988 -213 408246 186598 -214 1158 307412 -215 362904 746734 -216 469227 878453 -217 492110 491275 -218 790521 415898 -219 46589 405321 -220 10699 880788 -221 464131 144739 -222 260638 740941 -223 589773 137568 -224 241702 746129 -225 885353 754253 -226 925443 276706 -227 962426 513646 -228 305927 521330 -229 800699 41027 -230 767736 250408 -231 144899 333543 -232 634144 940388 -233 277945 972845 -234 699797 969441 -235 241929 898640 -236 726356 577193 -237 807739 603145 -238 439222 312870 -239 887848 155658 -240 282061 886916 -241 385899 960130 -242 539675 112496 -243 601834 135684 -244 244309 551722 -245 213330 817675 -246 716100 77147 -247 163391 112059 -248 154432 886937 -249 337000 657492 -250 301719 276902 -251 249719 86043 -252 464068 998437 -253 368354 965312 -254 737030 849929 -255 911811 170243 -256 277020 248976 -257 827697 950060 -258 534307 67752 -259 170151 522085 -260 785956 391007 -261 635845 42304 -262 667722 965027 -263 648920 640312 -264 113125 809302 -265 234791 473910 -266 575840 37919 -267 243846 111818 -268 609895 136922 -269 132433 589615 -270 578189 534082 -271 965532 722224 -272 765766 822322 -273 181830 673795 -274 409425 198364 -275 463138 514119 -276 773812 527697 -277 422701 827808 -278 701062 211800 -279 842197 352249 -280 388541 231432 -281 832878 147415 -282 271740 377729 -283 204710 554796 -284 483210 5374 -285 768230 860512 -286 658326 971787 -287 58946 271836 -288 617195 108147 -289 619603 839913 -290 263965 437120 -291 798115 760875 -292 420760 3358 -293 641032 428425 -294 766178 866117 -295 907107 55098 -296 653712 127059 -297 821384 717676 -298 875756 993744 -299 663277 493929 -300 205127 73682 -301 54192 569512 -302 934399 26017 -303 716003 12937 -304 106936 419342 -305 186776 272636 -306 445621 976080 -307 445142 333442 -308 577720 705818 -309 326030 554063 -310 501973 210965 -311 891519 989280 -312 800246 694548 -313 806319 88813 -314 37388 32929 -315 555833 604257 -316 1211 200260 -317 653136 991328 -318 991498 822035 -319 315733 87318 -320 425638 935213 -321 102394 212115 -322 364143 696141 -323 163579 664431 -324 326812 126836 -325 911357 786942 -326 956356 630348 -327 938096 600869 -328 72470 853931 -329 916375 943070 -330 34518 893968 -331 697204 19618 -332 993703 251563 -333 170241 233505 -334 611936 749000 -335 169862 913288 -336 379005 199194 -337 642706 715031 -338 254609 261171 -339 51184 199377 -340 622077 952388 -341 172438 94317 -342 998410 661864 -343 907053 981592 -344 206556 401573 -345 821086 757993 -346 210099 566732 -347 917455 512350 -348 556207 903200 -349 569409 649112 -350 441532 902408 -351 613246 127435 -352 504758 958969 -353 614503 862038 -354 631937 276231 -355 693815 90878 -356 647375 514802 -357 121984 276525 -358 683869 130471 -359 102163 481353 -360 695355 267041 -361 664095 962823 -362 307467 267454 -363 300042 251570 -364 694436 683736 -365 100527 392570 -366 629233 774953 -367 505561 531199 -368 305012 657635 -369 972333 721884 -370 977995 776582 -371 879429 725202 -372 706218 554044 -373 93897 247275 -374 259265 650001 -375 212307 304637 -376 208764 885672 -377 548585 48962 -378 273175 838293 -379 621874 973559 -380 653956 956868 -381 889705 910052 -382 298235 917233 -383 211448 922172 -384 808584 567939 -385 182628 436593 -386 871205 452162 -387 269045 390717 -388 58277 778423 -389 414238 258505 -390 994278 461749 -391 629695 720877 -392 29780 143659 -393 502865 719180 -394 476717 588327 -395 319750 382840 -396 849051 404394 -397 539256 541402 -398 905376 427267 -399 456156 315500 -400 495767 315474 -401 833037 760 -402 655722 750558 -403 674942 487887 -404 855892 404926 -405 546097 553994 -406 361575 33547 -407 653808 271116 -408 107816 485310 -409 370796 756832 -410 11857 784181 -411 466015 493083 -412 72171 867153 -413 603555 870445 -414 796440 518486 -415 715775 570390 -416 922530 9311 -417 712407 440283 -418 100174 596148 -419 67068 758664 -420 35842 17554 -421 348384 719885 -422 576470 535569 -423 916824 355968 -424 332222 672102 -425 937847 34930 -426 630827 937669 -427 599725 925376 -428 392943 823726 -429 288352 215438 -430 650384 78794 -431 420818 97227 -432 369083 528542 -433 205610 641689 -434 457076 118238 -435 354291 751848 -436 153087 698693 -437 818985 976926 -438 153353 528345 -439 893357 146795 -440 474210 779828 -441 582093 581001 -442 868102 636981 -443 149572 825303 -444 640227 183865 -445 234673 458484 -446 139071 948830 -447 404372 283994 -448 864467 649691 -449 900899 599544 -450 382215 388478 -451 462610 185427 -452 197892 158018 -453 452837 49825 -454 69567 962744 -455 775804 567640 -456 183498 104487 -457 980764 191900 -458 939722 471988 -459 692855 85089 -460 664075 555919 -461 740789 857532 -462 736022 965813 -463 289237 967659 -464 500800 660966 -465 524088 103528 -466 458777 930613 -467 370990 906570 -468 596330 477606 -469 600237 676201 -470 697258 677584 -471 132448 555138 -472 519790 678753 -473 717694 281538 -474 212807 438559 -475 994756 896807 -476 148890 239933 -477 75455 278687 -478 929700 531996 -479 814437 291322 -480 561687 975231 -481 449588 393366 -482 265485 98220 -483 435191 628359 -484 584696 302010 -485 474206 658183 -486 259180 254295 -487 90332 767268 -488 407029 500856 -489 782077 457335 -490 36112 757240 -491 153222 209478 -492 99278 548856 -493 653940 65411 -494 665127 272770 -495 471378 967971 -496 892909 298226 -497 201994 39074 -498 418403 878152 -499 464805 752521 -500 271724 216837 -501 499461 755471 -502 402447 237515 -503 743584 939411 -504 140654 526598 -505 624747 864289 -506 866869 541666 -507 819943 593716 -508 82322 151362 -509 191371 226411 -510 679816 557039 -511 163554 832175 -512 30286 257368 -513 158722 503709 -514 851847 852817 -515 23683 467617 -516 360202 255478 -517 832588 171822 -518 890371 611556 -519 389534 505562 -520 466534 502577 -521 874039 438716 -522 592953 914339 -523 804416 60733 -524 267940 944626 -525 880352 914693 -526 26305 611988 -527 728838 804106 -528 856635 243983 -529 922882 230625 -530 347143 132028 -531 549876 635092 -532 60064 122169 -533 990250 428152 -534 948712 905604 -535 789299 21588 -536 883422 246745 -537 346058 765123 -538 530733 551565 -539 103336 226179 -540 400733 914739 -541 580826 621222 -542 505673 891655 -543 917741 725109 -544 195414 710419 -545 181572 462218 -546 662843 483930 -547 716262 444945 -548 619155 627294 -549 92657 827830 -550 383605 252850 -551 957397 41760 -552 543892 965613 -553 333866 405082 -554 106314 837183 -555 886365 131525 -556 48569 212463 -557 49052 884924 -558 469184 65946 -559 918830 615119 -560 503013 362955 -561 335494 120882 -562 521999 536448 -563 64191 841661 -564 702852 380445 -565 431256 125651 -566 445251 266153 -567 339813 269207 -568 866169 368363 -569 572169 620748 -570 422470 851795 -571 806279 580294 -572 207406 818616 -573 126724 541961 -574 961930 179813 -575 380754 777494 -576 924441 712212 -577 396574 257953 -578 807599 691243 -579 701946 274685 -580 99443 965503 -581 832912 485566 -582 414712 34570 -583 325246 468274 -584 5056 230436 -585 758200 927223 -586 104015 739016 -587 234365 725518 -588 438514 623282 -589 724307 264045 -590 728849 372947 -591 139714 428167 -592 281001 465752 -593 292738 959685 -594 851441 305243 -595 943961 397895 -596 363306 341733 -597 862547 495247 -598 747779 67262 -599 345929 481887 -600 380102 503441 -601 817653 697885 -602 450964 7806 -603 637779 496273 -604 431211 930821 -605 965215 847914 -606 839802 396703 -607 330723 701547 -608 602196 491179 -609 623018 919465 -610 286790 257983 -611 122344 523169 -612 850333 254758 -613 109569 406129 -614 288052 226558 -615 87738 942240 -616 192070 793486 -617 298829 880935 -618 533145 743011 -619 97443 579453 -620 863556 801559 -621 336667 931975 -622 18452 685978 -623 275551 840137 -624 491399 607788 -625 385678 341649 -626 779210 119371 -627 394148 437861 -628 311371 24167 -629 399055 570029 -630 474661 894768 -631 398830 851757 -632 67265 163656 -633 511247 907826 -634 378250 644745 -635 425996 762059 -636 999779 15229 -637 533787 945140 -638 478772 2972 -639 129020 412472 -640 943387 85401 -641 7074 718023 -642 751897 192970 -643 543410 340312 -644 726220 135173 -645 369687 625318 -646 364761 452698 -647 153457 101497 -648 801779 321437 -649 398188 73311 -650 207205 272578 -651 711117 78927 -652 664401 300276 -653 334575 61187 -654 367473 201177 -655 894451 971058 -656 297946 263882 -657 200341 849342 -658 530007 946131 -659 698300 965768 -660 361876 189809 -661 509637 304938 -662 437539 153417 -663 50942 920852 -664 350828 233510 -665 610565 417585 -666 635499 927842 -667 518021 972329 -668 787454 743192 -669 517682 902554 -670 662963 597278 -671 642012 760452 -672 773297 179878 -673 115680 59822 -674 15158 39 -675 50555 880927 -676 970609 164678 -677 462746 789619 -678 637079 783274 -679 560906 692071 -680 512822 591383 -681 543505 464918 -682 538214 297172 -683 329046 537494 -684 490585 20462 -685 733662 470184 -686 930972 698261 -687 915212 480949 -688 219200 344959 -689 842192 647919 -690 516338 267667 -691 359945 658756 -692 720688 19181 -693 874080 170580 -694 389628 220848 -695 643283 249960 -696 252607 497219 -697 168892 192778 -698 666306 943750 -699 845239 292347 -700 960677 770720 -701 217630 367238 -702 483700 782887 -703 520981 311852 -704 443990 443565 -705 915539 466498 -706 393646 340057 -707 48788 262861 -708 338776 46285 -709 296025 345435 -710 630866 385295 -711 692254 198237 -712 59785 962941 -713 252554 563734 -714 214561 132325 -715 959967 907348 -716 900969 398626 -717 732380 49840 -718 874021 19888 -719 609968 457826 -720 680404 827794 -721 874555 44193 -722 589982 257987 -723 557706 54369 -724 437433 205951 -725 940224 102571 -726 729188 712914 -727 332380 53329 -728 869751 819003 -729 634858 433859 -730 908866 501093 -731 701883 986163 -732 763161 87744 -733 591943 349453 -734 750074 790801 -735 705155 284406 -736 608852 90074 -737 405211 528294 -738 656107 979339 -739 865856 6455 -740 93189 433550 -741 928876 780701 -742 223459 458874 -743 205464 914805 -744 635465 644635 -745 457897 524568 -746 269813 819167 -747 752670 395997 -748 218942 784480 -749 638981 61688 -750 601879 578366 -751 188513 430978 -752 112761 506673 -753 173695 948486 -754 489455 873505 -755 847864 954945 -756 183538 990222 -757 975962 384297 -758 231279 882716 -759 233492 367163 -760 868802 807462 -761 710471 688386 -762 188921 126788 -763 95893 177873 -764 977312 898538 -765 354598 707620 -766 489883 992350 -767 158590 138243 -768 250012 938653 -769 804739 839161 -770 227595 322747 -771 681312 268301 -772 775833 650434 -773 814097 581427 -774 630245 625882 -775 300104 41068 -776 807167 740443 -777 707089 894259 -778 88482 196162 -779 272387 974517 -780 256660 235041 -781 143746 650294 -782 645910 525117 -783 273633 915237 -784 214388 325527 -785 570200 649852 -786 252471 607610 -787 67059 827733 -788 294 970027 -789 981296 294662 -790 38305 899730 -791 905486 2795 -792 641877 119556 -793 563873 839589 -794 346440 633473 -795 864610 334775 -796 724265 479212 -797 268960 577742 -798 70276 73701 -799 201241 948100 -800 650140 844069 -801 600130 335583 -802 587576 400373 -803 135581 804372 -804 98566 587532 -805 330385 248893 -806 849722 638913 -807 109907 921885 -808 510776 664534 -809 381333 68167 -810 454840 199931 -811 713995 266288 -812 675387 726131 -813 49722 916887 -814 20034 666686 -815 692151 195922 -816 871461 393764 -817 964276 789412 -818 50008 266572 -819 892887 719992 -820 608780 899339 -821 458256 278272 -822 178633 664678 -823 620780 457976 -824 803825 542829 -825 528020 153389 -826 53667 534555 -827 255948 454217 -828 972607 206365 -829 371307 798163 -830 350365 869009 -831 911484 378573 -832 978752 431046 -833 790637 571450 -834 460280 445228 -835 301105 52799 -836 860708 838503 -837 540146 301451 -838 146264 179439 -839 10339 221169 -840 753523 843356 -841 545579 221870 -842 316321 823142 -843 284437 492887 -844 415011 533229 -845 998775 478557 -846 806292 447659 -847 418887 555980 -848 38630 619752 -849 738125 877182 -850 518414 441340 -851 447636 582656 -852 789306 684664 -853 607809 831796 -854 218234 432805 -855 169779 479720 -856 791354 838077 -857 799388 871807 -858 62716 463824 -859 959686 422771 -860 392415 170884 -861 833325 583098 -862 926813 859764 -863 605162 278052 -864 49197 855482 -865 693641 314467 -866 961205 577534 -867 51389 273803 -868 52001 707502 -869 422482 444513 -870 760425 820612 -871 533200 992239 -872 140813 827891 -873 645232 223193 -874 520846 559122 -875 950817 760577 -876 570555 882642 -877 183540 203946 -878 480135 870102 -879 531266 515503 -880 632662 900307 -881 409313 773721 -882 672380 349167 -883 946519 799114 -884 697263 971497 -885 226575 839522 -886 942978 400564 -887 471954 631838 -888 600328 950682 -889 399557 722866 -890 379629 735059 -891 746785 533693 -892 222819 793334 -893 905153 187484 -894 905153 702222 -895 327284 252886 -896 10238 450985 -897 217938 920902 -898 877633 132636 -899 520285 508975 -900 227562 694550 -901 823635 797979 -902 179492 215757 -903 13791 36861 -904 659823 390206 -905 298792 292650 -906 964949 829044 -907 188219 379776 -908 890068 958328 -909 555782 751477 -910 471533 813882 -911 278829 188287 -912 743935 532024 -913 15886 144999 -914 221071 256196 -915 618046 563466 -916 290858 9351 -917 424073 86978 -918 770110 917741 -919 345474 843043 -920 903265 946825 -921 631701 153676 -922 230689 513401 -923 974057 821950 -924 707050 685913 -925 905015 732634 -926 911564 264089 -927 890929 664095 -928 403691 814284 -929 373905 92514 -930 445647 96050 -931 691387 816779 -932 486941 351967 -933 660949 811272 -934 598355 674817 -935 405718 68118 -936 251278 869869 -937 846461 738898 -938 902317 14740 -939 297358 79839 -940 128332 201602 -941 771094 128557 -942 810549 521996 -943 872079 474078 -944 522410 72106 -945 426029 958838 -946 319386 670656 -947 437324 835146 -948 695546 761831 -949 307215 491790 -950 611083 959317 -951 524592 627210 -952 557580 703413 -953 961539 783007 -954 453540 368933 -955 792015 929613 -956 291873 301799 -957 666484 486808 -958 776663 20731 -959 379455 651794 -960 656421 899117 -961 504057 106564 -962 63734 446401 -963 543526 624067 -964 312289 143048 -965 777359 119310 -966 561199 928424 -967 287823 198719 -968 909728 469295 -969 462072 323740 -970 745332 851348 -971 94623 870616 -972 415685 526911 -973 454780 212821 -974 606922 990923 -975 291620 71478 -976 449542 164166 -977 714431 491772 -978 398117 596167 -979 339386 358860 -980 793684 492244 -981 320934 129800 -982 623601 940667 -983 834989 421257 -984 886114 139573 -985 32027 563842 -986 413808 388531 -987 360173 584942 -988 449575 339890 -989 392133 571961 -990 48283 947359 -991 284681 953428 -992 349363 285114 -993 798376 431399 -994 304823 347156 -995 363730 488471 -996 583181 322499 -997 291712 181489 -998 437539 706092 -999 510442 830743 -1000 77336 114890 diff --git a/devel-examples/vignette-example/instances/1000-17.tsp b/devel-examples/vignette-example/instances/1000-17.tsp deleted file mode 100644 index 275ef249..00000000 --- a/devel-examples/vignette-example/instances/1000-17.tsp +++ /dev/null @@ -1,1006 +0,0 @@ -NAME : portgen-1000-17 -COMMENT : portgen N=1000, seed=17 -TYPE : TSP -DIMENSION : 1000 -EDGE_WEIGHT_TYPE : EUC_2D -NODE_COORD_SECTION -1 478728 448080 -2 604410 738692 -3 6861 233573 -4 207567 198684 -5 713435 598186 -6 659680 935671 -7 383731 809869 -8 340904 620802 -9 719704 114802 -10 655004 477098 -11 785658 542419 -12 602385 944470 -13 616435 417584 -14 272565 782171 -15 722674 791913 -16 158804 501225 -17 958974 784155 -18 291809 747359 -19 677010 679559 -20 132083 886876 -21 66967 283910 -22 15412 776601 -23 152789 225476 -24 140307 397054 -25 190582 382769 -26 364158 574811 -27 276866 550401 -28 413349 977503 -29 489105 820255 -30 446882 259502 -31 556563 528007 -32 66600 826558 -33 531219 375769 -34 920258 607130 -35 657079 115427 -36 480495 322649 -37 924220 272234 -38 112940 210846 -39 265553 51984 -40 531120 638931 -41 928478 452310 -42 335288 463172 -43 235350 630796 -44 434624 132415 -45 252936 916039 -46 827100 69879 -47 22479 16656 -48 406380 744318 -49 359690 743178 -50 663660 941943 -51 959922 88322 -52 865934 551650 -53 454291 911848 -54 239523 813693 -55 315050 782553 -56 542879 356690 -57 567319 530843 -58 432401 486683 -59 505527 49944 -60 420177 786900 -61 16079 177080 -62 943469 715135 -63 155504 392173 -64 456714 372569 -65 817943 201091 -66 971322 451860 -67 736933 748567 -68 96052 571788 -69 884990 804445 -70 30770 748666 -71 125268 384680 -72 712237 466035 -73 899960 650019 -74 126409 307344 -75 861151 14207 -76 287603 987120 -77 925235 462569 -78 970621 508062 -79 351388 117367 -80 455598 882502 -81 26857 435078 -82 782922 566384 -83 657285 158198 -84 644452 101283 -85 630882 782381 -86 360274 198183 -87 188792 405970 -88 499296 28958 -89 251845 480900 -90 744514 647441 -91 40784 339347 -92 916971 935440 -93 174233 536243 -94 668937 170549 -95 91281 937854 -96 927336 783707 -97 173562 248388 -98 388391 927084 -99 195888 306267 -100 966739 871002 -101 398174 645508 -102 562829 213709 -103 973423 948256 -104 70149 989794 -105 288335 434377 -106 839125 180839 -107 26085 517744 -108 955166 243149 -109 261515 534533 -110 177992 730200 -111 962310 338184 -112 134544 759880 -113 384207 714765 -114 635354 975082 -115 432546 551040 -116 958809 262050 -117 192565 310137 -118 808316 859944 -119 313262 399227 -120 980274 931083 -121 274728 134403 -122 992556 361081 -123 975543 589152 -124 649163 413682 -125 864181 673626 -126 291730 220805 -127 873721 415698 -128 912193 136123 -129 452943 252692 -130 405393 113478 -131 634993 670921 -132 9520 357251 -133 159649 704721 -134 188282 665003 -135 542200 366014 -136 593986 847833 -137 670352 504366 -138 438470 741504 -139 464463 718845 -140 847687 248083 -141 261821 382661 -142 569689 319067 -143 916046 287887 -144 252530 835313 -145 150488 103594 -146 671661 648258 -147 857027 614260 -148 337097 426895 -149 464051 488190 -150 922611 234039 -151 124688 930318 -152 565995 616098 -153 411804 909068 -154 651116 554653 -155 499651 624305 -156 883593 617629 -157 102204 301798 -158 441817 986735 -159 813894 395260 -160 20153 732754 -161 240670 700092 -162 742392 308161 -163 241325 663668 -164 281837 54254 -165 92561 529401 -166 90387 909809 -167 219193 223381 -168 364489 644192 -169 280457 267890 -170 877250 929311 -171 473992 857269 -172 815159 417733 -173 862924 971569 -174 905866 548866 -175 372934 673429 -176 145057 409797 -177 395628 393209 -178 143651 214879 -179 711124 342614 -180 251608 767612 -181 628611 383225 -182 677402 570339 -183 150313 26323 -184 802469 684470 -185 438874 470247 -186 80869 265028 -187 22325 346724 -188 587696 830873 -189 304463 349182 -190 164509 26445 -191 952543 939223 -192 802645 324949 -193 900789 707162 -194 232406 648853 -195 73068 338165 -196 841722 595986 -197 829016 407003 -198 848442 208964 -199 834944 468435 -200 830037 32050 -201 667105 556683 -202 384356 346488 -203 720885 205833 -204 607151 70679 -205 492420 436489 -206 982472 62270 -207 269546 913442 -208 925890 32625 -209 554209 270399 -210 721897 941349 -211 191379 334034 -212 854433 406823 -213 803142 524185 -214 880671 675836 -215 625838 381863 -216 223721 233784 -217 856762 728020 -218 43973 890273 -219 669677 889203 -220 399059 868164 -221 152953 962007 -222 926956 131718 -223 146786 507687 -224 741553 422193 -225 603861 324256 -226 328292 159107 -227 842597 448173 -228 808329 433321 -229 699921 656336 -230 302515 830612 -231 536155 717948 -232 671620 624255 -233 283536 20465 -234 135314 137827 -235 766656 418202 -236 291071 132015 -237 666538 397640 -238 613057 32272 -239 491437 406259 -240 598494 369820 -241 824264 224335 -242 373321 795225 -243 845708 505773 -244 562164 232506 -245 444484 23508 -246 754959 531850 -247 122546 980856 -248 577092 20937 -249 295468 529316 -250 518661 114513 -251 16250 335293 -252 823699 234040 -253 499992 103956 -254 785786 47371 -255 602465 302555 -256 871156 467414 -257 211852 279006 -258 75653 4304 -259 595401 690763 -260 47162 262598 -261 724996 605997 -262 619165 652142 -263 401951 955778 -264 308316 432498 -265 897647 509100 -266 246486 444065 -267 803794 295938 -268 498664 356850 -269 12483 94314 -270 719571 841403 -271 910371 871401 -272 185343 181885 -273 298512 148961 -274 912685 470403 -275 578904 621314 -276 712621 862970 -277 631668 9560 -278 868027 572184 -279 531498 527760 -280 735375 143142 -281 91472 691471 -282 327799 761061 -283 392184 999755 -284 282070 29966 -285 980494 926692 -286 91619 124997 -287 111858 425848 -288 549977 862025 -289 974329 609604 -290 784115 829767 -291 424279 780555 -292 697122 754505 -293 417627 555014 -294 116266 42732 -295 903754 498908 -296 74779 982516 -297 113819 792879 -298 749783 785373 -299 759542 759495 -300 631907 436486 -301 174631 303080 -302 686288 749137 -303 197034 932066 -304 165847 877162 -305 591932 313013 -306 455918 488766 -307 624006 236466 -308 68362 108956 -309 577652 534920 -310 11278 606810 -311 240212 522574 -312 398058 544008 -313 752060 788539 -314 438709 362720 -315 228813 617911 -316 696178 97166 -317 17671 471101 -318 373848 935513 -319 156548 460656 -320 686142 308671 -321 977361 581346 -322 31454 296943 -323 258696 552204 -324 584457 569811 -325 40818 961244 -326 346664 396821 -327 530681 13996 -328 740308 77464 -329 285408 215186 -330 375289 261521 -331 775517 705191 -332 191019 283261 -333 335651 874572 -334 457312 327062 -335 977770 516158 -336 524499 7841 -337 494101 50034 -338 260146 843391 -339 991892 384062 -340 803700 674596 -341 503131 223522 -342 987431 967292 -343 842393 990986 -344 906147 734410 -345 135449 499276 -346 478201 829485 -347 482885 169984 -348 784172 969520 -349 87244 981420 -350 36797 415304 -351 560311 200395 -352 766111 366222 -353 458112 123141 -354 409389 563389 -355 171603 749321 -356 171317 550998 -357 79737 876012 -358 783320 946031 -359 222306 21035 -360 499089 366131 -361 787327 475892 -362 290264 562466 -363 955846 324103 -364 241729 127878 -365 591921 137004 -366 434001 428503 -367 212458 54378 -368 503278 952133 -369 143785 111418 -370 183972 896361 -371 768680 885112 -372 235320 769318 -373 711948 2309 -374 539220 920419 -375 214138 460068 -376 727790 959365 -377 389498 899793 -378 981302 131808 -379 987936 711732 -380 862943 505979 -381 979356 297970 -382 379417 275242 -383 980640 286205 -384 315677 310418 -385 164064 781010 -386 406811 301887 -387 806896 39021 -388 638340 827961 -389 86393 390471 -390 581163 824038 -391 336166 529997 -392 264935 85941 -393 157647 136031 -394 49085 937216 -395 73737 217073 -396 636455 833366 -397 947354 402961 -398 489550 466793 -399 78215 196299 -400 130978 883986 -401 915915 148749 -402 339255 390099 -403 123901 197792 -404 694430 303557 -405 742146 845271 -406 82722 50720 -407 637994 645870 -408 869524 145989 -409 350616 976456 -410 785692 513847 -411 207990 119378 -412 179439 280077 -413 865094 258061 -414 962631 416796 -415 915119 440547 -416 133530 782836 -417 648324 735891 -418 741315 285445 -419 892003 619064 -420 216417 11657 -421 785414 72628 -422 151524 559890 -423 9082 517077 -424 653926 667277 -425 537866 231488 -426 504161 661418 -427 281180 690430 -428 750455 133079 -429 500424 603364 -430 648784 838455 -431 305789 75366 -432 87140 730488 -433 59856 10093 -434 899196 78103 -435 636788 352446 -436 492062 683338 -437 438590 554203 -438 9685 546572 -439 838197 489009 -440 529621 732015 -441 757636 359267 -442 768012 76663 -443 134758 58164 -444 695696 917836 -445 676034 731221 -446 386249 813899 -447 982275 863970 -448 519594 102076 -449 634038 597320 -450 550204 462510 -451 678879 164917 -452 137655 805851 -453 473852 144894 -454 893405 204516 -455 555672 692291 -456 437383 582588 -457 927329 917563 -458 452206 491889 -459 93090 223169 -460 210893 957780 -461 376055 301875 -462 527899 174277 -463 673567 327144 -464 545683 632739 -465 80350 864790 -466 653166 633681 -467 933336 837329 -468 294631 175048 -469 431937 850448 -470 624456 642869 -471 965073 472526 -472 706943 718254 -473 355166 84373 -474 286000 807998 -475 190402 192450 -476 556392 1299 -477 516970 685413 -478 809344 45198 -479 231581 300325 -480 511219 298804 -481 712957 42957 -482 580060 912803 -483 727218 964857 -484 875645 209075 -485 562397 367833 -486 205888 285092 -487 32766 18443 -488 401387 374755 -489 784905 842485 -490 364932 628368 -491 95563 245357 -492 121519 781546 -493 151833 610208 -494 53620 20533 -495 110111 329773 -496 299403 222862 -497 288051 256622 -498 436980 679981 -499 439759 688499 -500 316866 980410 -501 299467 443515 -502 443065 562034 -503 96886 311035 -504 879780 735423 -505 533580 199135 -506 991577 211048 -507 190213 181446 -508 999400 490094 -509 754905 323437 -510 475822 47236 -511 525098 187145 -512 892208 581986 -513 68366 762373 -514 842026 470732 -515 921556 90352 -516 494975 49481 -517 308905 165797 -518 636791 884515 -519 55143 940072 -520 782145 661738 -521 855303 730183 -522 544710 62875 -523 804675 112257 -524 330653 706064 -525 188256 674607 -526 837955 969026 -527 766943 226513 -528 485434 249985 -529 134610 277268 -530 925242 212371 -531 255891 939707 -532 953277 871841 -533 343832 261393 -534 666337 127338 -535 376771 887143 -536 159441 48840 -537 135180 801215 -538 209281 556072 -539 420202 665695 -540 96551 818380 -541 627763 564757 -542 545490 709185 -543 834461 555267 -544 96203 437063 -545 821965 375397 -546 218178 927805 -547 563301 895002 -548 502297 806463 -549 595003 743495 -550 853593 248603 -551 692055 664958 -552 609512 369876 -553 46844 273197 -554 423536 57758 -555 392051 930167 -556 153781 697546 -557 455303 549845 -558 994193 328085 -559 376405 58275 -560 369544 537368 -561 666390 922841 -562 273744 128167 -563 195087 494483 -564 439327 765304 -565 754370 936084 -566 132535 362443 -567 273643 166384 -568 664598 930216 -569 109453 995644 -570 714991 506376 -571 178861 37928 -572 67519 284596 -573 709006 295336 -574 654060 435133 -575 699914 7813 -576 367135 829698 -577 989125 917509 -578 116067 329611 -579 391314 443127 -580 705277 116627 -581 163743 427891 -582 342766 885675 -583 751305 115853 -584 630027 170707 -585 840838 698856 -586 674024 941272 -587 358360 361730 -588 170233 836691 -589 933716 356235 -590 12100 865476 -591 103168 996199 -592 60026 637742 -593 772340 704644 -594 19677 387967 -595 415079 548745 -596 300189 938746 -597 154805 16134 -598 832351 237589 -599 679567 705788 -600 114362 872314 -601 361620 297825 -602 423032 834438 -603 904645 370935 -604 769672 351382 -605 145169 411422 -606 309934 3347 -607 28048 156532 -608 816438 224997 -609 273085 326631 -610 53324 513715 -611 436285 924238 -612 56344 968524 -613 337236 376199 -614 518239 523921 -615 457085 799297 -616 67019 582333 -617 211066 600677 -618 555541 99821 -619 968151 903493 -620 821304 547342 -621 431558 693045 -622 334642 901364 -623 112459 375950 -624 882401 186281 -625 678898 456151 -626 719350 155645 -627 248703 315065 -628 805295 779286 -629 86759 822355 -630 278896 804823 -631 402784 866178 -632 530078 597826 -633 979864 616888 -634 668704 126683 -635 44073 440487 -636 342596 86804 -637 647733 597173 -638 794364 280639 -639 675534 741279 -640 163228 557950 -641 289440 695884 -642 245024 652261 -643 396513 200840 -644 52255 613239 -645 620813 938653 -646 431117 841467 -647 859420 380816 -648 204746 344754 -649 45312 737469 -650 106999 831819 -651 700415 141121 -652 23052 120947 -653 166711 23466 -654 910621 596442 -655 918551 604455 -656 727030 473519 -657 201541 340243 -658 373706 561316 -659 6758 149261 -660 393080 635110 -661 571575 931235 -662 19684 212253 -663 740071 201474 -664 63751 526786 -665 430462 770898 -666 370018 79092 -667 822727 558773 -668 830919 815920 -669 494342 904781 -670 278554 835196 -671 194082 902993 -672 220159 985703 -673 367077 499881 -674 821783 647167 -675 640744 3271 -676 281002 518526 -677 307007 336101 -678 461801 621323 -679 318394 464278 -680 290027 350790 -681 529123 5839 -682 317887 83354 -683 410372 824037 -684 253360 215837 -685 973165 873824 -686 739532 359591 -687 508517 389808 -688 354107 53049 -689 624227 683583 -690 750452 118748 -691 883080 599472 -692 236758 79671 -693 241774 364178 -694 761204 739372 -695 148400 6882 -696 562560 278505 -697 931615 404729 -698 95664 834491 -699 394476 830350 -700 631596 314027 -701 875654 138200 -702 896714 521996 -703 120191 681529 -704 281767 227335 -705 94326 97622 -706 860118 579021 -707 315878 283144 -708 788230 250618 -709 74223 913158 -710 987690 575881 -711 429560 423010 -712 584241 659137 -713 998170 601332 -714 462876 986520 -715 269617 672578 -716 771282 396891 -717 589256 652830 -718 258629 304059 -719 283594 953614 -720 291440 991156 -721 289955 848046 -722 751681 572518 -723 577322 139550 -724 694263 272478 -725 406558 494331 -726 371614 407955 -727 560732 959017 -728 542745 478762 -729 548943 243884 -730 263366 816131 -731 397934 328152 -732 935895 103170 -733 807666 12072 -734 827339 743359 -735 705822 648680 -736 556354 801745 -737 506599 493358 -738 204266 21605 -739 862277 625223 -740 116392 519408 -741 52388 218991 -742 723154 453485 -743 274643 443129 -744 460996 486086 -745 845163 246557 -746 476719 540235 -747 247792 642759 -748 434801 488209 -749 341447 258323 -750 368251 555716 -751 277272 69039 -752 156085 887149 -753 441943 152623 -754 684801 107246 -755 684374 99615 -756 17765 62857 -757 398721 16809 -758 339411 857699 -759 80360 293135 -760 668368 319457 -761 670625 569016 -762 375107 150105 -763 371407 487314 -764 645659 619449 -765 51415 51643 -766 336804 755030 -767 940849 16776 -768 501642 989530 -769 820269 706344 -770 114073 416943 -771 362768 167861 -772 817717 525705 -773 575932 907703 -774 165127 97686 -775 271352 947486 -776 842550 721997 -777 206907 316608 -778 218912 522242 -779 128190 139308 -780 385506 452412 -781 332354 978457 -782 993173 267430 -783 736846 849904 -784 245140 873015 -785 440877 431708 -786 692571 982674 -787 21783 720882 -788 476906 948627 -789 362108 58498 -790 931192 849165 -791 359124 506350 -792 233942 599002 -793 719288 358347 -794 761857 673418 -795 279930 651738 -796 744390 947253 -797 265466 682365 -798 724372 380094 -799 146077 96834 -800 48799 627304 -801 545594 106628 -802 166493 422186 -803 588362 336199 -804 488055 607905 -805 597319 860565 -806 760385 454772 -807 859378 733768 -808 708022 385100 -809 712990 310807 -810 543058 356752 -811 703827 148305 -812 824216 813573 -813 886113 585942 -814 816180 599596 -815 132520 140706 -816 460572 754203 -817 461179 70627 -818 88779 904351 -819 646971 500173 -820 890980 334188 -821 645356 451049 -822 130359 923178 -823 947911 596084 -824 123037 451893 -825 796251 138429 -826 563913 546481 -827 964314 908092 -828 166731 791391 -829 645449 95866 -830 333407 684010 -831 689228 987881 -832 716925 263131 -833 215208 309336 -834 324412 936200 -835 785857 111937 -836 262063 261097 -837 514555 404629 -838 792838 157345 -839 183990 916123 -840 646841 94722 -841 940493 720314 -842 266189 448509 -843 451478 472691 -844 37278 198047 -845 855418 779443 -846 579938 710771 -847 714316 779042 -848 72124 384259 -849 936493 725730 -850 130340 790565 -851 412094 206913 -852 805051 701529 -853 197936 843599 -854 280291 515805 -855 456614 694040 -856 754112 447402 -857 240447 553963 -858 104072 978456 -859 273564 937882 -860 191007 830949 -861 372842 598682 -862 805860 995292 -863 699843 55150 -864 456045 813026 -865 206693 949239 -866 877054 668185 -867 459509 952801 -868 340609 493090 -869 479866 712225 -870 344437 473021 -871 199126 99396 -872 7039 24469 -873 406600 981255 -874 904911 719024 -875 79199 16973 -876 928213 123466 -877 519037 181100 -878 913511 743908 -879 747403 852250 -880 360919 704845 -881 363732 568066 -882 171368 983592 -883 494913 654716 -884 440362 215978 -885 147363 122816 -886 73545 554540 -887 858683 174033 -888 902735 249376 -889 79645 624759 -890 81781 955935 -891 307746 603795 -892 452107 501847 -893 585506 308987 -894 496817 475917 -895 457887 685893 -896 52727 263888 -897 564861 221620 -898 399476 644585 -899 240712 833006 -900 121733 157223 -901 901610 280152 -902 637242 123264 -903 709227 324418 -904 671359 17189 -905 595594 604523 -906 247090 271486 -907 394362 675025 -908 652117 99844 -909 3204 949747 -910 584115 850327 -911 414003 607356 -912 94245 990139 -913 221205 793393 -914 917297 735419 -915 464806 578317 -916 578017 62455 -917 29164 477257 -918 708844 36259 -919 209433 777081 -920 849729 485662 -921 305782 547069 -922 891801 607559 -923 271889 445371 -924 169642 574721 -925 414 606083 -926 727287 505293 -927 368199 543416 -928 579206 839154 -929 250987 159985 -930 414419 672967 -931 114985 894277 -932 167459 109931 -933 298740 700020 -934 379685 786802 -935 403136 206746 -936 930201 428483 -937 949332 978032 -938 123039 908710 -939 239157 550828 -940 410933 382051 -941 542406 757312 -942 320999 791839 -943 463331 683739 -944 894995 919233 -945 178517 8823 -946 656574 422630 -947 373944 642982 -948 555460 877299 -949 597737 913769 -950 484520 363178 -951 206213 618813 -952 163788 618363 -953 63677 969975 -954 184293 576360 -955 80084 895467 -956 944158 331754 -957 981467 405596 -958 16392 692354 -959 520332 524476 -960 554470 421440 -961 102283 465916 -962 302282 39957 -963 532 311387 -964 264694 330968 -965 914354 153064 -966 724417 662797 -967 470744 515465 -968 437892 210651 -969 775845 915402 -970 775446 770977 -971 163406 370518 -972 364762 757076 -973 906539 190657 -974 120348 333987 -975 642450 244073 -976 612605 266768 -977 999414 331455 -978 638761 543416 -979 148069 648322 -980 180471 853025 -981 194129 268890 -982 800913 309107 -983 732060 573639 -984 966991 224391 -985 499056 825734 -986 572006 186345 -987 882026 310397 -988 808835 835515 -989 466502 970827 -990 401195 457116 -991 163318 616371 -992 150497 61329 -993 958935 455526 -994 861883 161637 -995 783405 864252 -996 243660 551453 -997 416346 949711 -998 198971 977060 -999 488492 54364 -1000 948240 71024 diff --git a/devel-examples/vignette-example/instances/1000-18.tsp b/devel-examples/vignette-example/instances/1000-18.tsp deleted file mode 100644 index 3fae42ee..00000000 --- a/devel-examples/vignette-example/instances/1000-18.tsp +++ /dev/null @@ -1,1006 +0,0 @@ -NAME : portgen-1000-18 -COMMENT : portgen N=1000, seed=18 -TYPE : TSP -DIMENSION : 1000 -EDGE_WEIGHT_TYPE : EUC_2D -NODE_COORD_SECTION -1 576044 772956 -2 993592 603857 -3 809179 172023 -4 853183 972765 -5 625315 423071 -6 636498 592998 -7 525593 97328 -8 741938 848949 -9 877545 870524 -10 693108 971295 -11 755722 250916 -12 564283 796019 -13 440998 219859 -14 401337 392184 -15 221967 798210 -16 652210 532099 -17 659178 748913 -18 572612 401198 -19 644288 896457 -20 154105 133704 -21 279462 567217 -22 893964 2297 -23 951813 860734 -24 153580 837238 -25 891476 167963 -26 159866 897432 -27 812715 596196 -28 492048 43945 -29 113778 244678 -30 31244 407981 -31 527734 956725 -32 818659 491611 -33 143609 69280 -34 699033 523296 -35 145515 881203 -36 695369 40306 -37 979048 525145 -38 811428 858290 -39 438200 968086 -40 303971 397053 -41 106081 156659 -42 360940 813986 -43 270475 695484 -44 713439 167566 -45 605304 503331 -46 702164 120992 -47 750942 272901 -48 438335 239156 -49 588169 368819 -50 190868 93522 -51 422534 185493 -52 533267 494422 -53 61882 3206 -54 536492 998729 -55 325721 796563 -56 330505 946211 -57 639373 527912 -58 705816 406742 -59 205782 545757 -60 53275 904452 -61 481111 330214 -62 332427 51992 -63 458669 509875 -64 507039 484626 -65 463263 808221 -66 321798 439471 -67 642365 507407 -68 66548 159869 -69 517285 833027 -70 108170 863733 -71 489702 167682 -72 114291 700852 -73 22220 371950 -74 788564 698950 -75 814232 928459 -76 732116 103542 -77 905556 382646 -78 771724 983062 -79 543127 25859 -80 427873 902012 -81 485921 703465 -82 890559 461987 -83 306861 162823 -84 831920 938521 -85 505692 333865 -86 618178 506831 -87 731525 124815 -88 172335 377568 -89 424658 949781 -90 280267 475606 -91 966748 481179 -92 56752 561250 -93 322300 618333 -94 548912 180377 -95 200546 903725 -96 327949 578764 -97 327334 774304 -98 245554 982870 -99 436156 989475 -100 528516 644651 -101 947291 838782 -102 418683 338626 -103 961711 250937 -104 46790 344305 -105 60345 153391 -106 434150 362749 -107 825313 524148 -108 574062 907156 -109 376130 116254 -110 216433 323991 -111 726666 842444 -112 410005 861041 -113 386573 779395 -114 88148 392898 -115 163104 921398 -116 330778 80353 -117 889435 126876 -118 41455 603998 -119 655865 532604 -120 987188 415143 -121 242202 432657 -122 963944 876554 -123 177059 485505 -124 168758 466293 -125 387730 466158 -126 894721 43257 -127 826371 607117 -128 313872 866938 -129 949346 291806 -130 297170 357713 -131 595071 514186 -132 357116 645201 -133 911188 1493 -134 398805 948758 -135 347089 88557 -136 738398 909837 -137 728524 750274 -138 429269 683408 -139 16072 802887 -140 547168 519635 -141 830048 796341 -142 95728 805390 -143 326326 816592 -144 723236 244234 -145 215687 39962 -146 205192 707107 -147 185514 898630 -148 676745 332365 -149 704133 213669 -150 447285 493650 -151 469432 365871 -152 919124 868095 -153 636110 98379 -154 947529 20980 -155 280790 497280 -156 143702 705112 -157 76119 257208 -158 152521 887964 -159 328671 458486 -160 968456 578823 -161 297359 185135 -162 501473 853438 -163 619125 372527 -164 990712 860428 -165 114164 330889 -166 735878 995092 -167 522097 49887 -168 375932 124936 -169 720222 838520 -170 652869 438362 -171 487921 264750 -172 275778 636863 -173 742602 20057 -174 205634 332076 -175 279505 304218 -176 341652 843704 -177 99505 116395 -178 757771 474340 -179 843774 869237 -180 492162 511173 -181 378157 109009 -182 368111 842427 -183 9359 878951 -184 429334 439255 -185 514605 132464 -186 682329 996594 -187 178980 664237 -188 237170 453655 -189 85630 385077 -190 95666 144785 -191 528752 121475 -192 368266 602990 -193 952732 626869 -194 626980 679669 -195 40528 496980 -196 695601 280966 -197 323914 520405 -198 756033 491326 -199 85769 611540 -200 399693 288947 -201 934426 820556 -202 236409 134720 -203 775465 220177 -204 475438 496514 -205 163663 130902 -206 847359 164104 -207 828708 995181 -208 815571 97191 -209 785094 847705 -210 86394 518032 -211 793182 817794 -212 39562 225657 -213 198037 861773 -214 760185 44260 -215 888772 16992 -216 978216 589115 -217 221413 964764 -218 297425 364648 -219 292766 373084 -220 787418 855540 -221 841775 779274 -222 593274 522495 -223 703798 877807 -224 241404 98257 -225 322368 894259 -226 731141 41508 -227 722768 382700 -228 310731 345311 -229 599143 271644 -230 837294 410817 -231 927411 102353 -232 709095 308122 -233 289127 68084 -234 570830 306213 -235 291383 937764 -236 855787 686837 -237 525337 192134 -238 786891 751673 -239 95025 656861 -240 914926 852725 -241 262629 488540 -242 206966 477954 -243 89581 875862 -244 880019 913290 -245 675637 229341 -246 793817 986553 -247 81701 849654 -248 999753 154937 -249 253937 401139 -250 735603 952124 -251 782781 584542 -252 183330 469643 -253 631630 242600 -254 834542 244813 -255 293118 434868 -256 465292 685852 -257 596007 607952 -258 616999 940857 -259 20651 859441 -260 308369 134189 -261 814146 169690 -262 570609 339258 -263 154982 271244 -264 503507 55694 -265 560504 544291 -266 917131 850212 -267 363742 480058 -268 387433 576777 -269 892532 599013 -270 860954 148724 -271 855210 20577 -272 604921 541447 -273 415194 624126 -274 415944 742443 -275 694671 728509 -276 651430 198243 -277 840634 191312 -278 34993 932569 -279 220800 703272 -280 82209 54852 -281 350067 235528 -282 383858 144394 -283 579657 444714 -284 80930 54559 -285 192758 992872 -286 524913 278208 -287 164770 579859 -288 482759 615903 -289 329055 379296 -290 304265 222413 -291 50443 800235 -292 973484 505652 -293 194223 681602 -294 466353 219347 -295 900400 942718 -296 495846 837973 -297 406254 868081 -298 623811 577002 -299 855807 25061 -300 58688 799290 -301 295071 36647 -302 438178 472257 -303 678065 851195 -304 224758 334982 -305 997089 353390 -306 466215 1452 -307 802871 139490 -308 559005 512094 -309 829274 515776 -310 520583 2655 -311 588906 55868 -312 995871 393468 -313 697801 488265 -314 840030 692512 -315 901794 631564 -316 391145 994073 -317 382207 950874 -318 756198 48991 -319 997363 833993 -320 946646 682128 -321 852328 950576 -322 698764 897745 -323 353812 439977 -324 842102 12786 -325 170280 135546 -326 736972 163294 -327 123267 427124 -328 408145 300997 -329 654440 487303 -330 716059 629074 -331 853831 390764 -332 388335 314960 -333 501061 515639 -334 302688 905125 -335 785678 119027 -336 669992 816487 -337 345496 385037 -338 265682 425611 -339 932601 568747 -340 985323 396804 -341 833824 352727 -342 976453 272719 -343 777732 380 -344 605737 67246 -345 449813 240559 -346 746303 92238 -347 48315 827619 -348 12136 35841 -349 605079 313727 -350 632062 928200 -351 507376 273354 -352 27463 773475 -353 301721 384245 -354 186841 850548 -355 649391 407764 -356 695259 587194 -357 37490 475500 -358 882771 761592 -359 342449 560716 -360 302823 465220 -361 910559 988960 -362 273062 857478 -363 611650 396637 -364 789023 572020 -365 83315 881437 -366 238770 82052 -367 919355 577558 -368 701544 246630 -369 315236 500953 -370 389948 16140 -371 657931 45021 -372 764422 984592 -373 329999 757342 -374 819176 190837 -375 215968 615499 -376 246817 33058 -377 230412 750624 -378 689430 425323 -379 353999 449905 -380 71931 55091 -381 69008 685888 -382 460599 633251 -383 749833 650238 -384 822771 52897 -385 145500 125428 -386 942416 151612 -387 344748 687324 -388 218403 877500 -389 758548 522437 -390 168047 186327 -391 42638 339118 -392 500089 28223 -393 812429 552881 -394 621453 286104 -395 827725 51305 -396 423859 262339 -397 355452 264519 -398 73724 506319 -399 700273 77098 -400 766189 452499 -401 998793 296739 -402 22789 29641 -403 572861 907698 -404 532969 202188 -405 938195 136548 -406 803870 67894 -407 622179 20625 -408 631232 806668 -409 330436 196080 -410 559526 243514 -411 949964 745672 -412 286708 693001 -413 126634 645677 -414 128822 315106 -415 114463 310704 -416 344531 556360 -417 584242 31499 -418 382456 974743 -419 716938 479463 -420 396991 5760 -421 222445 425372 -422 726577 584211 -423 101341 678186 -424 975631 662451 -425 137884 428047 -426 377496 385166 -427 962634 455485 -428 107968 442433 -429 712497 991290 -430 647184 598117 -431 190759 53505 -432 805196 932434 -433 914102 378497 -434 341316 37968 -435 919284 953045 -436 831037 667984 -437 58195 131479 -438 866017 564798 -439 783037 831223 -440 585032 684201 -441 933180 137532 -442 667922 516345 -443 119944 291025 -444 751164 651807 -445 117396 3958 -446 633426 678969 -447 560179 443946 -448 174722 554461 -449 367177 595097 -450 718193 536542 -451 895148 144408 -452 77418 453683 -453 494867 239964 -454 717244 446289 -455 335540 816943 -456 691269 60689 -457 873894 643225 -458 964691 511790 -459 493326 361250 -460 757711 359641 -461 11320 746219 -462 319775 382741 -463 57896 686629 -464 590565 604511 -465 636612 626053 -466 847554 336748 -467 495682 768089 -468 992931 872490 -469 263638 24696 -470 551654 608154 -471 797698 389913 -472 894096 757754 -473 992637 887207 -474 359194 177437 -475 386049 488093 -476 963895 762665 -477 958485 92140 -478 688988 558400 -479 648725 309329 -480 460752 622376 -481 976326 692547 -482 894634 727386 -483 19244 301355 -484 166593 116139 -485 650588 77483 -486 152595 315889 -487 975200 269618 -488 395746 248654 -489 787733 227635 -490 693753 499496 -491 37904 281236 -492 143759 14236 -493 649726 155006 -494 442114 768295 -495 748845 691576 -496 705897 147498 -497 374108 474171 -498 455558 481809 -499 414712 624477 -500 362008 743983 -501 99473 131559 -502 483684 886552 -503 450188 682658 -504 618906 944248 -505 442413 533981 -506 116285 880429 -507 560483 769175 -508 916479 828827 -509 318439 420463 -510 271828 537434 -511 886642 542115 -512 754131 906605 -513 978010 21036 -514 832204 88648 -515 819429 713087 -516 629748 843485 -517 785221 159771 -518 383210 157474 -519 720753 374583 -520 97757 820899 -521 836567 21650 -522 496467 211411 -523 804933 163781 -524 393366 467503 -525 496160 434521 -526 649604 326064 -527 805048 648920 -528 114234 255988 -529 346337 323912 -530 503341 292714 -531 961905 244322 -532 846491 621514 -533 697414 94635 -534 383961 349072 -535 964242 752697 -536 435460 850935 -537 924302 837306 -538 887829 560578 -539 737067 105211 -540 792370 722021 -541 674699 508292 -542 585306 526714 -543 751181 385425 -544 996993 163707 -545 462357 288575 -546 773512 371680 -547 410341 345059 -548 385438 985632 -549 97348 659160 -550 323581 244355 -551 426713 288155 -552 675132 774139 -553 759821 141312 -554 740757 278333 -555 897738 728808 -556 258994 182629 -557 861554 214766 -558 519202 590224 -559 833981 501431 -560 236075 711781 -561 997286 724801 -562 25490 719887 -563 325983 147305 -564 175802 150163 -565 77484 746516 -566 819820 458734 -567 207472 63562 -568 463027 492069 -569 646737 370540 -570 7512 160957 -571 551444 495561 -572 927632 750575 -573 291288 48711 -574 346190 690453 -575 19075 238133 -576 809829 947185 -577 581675 577065 -578 424534 967979 -579 80682 611570 -580 311111 267752 -581 494574 370216 -582 270820 736781 -583 177364 763432 -584 254997 110979 -585 923477 470490 -586 244034 143527 -587 482356 997942 -588 901952 50101 -589 143125 448425 -590 295352 358004 -591 66622 564231 -592 839051 809732 -593 251942 449603 -594 187913 470691 -595 886198 699594 -596 237071 535758 -597 447063 537022 -598 916922 407916 -599 13205 929689 -600 848623 241187 -601 905586 897765 -602 395101 661071 -603 171510 245597 -604 108133 771943 -605 325122 974931 -606 780065 609991 -607 725372 611517 -608 30680 958816 -609 923153 733798 -610 819858 769447 -611 750227 325307 -612 262355 682289 -613 564904 346269 -614 748425 821284 -615 826431 656354 -616 941967 371181 -617 123302 320420 -618 577938 456631 -619 838859 227534 -620 779052 293126 -621 526450 454115 -622 650832 116750 -623 949367 911764 -624 273402 764774 -625 972117 570653 -626 659490 191920 -627 103258 192268 -628 299219 534404 -629 774462 74680 -630 83132 714879 -631 406738 880599 -632 992891 31996 -633 448481 325949 -634 959158 608622 -635 662149 118916 -636 685413 158379 -637 761680 249205 -638 109956 558306 -639 222049 70086 -640 383069 30499 -641 571806 673744 -642 738151 111551 -643 249616 61367 -644 378290 91305 -645 871939 251988 -646 497472 230237 -647 565384 660135 -648 607712 368070 -649 692435 401626 -650 6793 391060 -651 689714 203316 -652 381704 941618 -653 998846 985745 -654 453768 991706 -655 942652 237851 -656 156114 683157 -657 202741 831143 -658 217406 176501 -659 315215 332755 -660 424284 80410 -661 633513 557531 -662 601828 271088 -663 429201 482097 -664 776675 820062 -665 250359 124211 -666 104537 230342 -667 127434 145217 -668 874385 888649 -669 471003 907008 -670 894145 73115 -671 746152 45534 -672 667021 791529 -673 618475 939941 -674 628408 294295 -675 230934 125615 -676 591395 872373 -677 151267 882582 -678 286523 459372 -679 75881 236486 -680 67232 110197 -681 514742 546760 -682 97561 869537 -683 491699 110579 -684 16136 411212 -685 212668 277464 -686 548092 20920 -687 101821 298668 -688 489014 761140 -689 406263 719246 -690 984565 969828 -691 406215 540188 -692 752829 140161 -693 609468 557776 -694 132781 257896 -695 653518 763805 -696 872513 59791 -697 694339 616680 -698 525022 725231 -699 943712 368353 -700 302514 857334 -701 533677 909162 -702 309729 261105 -703 719400 51206 -704 119544 11106 -705 273113 728746 -706 326591 817985 -707 582967 303427 -708 237684 454951 -709 852421 480880 -710 344515 766467 -711 166867 647783 -712 108697 355333 -713 743787 638930 -714 711190 840716 -715 579267 437807 -716 641595 395157 -717 446132 255819 -718 643237 588229 -719 957221 449402 -720 902476 154517 -721 705355 651900 -722 913381 887050 -723 596938 224730 -724 951093 339005 -725 872893 886091 -726 14041 102995 -727 789085 864706 -728 215738 138520 -729 463029 53910 -730 617867 131170 -731 93985 670141 -732 108629 118595 -733 23390 674691 -734 904603 695917 -735 706489 12954 -736 503857 513416 -737 607987 458423 -738 752426 63871 -739 858697 243991 -740 139595 605266 -741 175900 657280 -742 222848 448097 -743 343822 971453 -744 286527 327536 -745 232428 968545 -746 683626 261303 -747 742913 889522 -748 650660 191939 -749 43912 454957 -750 134623 533066 -751 366032 707101 -752 199410 267626 -753 710191 356760 -754 880147 340988 -755 520884 244285 -756 851992 135493 -757 821481 649321 -758 447544 832682 -759 927228 219107 -760 467935 831451 -761 630779 449646 -762 561293 173422 -763 646922 796756 -764 314005 340360 -765 748232 395665 -766 183724 517709 -767 723107 895310 -768 753273 40407 -769 835798 573527 -770 552 511139 -771 44225 67420 -772 859601 400977 -773 337766 233979 -774 700009 569490 -775 242600 853904 -776 877933 703551 -777 706725 738957 -778 349342 848322 -779 983994 304100 -780 514353 669783 -781 520961 306619 -782 340435 9744 -783 200059 784572 -784 275891 420504 -785 311554 213564 -786 132672 357737 -787 976507 614031 -788 953518 927227 -789 742920 822335 -790 824080 798599 -791 812761 9905 -792 826007 78448 -793 874704 877104 -794 177274 713363 -795 695250 968701 -796 764515 415294 -797 261972 786987 -798 378467 686487 -799 90913 245569 -800 447458 410539 -801 491059 877674 -802 745410 444000 -803 41142 868027 -804 877543 628277 -805 864253 472237 -806 671048 270631 -807 608850 545652 -808 905268 105667 -809 44647 553447 -810 631276 513572 -811 693658 30322 -812 973045 901015 -813 722505 254998 -814 612327 532506 -815 572888 85490 -816 49683 114643 -817 958081 351842 -818 127551 542129 -819 401055 280355 -820 173179 769036 -821 832457 623826 -822 82086 181677 -823 275042 734193 -824 442248 360956 -825 64482 123469 -826 74160 558406 -827 672680 361967 -828 360855 376416 -829 919592 393568 -830 316449 499012 -831 466971 597188 -832 455097 95217 -833 639780 47222 -834 188545 427172 -835 270609 171075 -836 663418 683690 -837 488965 507807 -838 439412 135252 -839 357641 611078 -840 540160 346088 -841 335406 218758 -842 216057 73876 -843 618519 452495 -844 659545 862864 -845 712061 80328 -846 353584 973883 -847 9745 2104 -848 105617 148766 -849 134861 574278 -850 742265 139790 -851 376551 831170 -852 820796 718393 -853 788062 855401 -854 342349 598804 -855 743448 908360 -856 716870 56727 -857 681507 236120 -858 145427 493088 -859 587442 452993 -860 989599 491014 -861 912360 614266 -862 684906 130819 -863 794524 832248 -864 862894 770571 -865 719744 584010 -866 792903 758837 -867 867629 631799 -868 629218 278678 -869 537251 979937 -870 928448 125430 -871 865052 206552 -872 873265 221046 -873 167968 739318 -874 288976 878925 -875 207580 273369 -876 285872 364290 -877 854533 158255 -878 346886 617713 -879 963541 188996 -880 89175 509383 -881 318150 362411 -882 670356 618017 -883 43308 510318 -884 183461 460460 -885 68152 406109 -886 204111 708517 -887 245412 716229 -888 205142 548070 -889 759732 526651 -890 783932 176810 -891 868707 673897 -892 681395 210360 -893 265859 430491 -894 88481 249611 -895 588491 118899 -896 95217 76790 -897 911785 522338 -898 921319 156535 -899 961139 157036 -900 15904 619897 -901 979585 762324 -902 94992 30770 -903 404662 611974 -904 682894 644172 -905 892395 916395 -906 529232 713929 -907 600505 970275 -908 414166 241359 -909 450626 148017 -910 696698 886772 -911 549178 26425 -912 444555 448254 -913 426524 441786 -914 613524 214642 -915 311567 593167 -916 865175 115559 -917 634256 867536 -918 647577 154778 -919 73391 711119 -920 796194 24499 -921 979864 940463 -922 552913 701718 -923 569721 68791 -924 632235 463530 -925 95813 479532 -926 543011 746497 -927 845469 422737 -928 754721 864025 -929 128068 227455 -930 383192 249884 -931 538583 971775 -932 847978 867895 -933 936531 588769 -934 161015 898786 -935 400554 345375 -936 609124 987096 -937 52204 217165 -938 343761 802681 -939 180956 21818 -940 693533 562499 -941 313717 386068 -942 831450 61682 -943 54584 893400 -944 267581 766360 -945 931005 58808 -946 993762 174604 -947 310564 450819 -948 415375 992767 -949 888259 335747 -950 357956 767040 -951 887835 610417 -952 769996 533314 -953 165815 156942 -954 915047 783786 -955 368153 861321 -956 596443 361708 -957 296450 324384 -958 256121 363978 -959 661210 397159 -960 452520 943763 -961 700509 825267 -962 540830 633514 -963 457539 998707 -964 217099 518889 -965 51350 186818 -966 887634 397169 -967 653665 832211 -968 966055 952008 -969 89617 507065 -970 805560 690606 -971 232189 870421 -972 313839 987241 -973 358298 168494 -974 633774 677050 -975 993279 416667 -976 775668 369369 -977 284397 171137 -978 879405 490666 -979 956752 937784 -980 567258 213806 -981 67325 407981 -982 978225 677546 -983 629131 726022 -984 47869 309209 -985 966085 87627 -986 730203 984159 -987 403880 35853 -988 168095 331140 -989 540870 369692 -990 754108 966873 -991 41955 279314 -992 951630 837544 -993 119493 479652 -994 418943 976118 -995 203080 240032 -996 904138 780408 -997 540979 717933 -998 960402 248029 -999 466541 277986 -1000 819145 27158 diff --git a/devel-examples/vignette-example/instances/1000-19.tsp b/devel-examples/vignette-example/instances/1000-19.tsp deleted file mode 100644 index 69f69933..00000000 --- a/devel-examples/vignette-example/instances/1000-19.tsp +++ /dev/null @@ -1,1006 +0,0 @@ -NAME : portgen-1000-19 -COMMENT : portgen N=1000, seed=19 -TYPE : TSP -DIMENSION : 1000 -EDGE_WEIGHT_TYPE : EUC_2D -NODE_COORD_SECTION -1 673360 97832 -2 382773 469022 -3 611497 110472 -4 498799 746846 -5 537196 247957 -6 613316 250326 -7 667454 384788 -8 142972 77096 -9 35386 626246 -10 731213 465491 -11 725787 959413 -12 526181 647568 -13 265561 22134 -14 530109 2198 -15 721260 804508 -16 145616 562974 -17 359381 713672 -18 853415 55038 -19 611566 113356 -20 176127 380533 -21 491958 850524 -22 772517 227993 -23 750837 495993 -24 166853 277422 -25 592369 953156 -26 955573 220052 -27 348565 641992 -28 570746 110386 -29 738451 669101 -30 615606 556459 -31 498906 385442 -32 570719 156663 -33 755999 762791 -34 477809 439461 -35 633950 646979 -36 910243 757964 -37 33876 778056 -38 509917 505735 -39 610847 884188 -40 76821 155175 -41 283683 861008 -42 386591 164800 -43 305601 760173 -44 992255 202718 -45 957673 90623 -46 577229 172104 -47 479405 529147 -48 470290 733993 -49 816647 994460 -50 718075 245102 -51 885145 282664 -52 200600 437193 -53 669472 94565 -54 833460 183765 -55 336391 810573 -56 118131 535733 -57 711427 524982 -58 979230 326801 -59 906037 41571 -60 686373 22005 -61 946144 483348 -62 721385 388848 -63 761834 627578 -64 557363 596682 -65 108583 415352 -66 672274 427082 -67 547797 266248 -68 37044 747950 -69 149580 861609 -70 185569 978799 -71 854135 950683 -72 516344 935668 -73 144479 93880 -74 450719 90557 -75 767313 842711 -76 176630 219964 -77 885877 302723 -78 572827 458063 -79 734866 934352 -80 400149 921522 -81 944984 971851 -82 998195 357591 -83 956437 167447 -84 19388 775759 -85 380502 885349 -86 876082 815479 -87 274258 843661 -88 845375 726179 -89 597471 418662 -90 816020 303770 -91 892711 623011 -92 196533 187061 -93 470367 700422 -94 428886 190206 -95 309810 869596 -96 728562 373820 -97 481106 300220 -98 102717 38656 -99 676425 672683 -100 90293 418300 -101 496409 32056 -102 274536 463542 -103 949999 553618 -104 23431 698815 -105 832355 872405 -106 29176 544660 -107 624541 530552 -108 192959 571164 -109 490745 697975 -110 254874 917781 -111 491022 346704 -112 685466 962202 -113 388940 844025 -114 540943 810715 -115 893661 291756 -116 702747 898655 -117 586306 943615 -118 274594 348051 -119 998469 665980 -120 994102 899203 -121 209677 730911 -122 935332 392028 -123 378574 381858 -124 688353 518904 -125 911279 258691 -126 497712 865709 -127 779021 798536 -128 715552 597753 -129 445749 330921 -130 188947 601948 -131 555149 357450 -132 704713 933152 -133 662728 298264 -134 609328 232513 -135 151978 811100 -136 882810 971841 -137 786695 996182 -138 420069 625312 -139 567682 886929 -140 246649 791186 -141 398275 210021 -142 621767 291713 -143 736607 345297 -144 193942 653154 -145 280886 976330 -146 738723 765956 -147 514002 183001 -148 16393 237835 -149 944215 939149 -150 971959 753262 -151 814175 801424 -152 272254 120092 -153 860415 287690 -154 243942 487308 -155 61929 370255 -156 403810 792595 -157 50034 212617 -158 863225 789193 -159 843447 521711 -160 916758 424892 -161 354049 670178 -162 260553 398716 -163 996925 81386 -164 699586 666603 -165 135767 132378 -166 381369 80374 -167 825000 876394 -168 387376 605680 -169 159987 409149 -170 428488 947414 -171 501849 672230 -172 736396 855993 -173 622281 68544 -174 505402 115286 -175 186075 935007 -176 538248 277612 -177 803382 839581 -178 371892 733800 -179 976423 395860 -180 732716 254735 -181 127703 834792 -182 58819 114515 -183 868405 731580 -184 56199 194040 -185 590336 794680 -186 283790 728161 -187 335636 981751 -188 886644 76436 -189 866796 420972 -190 26824 263124 -191 104962 303726 -192 933886 881032 -193 4674 546577 -194 21554 710484 -195 7988 655796 -196 549481 965946 -197 818812 633807 -198 663623 773688 -199 336594 754645 -200 969349 545844 -201 201748 84430 -202 88462 922951 -203 830045 234522 -204 343725 922350 -205 834907 825315 -206 712246 265939 -207 387871 76920 -208 705253 161757 -209 15980 425012 -210 450892 94716 -211 394985 301554 -212 224690 44492 -213 592932 199360 -214 639698 412684 -215 151705 652121 -216 732711 944446 -217 586065 201509 -218 550878 839023 -219 915855 856966 -220 175778 842916 -221 530596 596542 -222 259592 913272 -223 260810 247927 -224 741255 774320 -225 40875 464263 -226 133990 923909 -227 602939 317227 -228 813133 257302 -229 498365 886953 -230 372073 991022 -231 318666 486759 -232 746571 991990 -233 294718 115703 -234 6346 474598 -235 816109 457326 -236 420502 241659 -237 384136 986629 -238 960726 471075 -239 698614 907462 -240 231359 335629 -241 700995 752745 -242 40611 160683 -243 333455 245952 -244 197874 594074 -245 906790 435174 -246 832676 441256 -247 40856 718451 -248 422414 288937 -249 212406 272963 -250 952546 789735 -251 549312 833792 -252 542961 705245 -253 763267 381245 -254 883298 442255 -255 983772 567180 -256 59427 904290 -257 980162 936899 -258 158345 877410 -259 445902 28119 -260 569575 5780 -261 903297 733383 -262 522052 26373 -263 908014 586709 -264 698698 678890 -265 223361 579481 -266 587776 256358 -267 923690 664178 -268 276202 796705 -269 772582 103712 -270 2337 456045 -271 800050 169754 -272 24499 901010 -273 531876 99292 -274 919203 14483 -275 810437 835704 -276 590239 533515 -277 49601 373064 -278 201958 292953 -279 910101 878783 -280 429043 966562 -281 608662 779585 -282 439918 527727 -283 767130 889673 -284 879791 79152 -285 405022 59053 -286 958206 431419 -287 217681 733871 -288 415541 369781 -289 683782 148988 -290 824415 615060 -291 676607 819915 -292 249846 256799 -293 970818 808191 -294 816440 395963 -295 897047 386528 -296 916914 693430 -297 698689 943284 -298 497838 368631 -299 952072 290628 -300 485468 162094 -301 415510 770215 -302 190067 195377 -303 159096 770324 -304 283669 792802 -305 402245 393767 -306 476512 514138 -307 981735 42514 -308 49648 915232 -309 80895 496633 -310 29888 398499 -311 937600 589163 -312 593683 242928 -313 643542 187990 -314 241351 22304 -315 574774 645217 -316 86112 890979 -317 746742 430648 -318 138547 162469 -319 838179 207331 -320 207150 55586 -321 727295 319807 -322 366074 498547 -323 448928 327751 -324 99746 455761 -325 299741 309848 -326 127279 929768 -327 715853 840251 -328 75982 524530 -329 23473 759419 -330 56829 996627 -331 932144 76337 -332 585651 346659 -333 666471 156705 -334 148064 483187 -335 593586 721897 -336 815485 625133 -337 196892 720040 -338 271219 7832 -339 873309 753432 -340 166945 119012 -341 164517 481931 -342 965474 578146 -343 713072 9774 -344 305328 400083 -345 764176 981841 -346 14405 354991 -347 613745 485253 -348 240101 102161 -349 122914 646034 -350 227328 441096 -351 454441 346314 -352 288816 180729 -353 145330 645348 -354 964293 137706 -355 127179 66207 -356 219202 623389 -357 995243 74987 -358 982222 577153 -359 462592 100397 -360 106558 564309 -361 33790 502029 -362 255859 152489 -363 267455 469171 -364 336317 16162 -365 574709 625871 -366 43538 735602 -367 626252 100737 -368 899810 541127 -369 486688 890487 -370 595924 135919 -371 547182 204930 -372 293525 199866 -373 948051 512375 -374 99131 461255 -375 217798 770929 -376 765844 106751 -377 71325 601456 -378 397557 718839 -379 720061 188078 -380 280919 604203 -381 158659 73806 -382 541782 991259 -383 519025 14271 -384 329864 795376 -385 126936 469846 -386 478021 1337 -387 882599 335628 -388 798465 927039 -389 430703 654403 -390 754932 548616 -391 749109 148239 -392 735243 970506 -393 467212 969732 -394 193820 634993 -395 581712 885538 -396 211263 691311 -397 763550 126078 -398 657898 545845 -399 322331 957897 -400 401400 21011 -401 81672 444728 -402 706323 669182 -403 21820 617604 -404 371508 100819 -405 134244 427825 -406 525018 85068 -407 606365 395380 -408 392939 467348 -409 310255 415704 -410 333361 973180 -411 691939 371967 -412 393976 105924 -413 388174 33292 -414 295013 213416 -415 313807 180861 -416 555531 329884 -417 520159 327106 -418 23597 664041 -419 541873 339863 -420 577566 999863 -421 659476 778116 -422 301631 608531 -423 193599 839295 -424 297335 657625 -425 737903 624605 -426 250831 108914 -427 644089 220539 -428 465480 751787 -429 924569 379216 -430 645584 357779 -431 75730 31644 -432 523252 134380 -433 768349 746902 -434 783437 997834 -435 201780 553644 -436 170012 652630 -437 677800 708756 -438 722349 583024 -439 727877 173437 -440 640443 636387 -441 108723 915797 -442 567832 956028 -443 105130 523886 -444 806632 385778 -445 558757 276695 -446 880604 544039 -447 138082 23922 -448 829850 6846 -449 100316 592874 -450 886182 610574 -451 111417 123898 -452 17181 101516 -453 515881 335033 -454 541082 688061 -455 115408 941594 -456 945155 538790 -457 820458 368888 -458 477175 531691 -459 893562 499330 -460 304529 761502 -461 646586 190562 -462 111651 591206 -463 442226 46114 -464 635448 576284 -465 192874 387315 -466 41942 39816 -467 58028 698849 -468 691232 569932 -469 95338 198944 -470 478852 573439 -471 630324 307301 -472 81249 797254 -473 630109 690041 -474 432387 546876 -475 581695 783736 -476 371397 524031 -477 399999 498866 -478 568632 71601 -479 65869 318332 -480 410284 945948 -481 239695 342138 -482 209208 541969 -483 311270 637853 -484 457541 23204 -485 738778 787133 -486 99303 346685 -487 917634 520793 -488 390104 122554 -489 790562 612784 -490 22573 370624 -491 980244 317115 -492 165999 246926 -493 147620 699804 -494 830607 516058 -495 387579 53378 -496 112391 72134 -497 460165 691719 -498 474135 283638 -499 389666 560455 -500 407149 507555 -501 899479 819603 -502 524302 211070 -503 803491 54282 -504 358032 153073 -505 351246 868827 -506 240994 549811 -507 930753 356905 -508 833557 167560 -509 881972 517489 -510 67833 27631 -511 248186 897086 -512 616054 231223 -513 887654 279700 -514 822383 706563 -515 717302 335822 -516 764521 637488 -517 261538 153746 -518 129630 430433 -519 386362 809094 -520 413368 980059 -521 817831 313117 -522 448224 359947 -523 805191 215305 -524 456080 228942 -525 804065 194435 -526 461254 683102 -527 843153 71327 -528 743033 261990 -529 558064 370556 -530 81440 373057 -531 667920 548937 -532 739704 371186 -533 50996 927876 -534 101586 570806 -535 551713 618252 -536 711480 653030 -537 713424 873398 -538 566376 565084 -539 53933 544727 -540 488189 625663 -541 721635 451827 -542 625123 344244 -543 667902 215584 -544 897783 890351 -545 102750 201754 -546 328846 815555 -547 257380 795116 -548 268579 164801 -549 599693 574825 -550 793570 240107 -551 161372 911353 -552 740752 178402 -553 472799 9427 -554 57979 498908 -555 403424 527449 -556 364207 667713 -557 267805 879686 -558 44210 852364 -559 291557 944588 -560 102606 886194 -561 328182 526760 -562 777235 311606 -563 456879 800127 -564 912277 535022 -565 400599 556949 -566 507104 555024 -567 141302 960739 -568 261456 53922 -569 184021 745436 -570 300034 815537 -571 924026 953195 -572 787744 216555 -573 873571 802086 -574 38320 945774 -575 338236 468452 -576 252523 64671 -577 174226 236620 -578 733002 606347 -579 770050 780013 -580 916945 418876 -581 825405 312542 -582 198874 587886 -583 603423 411011 -584 879968 51250 -585 6115 242124 -586 814044 345783 -587 606351 634153 -588 633671 263511 -589 352533 540615 -590 578603 850532 -591 30076 132264 -592 618076 981722 -593 731544 194562 -594 356150 553415 -595 357316 850444 -596 173954 132771 -597 739321 57909 -598 1493 578243 -599 346843 153591 -600 582884 610060 -601 449552 497705 -602 367170 487704 -603 438376 120259 -604 446595 192504 -605 505076 538440 -606 250197 216635 -607 422697 66501 -608 244922 692634 -609 573220 140964 -610 586392 25179 -611 64168 726377 -612 468366 396055 -613 792571 316339 -614 978612 118647 -615 195777 513411 -616 816916 160029 -617 35538 40163 -618 600335 813441 -619 709566 551574 -620 736799 38909 -621 621341 215185 -622 967022 332136 -623 786275 447577 -624 664404 343266 -625 265337 685154 -626 599630 228195 -627 957813 69472 -628 793144 289522 -629 462166 327006 -630 887368 624935 -631 410692 895020 -632 455704 466167 -633 917098 35011 -634 249612 90560 -635 280226 797345 -636 28230 229954 -637 875626 901238 -638 425548 835973 -639 768563 398894 -640 602910 503049 -641 854172 651605 -642 231279 570842 -643 102719 921895 -644 704324 569371 -645 123065 565324 -646 563828 619006 -647 271347 939454 -648 10679 391387 -649 339559 65784 -650 906587 950302 -651 679013 265510 -652 740355 762288 -653 830982 948025 -654 996916 386971 -655 966753 871248 -656 585198 892795 -657 203941 322044 -658 61106 791686 -659 623672 516249 -660 455487 525710 -661 695451 183827 -662 183972 329924 -663 118331 762719 -664 489598 113337 -665 70256 477523 -666 839056 381592 -667 432140 731662 -668 917850 961377 -669 447664 909234 -670 509736 311033 -671 298222 188075 -672 113883 597354 -673 869872 380001 -674 435033 941423 -675 821123 247959 -676 901789 226221 -677 995528 429063 -678 111245 297421 -679 833369 8693 -680 844437 869604 -681 500360 87681 -682 877235 655720 -683 573026 397122 -684 778911 606587 -685 452172 681104 -686 356652 682249 -687 695126 207528 -688 623921 469230 -689 188299 754909 -690 218679 820909 -691 929349 480905 -692 268900 200651 -693 977163 751375 -694 504357 776420 -695 158635 520728 -696 182466 841077 -697 457062 828631 -698 954380 615972 -699 492948 906355 -700 973432 400641 -701 191701 680124 -702 722744 213 -703 318610 420883 -704 957321 794876 -705 451900 359870 -706 793064 56949 -707 850057 323709 -708 687138 659283 -709 630619 48603 -710 701339 957053 -711 904174 872555 -712 633154 51530 -713 489403 676528 -714 959505 694912 -715 888917 203037 -716 511909 393422 -717 303008 858808 -718 27845 872400 -719 630847 945190 -720 513513 317879 -721 120755 455753 -722 75080 201582 -723 616554 309910 -724 207922 405532 -725 339227 277851 -726 656467 798036 -727 17437 770394 -728 888732 798279 -729 377115 863935 -730 972368 446209 -731 790036 12131 -732 281363 134021 -733 239114 337310 -734 981868 648475 -735 707155 377228 -736 451360 225087 -737 709375 423488 -738 300586 106137 -739 855118 862759 -740 162797 691124 -741 299412 95569 -742 722543 442708 -743 413001 499778 -744 112058 168987 -745 619694 690534 -746 890532 982371 -747 238034 136284 -748 866518 895668 -749 746377 651592 -750 900995 510416 -751 454791 345163 -752 242734 648103 -753 978439 560897 -754 75492 574729 -755 357393 388954 -756 686220 208128 -757 244241 281833 -758 555677 807665 -759 774096 145078 -760 267502 343446 -761 590933 330275 -762 747479 196739 -763 922436 106197 -764 982352 61272 -765 445049 739688 -766 30644 280389 -767 505365 773843 -768 4904 91284 -769 851328 440710 -770 887031 605336 -771 725681 966980 -772 901484 276248 -773 99601 560256 -774 234891 41295 -775 213848 760321 -776 913315 685105 -777 206543 161307 -778 479772 174402 -779 839798 468891 -780 643199 887154 -781 709569 634782 -782 687698 752057 -783 663273 719240 -784 306643 967993 -785 182232 995420 -786 572773 732801 -787 931230 507180 -788 430130 905827 -789 123732 586172 -790 716967 748034 -791 266398 513461 -792 418072 557894 -793 30119 395861 -794 592691 753307 -795 110570 285664 -796 784640 883335 -797 258478 891610 -798 32562 992880 -799 35749 394304 -800 846117 193774 -801 436524 648719 -802 324328 465813 -803 493922 399854 -804 267032 648649 -805 131187 83910 -806 581711 86490 -807 358321 357535 -808 102514 826233 -809 376304 796087 -810 719494 670392 -811 683490 912339 -812 121875 988457 -813 558896 924054 -814 408473 465416 -815 13257 30275 -816 638794 475082 -817 454984 633057 -818 166322 179908 -819 155139 60537 -820 455379 203885 -821 19557 796603 -822 33812 440177 -823 602173 872301 -824 761460 270020 -825 332713 108508 -826 584406 570332 -827 381046 815842 -828 554979 961441 -829 193734 691271 -830 299491 314014 -831 244715 206495 -832 193269 927302 -833 64352 785108 -834 52677 918144 -835 755362 230213 -836 64773 106284 -837 463374 610985 -838 85986 113158 -839 531292 306032 -840 433478 597455 -841 730319 717202 -842 165925 699243 -843 785559 432298 -844 281812 527682 -845 568704 381214 -846 127230 236995 -847 305174 225166 -848 139111 913273 -849 333229 422826 -850 354190 489014 -851 341008 455427 -852 836541 735258 -853 378189 867204 -854 404406 681803 -855 30282 122680 -856 679628 666052 -857 122566 918276 -858 186783 7719 -859 901320 968103 -860 788190 151079 -861 451878 629850 -862 563953 266347 -863 889204 609346 -864 269742 728115 -865 232796 218781 -866 708751 849489 -867 275749 310797 -868 917826 64267 -869 594635 247649 -870 512459 777839 -871 530977 313709 -872 739491 417624 -873 929335 497380 -874 673041 38827 -875 335961 529765 -876 643530 605113 -877 190030 135409 -878 780262 491519 -879 179678 525743 -880 817432 313922 -881 272569 156757 -882 169344 252442 -883 591702 365919 -884 926560 704941 -885 988940 689403 -886 334677 862493 -887 632142 258425 -888 507549 846765 -889 439820 428544 -890 486084 397685 -891 429667 743999 -892 910683 918874 -893 946212 551994 -894 680144 23306 -895 719095 551906 -896 137706 889693 -897 258708 823056 -898 443161 668484 -899 681566 481066 -900 910075 82570 -901 57559 244497 -902 552743 938276 -903 100097 899531 -904 694430 271156 -905 189196 228268 -906 811374 156371 -907 806648 265525 -908 176215 382875 -909 898048 346287 -910 809280 923218 -911 684352 445494 -912 794865 906369 -913 631843 90180 -914 309750 693865 -915 158327 608017 -916 152334 168664 -917 239347 257816 -918 586310 273296 -919 937350 645157 -920 742658 563336 -921 653945 333856 -922 214025 795877 -923 867553 692212 -924 94827 352338 -925 191212 352981 -926 358734 987700 -927 322739 302058 -928 930236 888895 -929 5150 294926 -930 351965 826801 -931 962180 49273 -932 528497 625860 -933 574322 477517 -934 942345 10771 -935 397972 484003 -936 288047 545710 -937 155075 456299 -938 564483 696651 -939 122754 492807 -940 976133 742947 -941 85029 14823 -942 341900 331526 -943 645836 103060 -944 640166 613486 -945 683493 108793 -946 330950 926579 -947 247184 258655 -948 275289 108235 -949 178781 757726 -950 231393 170901 -951 569457 602020 -952 376205 448264 -953 267952 343910 -954 645800 991213 -955 656221 827175 -956 248729 391663 -957 611433 243172 -958 495850 35601 -959 802088 269842 -960 350570 466087 -961 298735 184619 -962 779378 227070 -963 914546 686027 -964 169504 706810 -965 188346 220573 -966 50851 131541 -967 836585 148958 -968 494218 693365 -969 403389 98728 -970 835675 610235 -971 300972 370324 -972 262916 217406 -973 810057 146331 -974 147201 20114 -975 344109 589262 -976 938730 471971 -977 569380 10820 -978 120050 437915 -979 765434 227247 -980 954046 574587 -981 940521 547072 -982 155537 45986 -983 526202 878404 -984 128747 394027 -985 433115 349519 -986 888400 781974 -987 925733 761308 -988 527356 826764 -989 615238 768558 -990 107020 476630 -991 920592 942257 -992 752764 613758 -993 280051 503779 -994 976004 790599 -995 622755 615813 -996 564617 9362 -997 665613 486155 -998 721834 518998 -999 444590 501607 -1000 690050 983292 diff --git a/devel-examples/vignette-example/instances/1000-2.tsp b/devel-examples/vignette-example/instances/1000-2.tsp deleted file mode 100644 index 47ef0761..00000000 --- a/devel-examples/vignette-example/instances/1000-2.tsp +++ /dev/null @@ -1,1006 +0,0 @@ -NAME : portgen-1000-2 -COMMENT : portgen N=1000, seed=2 -TYPE : TSP -DIMENSION : 1000 -EDGE_WEIGHT_TYPE : EUC_2D -NODE_COORD_SECTION -1 18990 574941 -2 766688 761213 -3 972089 156829 -4 523324 587468 -5 35225 224908 -6 7406 75761 -7 255812 497977 -8 325390 198600 -9 352086 778976 -10 83438 64154 -11 234683 914966 -12 173916 171234 -13 247990 383456 -14 340982 631964 -15 233285 697453 -16 757714 38104 -17 455924 312776 -18 79759 939767 -19 167841 426080 -20 801760 184450 -21 879539 34304 -22 837129 391159 -23 167432 696602 -24 941211 794299 -25 677180 604868 -26 428545 735505 -27 239118 863463 -28 232872 980886 -29 119016 453911 -30 681453 32321 -31 988988 97244 -32 785708 850775 -33 345368 973101 -34 238631 864652 -35 330544 628788 -36 257389 557786 -37 101795 478569 -38 635609 499177 -39 675848 310453 -40 938361 267104 -41 264439 887071 -42 950511 200963 -43 708465 660470 -44 252395 605149 -45 967408 106658 -46 701135 303188 -47 95535 172972 -48 927060 321753 -49 932509 358560 -50 755550 668254 -51 20753 630758 -52 855938 410076 -53 340429 541473 -54 784994 38154 -55 154998 572402 -56 728490 513867 -57 486502 574795 -58 331186 685799 -59 1708 612736 -60 923714 23614 -61 40592 880071 -62 109101 662289 -63 608034 626631 -64 701848 691718 -65 138139 94135 -66 714183 637693 -67 155454 365959 -68 538613 750571 -69 400568 375715 -70 869777 22665 -71 658761 639659 -72 681434 943793 -73 66065 821064 -74 194086 433245 -75 564937 300429 -76 619905 240790 -77 220420 661414 -78 954071 383060 -79 475303 489978 -80 871462 589857 -81 140904 409278 -82 168377 132333 -83 913640 88830 -84 832432 542708 -85 508729 510121 -86 491713 568462 -87 47798 623284 -88 403709 799801 -89 659650 447687 -90 708218 224974 -91 151327 211869 -92 820256 548281 -93 953231 304904 -94 469315 23120 -95 452318 449783 -96 918138 857860 -97 866986 359655 -98 530952 90298 -99 591860 58149 -100 540084 266263 -101 161413 746399 -102 725027 339963 -103 149102 408036 -104 420534 672138 -105 708183 649166 -106 913744 452182 -107 37659 421679 -108 671718 283044 -109 542292 808722 -110 601381 823341 -111 496969 774282 -112 2623 242465 -113 348708 745313 -114 843435 707835 -115 474182 995672 -116 379267 987512 -117 739503 59051 -118 311230 699144 -119 174209 398576 -120 876563 670186 -121 762612 660593 -122 421739 628977 -123 952813 143856 -124 855236 624520 -125 10947 785638 -126 246863 884025 -127 583967 544412 -128 886996 173901 -129 6895 665975 -130 28732 449957 -131 233827 21957 -132 795575 37996 -133 886554 253150 -134 30443 408682 -135 468866 527862 -136 427808 917771 -137 797775 815742 -138 576478 612944 -139 190315 458211 -140 355469 174806 -141 738417 177460 -142 679103 24225 -143 761845 357310 -144 191936 701506 -145 172497 58079 -146 668701 765526 -147 929710 348700 -148 242378 844840 -149 862818 605997 -150 52498 339868 -151 953540 397024 -152 269051 836140 -153 47221 69403 -154 204922 559741 -155 782566 529685 -156 981964 305388 -157 493477 970652 -158 781256 468300 -159 92247 446875 -160 795618 41713 -161 390331 424445 -162 356183 128997 -163 574322 30783 -164 648720 961634 -165 768521 507075 -166 408022 630573 -167 675645 825783 -168 192842 433028 -169 683982 708450 -170 242968 293544 -171 265062 745061 -172 905888 130784 -173 667748 244255 -174 409342 800712 -175 774377 211594 -176 196120 901183 -177 837476 545423 -178 931846 322966 -179 721379 443267 -180 643297 614192 -181 385420 496471 -182 316773 489021 -183 264622 236903 -184 399500 362693 -185 302904 537001 -186 58958 291534 -187 672497 584024 -188 845593 489148 -189 586969 810759 -190 197150 251355 -191 309404 205452 -192 318336 154328 -193 121654 911550 -194 313800 186623 -195 561160 955939 -196 33528 321289 -197 405545 705966 -198 234586 973528 -199 72564 321864 -200 285191 178599 -201 657285 598582 -202 603562 523022 -203 902190 990667 -204 582846 683147 -205 423769 20296 -206 9166 534756 -207 882106 687358 -208 580664 64130 -209 90926 610806 -210 254435 291094 -211 164339 77636 -212 77502 124304 -213 879716 460375 -214 687972 149475 -215 681833 854925 -216 906301 903822 -217 386990 176854 -218 242189 774648 -219 323346 630978 -220 573664 57523 -221 820623 702993 -222 932188 270066 -223 791599 955891 -224 243787 281240 -225 826250 774211 -226 285556 923088 -227 640030 430265 -228 272298 753462 -229 211591 426707 -230 280832 127542 -231 667321 951868 -232 109482 366245 -233 199672 306172 -234 602568 612040 -235 895758 624772 -236 820343 809685 -237 784556 480223 -238 5538 241250 -239 437605 647236 -240 852006 126253 -241 248783 261264 -242 868642 554291 -243 187604 954432 -244 794340 20745 -245 977182 936016 -246 172079 711306 -247 735219 948892 -248 237180 10938 -249 918437 451964 -250 264528 550349 -251 518286 596550 -252 429234 699996 -253 525428 24291 -254 54446 85738 -255 242661 317866 -256 959122 190846 -257 449525 344815 -258 955462 956015 -259 216649 160590 -260 129065 188733 -261 387735 150603 -262 347511 345409 -263 106486 223792 -264 380450 84559 -265 954795 981246 -266 186804 351866 -267 404575 534139 -268 167131 57936 -269 811739 523826 -270 598829 231589 -271 737782 633749 -272 891680 788448 -273 548281 21475 -274 363794 389809 -275 842406 13387 -276 630487 833878 -277 497168 283281 -278 363545 166419 -279 191975 895095 -280 532865 467491 -281 212551 530619 -282 486909 11071 -283 580083 325372 -284 299166 661076 -285 796534 933986 -286 592220 826839 -287 318184 115677 -288 558246 553857 -289 653435 64229 -290 981864 940066 -291 31817 485355 -292 551694 487303 -293 768694 656184 -294 864957 393503 -295 954055 841759 -296 758769 150663 -297 727292 664842 -298 639368 910942 -299 315565 776002 -300 230202 994424 -301 368040 299564 -302 407945 902339 -303 981570 145131 -304 282183 9864 -305 514587 707360 -306 301462 798472 -307 941039 691105 -308 708721 61888 -309 803327 822067 -310 371703 669141 -311 9806 523164 -312 430874 802110 -313 565945 292655 -314 418894 415844 -315 134107 413114 -316 271673 643570 -317 549642 274503 -318 638604 233344 -319 544316 860588 -320 778582 706805 -321 852857 42890 -322 21800 284914 -323 831952 235605 -324 719788 925182 -325 98896 346712 -326 492048 899720 -327 641894 817087 -328 722751 724470 -329 749921 133441 -330 263734 748225 -331 600814 421595 -332 231282 807781 -333 854502 258571 -334 776672 656124 -335 859153 473116 -336 342100 878145 -337 723170 24990 -338 177093 110086 -339 881269 613786 -340 79359 841475 -341 542733 285452 -342 152109 385881 -343 812299 850078 -344 412287 741861 -345 420001 380032 -346 456672 888191 -347 1434 305466 -348 364705 974712 -349 319719 996809 -350 107821 721866 -351 354335 106001 -352 845822 257421 -353 803979 206595 -354 747610 256012 -355 4787 872672 -356 312183 8060 -357 713440 883702 -358 291553 712623 -359 420160 925816 -360 443075 879789 -361 938852 779862 -362 548303 137287 -363 118780 236098 -364 32322 465749 -365 221011 970497 -366 362475 625257 -367 608999 206686 -368 529291 534673 -369 572011 268407 -370 94328 99676 -371 429917 486471 -372 298785 540211 -373 441180 676809 -374 339888 864147 -375 186685 128606 -376 942390 853970 -377 775798 137324 -378 359390 729078 -379 497002 639136 -380 728129 269305 -381 634583 479203 -382 161684 905111 -383 442755 825712 -384 709274 173229 -385 442521 614744 -386 372734 556013 -387 739130 314468 -388 937399 84881 -389 4064 410979 -390 777896 389702 -391 739096 393185 -392 737619 951705 -393 335913 883272 -394 463572 703888 -395 763930 703579 -396 825398 398782 -397 825885 479584 -398 726942 873904 -399 747340 984317 -400 602812 356298 -401 672745 928909 -402 86250 796983 -403 389510 549205 -404 116351 824093 -405 801410 476118 -406 265505 793114 -407 875206 24550 -408 443907 235801 -409 653317 682100 -410 178168 568850 -411 78371 724957 -412 570417 86223 -413 941998 443825 -414 469762 942146 -415 924957 388193 -416 968530 179970 -417 609568 301777 -418 124196 945981 -419 517979 713069 -420 507797 100112 -421 229954 781472 -422 525719 195079 -423 625208 100440 -424 828365 739662 -425 537585 283117 -426 404142 805194 -427 59359 214618 -428 387768 492774 -429 319340 784473 -430 672787 443529 -431 31226 403282 -432 316295 701298 -433 246163 484033 -434 267395 680126 -435 399342 343466 -436 407436 913654 -437 144514 895051 -438 164708 273177 -439 665598 355799 -440 698454 449223 -441 124484 685289 -442 269359 481428 -443 356967 565248 -444 863675 908270 -445 55613 640163 -446 678586 837852 -447 313727 164331 -448 692676 316300 -449 636958 630667 -450 30370 352030 -451 434842 472566 -452 41207 88362 -453 158632 718853 -454 535834 577930 -455 857650 822520 -456 629099 411070 -457 728859 32624 -458 764943 193373 -459 89555 151964 -460 8622 929863 -461 847074 636728 -462 649755 47311 -463 908624 934869 -464 872446 56152 -465 736418 445855 -466 737342 87667 -467 498148 875934 -468 820124 713413 -469 956429 236735 -470 716485 163594 -471 475693 711711 -472 899648 125750 -473 793088 41858 -474 188097 266415 -475 255706 757806 -476 443853 580806 -477 894248 584515 -478 614688 347174 -479 974417 165273 -480 268238 445218 -481 762423 299099 -482 861445 694055 -483 346826 917388 -484 511422 603109 -485 239535 723085 -486 5275 823139 -487 896257 250815 -488 486009 266268 -489 742479 65240 -490 432623 561449 -491 960452 707173 -492 787914 291200 -493 683432 438242 -494 226222 804093 -495 529107 902735 -496 201991 353320 -497 997199 993399 -498 158318 652553 -499 815453 648832 -500 639740 526819 -501 299378 122857 -502 833792 694256 -503 797353 736680 -504 792891 603048 -505 901083 176445 -506 120952 170323 -507 636165 365502 -508 243227 409103 -509 301899 868045 -510 535737 694273 -511 101934 862590 -512 963368 712715 -513 423706 882418 -514 989346 202000 -515 453462 749328 -516 473376 139431 -517 164157 256178 -518 440497 790128 -519 71008 422411 -520 47972 274328 -521 136343 358176 -522 268356 834834 -523 800801 339401 -524 389952 284484 -525 569693 275899 -526 663206 613453 -527 195369 890411 -528 53443 159947 -529 958700 577614 -530 253759 7224 -531 665672 370480 -532 555075 626754 -533 40101 762775 -534 901967 801330 -535 564701 903825 -536 19151 17415 -537 298352 259837 -538 31066 488480 -539 667220 72956 -540 659272 263759 -541 923718 411732 -542 948240 446237 -543 83656 102896 -544 584355 537402 -545 216076 677721 -546 888161 269677 -547 857710 144147 -548 255177 118927 -549 59824 8519 -550 803768 312320 -551 672180 316995 -552 625212 305933 -553 352181 251474 -554 665212 749132 -555 806755 950546 -556 575591 421297 -557 361537 576037 -558 119063 395995 -559 512769 410928 -560 371576 921173 -561 702951 893448 -562 997562 252380 -563 231644 702155 -564 392203 992418 -565 907655 779592 -566 823268 918088 -567 266200 708725 -568 688167 502421 -569 50194 372203 -570 327173 687661 -571 590126 173427 -572 165826 294902 -573 974769 994712 -574 272114 605329 -575 912502 553022 -576 726724 67405 -577 100863 24176 -578 489052 754092 -579 50794 916486 -580 617766 849759 -581 201279 293009 -582 421958 119094 -583 360419 402164 -584 255470 66635 -585 601267 124351 -586 123880 907439 -587 498426 818554 -588 194449 635546 -589 792584 973386 -590 763328 477552 -591 651360 475716 -592 374652 57895 -593 578312 530258 -594 496130 147106 -595 348305 286003 -596 246950 983559 -597 770935 202822 -598 563780 682687 -599 675001 347272 -600 100453 339223 -601 202127 298728 -602 842001 434949 -603 901661 251007 -604 692753 42967 -605 445863 958793 -606 257961 903688 -607 568180 331762 -608 602809 217720 -609 522073 219135 -610 555314 677732 -611 727163 908198 -612 966182 262043 -613 922223 825152 -614 65438 63476 -615 916892 943441 -616 942792 749617 -617 527522 804534 -618 219590 747672 -619 907536 42889 -620 455086 360591 -621 8185 276995 -622 591792 670573 -623 558840 338751 -624 17376 508892 -625 280599 738627 -626 617248 611524 -627 430379 157011 -628 396431 452510 -629 771205 37466 -630 215358 153988 -631 343471 649864 -632 587881 85271 -633 950608 980966 -634 311895 897607 -635 772922 264057 -636 200343 13181 -637 938535 816686 -638 60484 115638 -639 477819 809170 -640 865612 469712 -641 53947 27971 -642 848117 762903 -643 599969 292928 -644 161735 442250 -645 853926 238624 -646 435777 9928 -647 269966 191028 -648 160247 995004 -649 338459 775105 -650 610088 443201 -651 860932 208205 -652 643279 810886 -653 684680 589276 -654 763406 667475 -655 557041 103502 -656 290775 328954 -657 183539 976734 -658 718211 333543 -659 379897 396853 -660 925023 955604 -661 642506 536789 -662 287518 329721 -663 403125 992138 -664 369901 127648 -665 132006 471207 -666 352231 810343 -667 252129 762109 -668 178937 724992 -669 844431 871383 -670 44691 266426 -671 913030 764882 -672 517226 898322 -673 596118 898987 -674 722409 940245 -675 787902 168109 -676 625102 210811 -677 643099 138881 -678 90970 50588 -679 956076 881169 -680 631949 959687 -681 744844 892023 -682 622783 290615 -683 190471 525892 -684 811727 285217 -685 380616 819224 -686 611133 439652 -687 608950 756914 -688 330501 431695 -689 893690 148636 -690 238750 352537 -691 36061 488731 -692 495699 172319 -693 726362 460208 -694 187560 961513 -695 571637 653044 -696 913264 559214 -697 490767 225467 -698 655292 473378 -699 155931 760312 -700 567821 164423 -701 5296 573772 -702 701494 435365 -703 132048 136370 -704 715111 470779 -705 412519 630761 -706 863027 994562 -707 309531 978904 -708 46422 185630 -709 401256 397316 -710 635322 717093 -711 369961 51415 -712 717396 216193 -713 813927 37360 -714 738157 173585 -715 624865 194130 -716 716583 422910 -717 736117 607989 -718 489509 41498 -719 179199 516795 -720 125896 540731 -721 58952 790244 -722 326191 854544 -723 283083 861849 -724 841818 274574 -725 411539 617932 -726 735221 982345 -727 135446 373690 -728 447839 582385 -729 837655 93505 -730 945855 90549 -731 957170 198316 -732 344883 871787 -733 571809 72783 -734 668371 454987 -735 695820 184573 -736 343812 126681 -737 985776 17390 -738 981872 387615 -739 915968 343705 -740 768353 231542 -741 199705 644652 -742 227730 534316 -743 236960 518266 -744 78027 864329 -745 36179 416725 -746 373127 724212 -747 820974 941323 -748 196919 932270 -749 804467 308801 -750 872672 895467 -751 945881 498112 -752 506220 179997 -753 418226 90569 -754 754615 601130 -755 136730 929572 -756 504357 973325 -757 57326 529129 -758 717422 232958 -759 377342 403559 -760 674868 639539 -761 268316 359570 -762 582315 800353 -763 238691 845699 -764 620461 805779 -765 599163 891303 -766 633000 314837 -767 206975 838780 -768 727184 226379 -769 587326 698600 -770 816894 4002 -771 140924 674468 -772 189461 396640 -773 148409 13556 -774 141897 20620 -775 702631 351220 -776 311809 998687 -777 709637 981369 -778 262467 631043 -779 291136 667440 -780 452812 191847 -781 503243 56015 -782 784236 132728 -783 788648 829889 -784 783864 660685 -785 380720 703865 -786 91060 356721 -787 700927 323647 -788 327730 269628 -789 649932 600946 -790 537886 607648 -791 554562 953021 -792 352967 407316 -793 388060 576985 -794 530601 74247 -795 50132 897294 -796 442514 926640 -797 317880 113028 -798 912941 784203 -799 973540 865813 -800 68909 878781 -801 363624 540951 -802 482734 94983 -803 796657 358787 -804 645720 302321 -805 593309 685481 -806 100442 216888 -807 617307 555517 -808 749332 576602 -809 738134 671207 -810 219786 4444 -811 856349 918051 -812 591776 501938 -813 340241 550109 -814 873987 605943 -815 526990 968942 -816 623907 347610 -817 7636 852404 -818 507206 337673 -819 335713 797449 -820 657983 811457 -821 838850 859393 -822 854461 45687 -823 40945 524463 -824 334864 815941 -825 772787 362831 -826 910216 367596 -827 338823 99967 -828 254873 16013 -829 533314 630329 -830 587777 458983 -831 23074 848271 -832 644338 781851 -833 846631 241048 -834 362427 571620 -835 514572 224869 -836 241738 922193 -837 898420 856954 -838 94228 488752 -839 579227 491808 -840 247065 324228 -841 16795 243657 -842 18165 68006 -843 945867 775636 -844 703272 225785 -845 5773 266157 -846 975245 764093 -847 282877 433114 -848 569718 916657 -849 960973 997506 -850 951458 552193 -851 945235 843055 -852 568876 448559 -853 346036 666558 -854 349431 270816 -855 154099 479237 -856 312740 307529 -857 624555 321619 -858 483738 258981 -859 565394 211224 -860 212133 929974 -861 280074 364921 -862 620161 962378 -863 279633 398683 -864 353317 449860 -865 510918 427670 -866 139321 308410 -867 337708 767827 -868 11487 709265 -869 619102 696545 -870 584268 686886 -871 210241 492048 -872 13651 75799 -873 986083 610324 -874 143931 320498 -875 153481 171034 -876 563339 511112 -877 486588 523788 -878 412871 636825 -879 505346 801048 -880 437071 636771 -881 47456 652885 -882 686548 467213 -883 268995 820692 -884 293878 548756 -885 335536 873413 -886 115049 244896 -887 57743 41099 -888 366634 768961 -889 878333 96373 -890 549506 642807 -891 893336 552268 -892 12789 874147 -893 380214 486436 -894 621862 870495 -895 498831 190795 -896 415387 70345 -897 361009 710854 -898 571836 965345 -899 434305 972552 -900 709164 217121 -901 731990 47558 -902 770991 510673 -903 277697 11070 -904 498323 612441 -905 143574 926435 -906 14963 634851 -907 302216 246276 -908 221383 977110 -909 291875 975694 -910 895376 303650 -911 386386 321325 -912 839591 118414 -913 141423 67491 -914 473904 547070 -915 763401 355563 -916 270638 265892 -917 952799 623071 -918 627844 258485 -919 250052 766512 -920 652763 403103 -921 194560 646167 -922 975118 195181 -923 804408 94061 -924 230754 242595 -925 569430 504345 -926 491440 887235 -927 209151 353601 -928 946483 466098 -929 94759 147920 -930 882828 19212 -931 761018 731810 -932 959677 740470 -933 731875 368795 -934 659732 107035 -935 441868 127321 -936 746355 49280 -937 406263 391031 -938 812209 499151 -939 112174 485990 -940 171930 675325 -941 972732 325984 -942 664241 744189 -943 594545 538827 -944 306215 212328 -945 891195 259049 -946 598753 143016 -947 324644 525442 -948 656748 145280 -949 239904 584087 -950 382971 305256 -951 981887 744764 -952 70664 894105 -953 531613 165456 -954 222993 464961 -955 759055 407656 -956 159883 882430 -957 256724 623779 -958 420458 618001 -959 407165 434234 -960 83722 586595 -961 128891 75644 -962 724063 136611 -963 145434 1591 -964 978615 512157 -965 859418 646753 -966 276158 647212 -967 726934 764274 -968 515441 90301 -969 69260 40462 -970 323730 976543 -971 131661 871980 -972 128606 304600 -973 130157 523108 -974 418953 188033 -975 380008 655156 -976 166664 727746 -977 724669 736218 -978 29098 334674 -979 17829 306390 -980 378663 441311 -981 96195 182530 -982 141231 782511 -983 275994 287902 -984 753823 952123 -985 493621 897349 -986 199048 219132 -987 54226 428565 -988 419931 401145 -989 350975 987845 -990 107512 810759 -991 983761 672225 -992 133493 418106 -993 550557 93627 -994 505981 944422 -995 488280 227539 -996 336477 117137 -997 546841 426380 -998 777495 912529 -999 817753 700040 -1000 884669 729012 diff --git a/devel-examples/vignette-example/instances/1000-3.tsp b/devel-examples/vignette-example/instances/1000-3.tsp deleted file mode 100644 index 1cf4912c..00000000 --- a/devel-examples/vignette-example/instances/1000-3.tsp +++ /dev/null @@ -1,1006 +0,0 @@ -NAME : portgen-1000-3 -COMMENT : portgen N=1000, seed=3 -TYPE : TSP -DIMENSION : 1000 -EDGE_WEIGHT_TYPE : EUC_2D -NODE_COORD_SECTION -1 116306 899817 -2 155869 626378 -3 774407 95279 -4 168940 361549 -5 947105 49793 -6 984224 733089 -7 397673 785436 -8 726424 426747 -9 509927 534698 -10 121542 558350 -11 204748 623463 -12 135814 22783 -13 72553 185731 -14 469754 241978 -15 732577 703750 -16 251120 68979 -17 156127 277535 -18 360563 593607 -19 135119 642978 -20 823781 431278 -21 92035 317611 -22 715682 616855 -23 966456 331860 -24 954484 234483 -25 378074 390062 -26 224252 58126 -27 774968 909259 -28 311571 47327 -29 743689 878334 -30 265815 180800 -31 960160 525961 -32 537767 515827 -33 957758 666612 -34 17407 780817 -35 818980 394564 -36 472263 275443 -37 156623 731480 -38 334098 146622 -39 848495 226555 -40 711212 25226 -41 442041 591420 -42 976163 551777 -43 743590 725158 -44 531211 640300 -45 319777 693950 -46 576200 354301 -47 823998 429217 -48 959015 816591 -49 160988 984201 -50 282757 819833 -51 483365 727929 -52 523271 352848 -53 948020 632831 -54 81963 223190 -55 165668 586412 -56 516116 103389 -57 558557 571865 -58 604600 605858 -59 701963 108550 -60 556811 141167 -61 505624 33205 -62 498059 999146 -63 911199 744333 -64 752172 803775 -65 783460 701266 -66 64659 625304 -67 60886 124799 -68 509109 338652 -69 32863 404297 -70 947176 137732 -71 23195 422661 -72 83488 178610 -73 188325 542994 -74 856241 824852 -75 518018 214681 -76 64418 357212 -77 200741 581491 -78 755174 858060 -79 667042 398471 -80 843738 609367 -81 599968 677665 -82 276014 27936 -83 563216 93455 -84 19900 379947 -85 383539 61605 -86 749617 877110 -87 590531 342130 -88 76748 148412 -89 832463 916568 -90 243971 53138 -91 77291 353701 -92 960037 174092 -93 101298 386993 -94 349290 32949 -95 561583 415654 -96 318752 652916 -97 20758 885571 -98 388114 146084 -99 832129 741357 -100 101861 39913 -101 710530 939673 -102 580880 464879 -103 137390 710717 -104 397175 26649 -105 480193 368180 -106 508769 634093 -107 836888 428083 -108 290615 947051 -109 656907 390443 -110 639821 417132 -111 261325 278542 -112 278085 343626 -113 351074 809943 -114 296230 125652 -115 204740 366030 -116 751237 805814 -117 436374 875791 -118 544369 443197 -119 516812 531953 -120 883477 154246 -121 730086 958847 -122 393127 144450 -123 154328 40209 -124 374831 677131 -125 534496 578171 -126 849854 706477 -127 536617 735831 -128 288675 904716 -129 503299 705089 -130 920510 694192 -131 193904 865221 -132 143171 325947 -133 638093 549921 -134 240965 692437 -135 273755 250406 -136 572220 979775 -137 855946 61650 -138 567277 554848 -139 741925 542253 -140 54950 446358 -141 306644 591140 -142 205142 510547 -143 172125 886015 -144 662642 110427 -145 237697 994447 -146 202231 824375 -147 258198 633070 -148 582026 750311 -149 102900 331476 -150 577173 599480 -151 298283 832577 -152 622180 88137 -153 271526 258714 -154 501335 26069 -155 563705 402660 -156 242073 392871 -157 467392 926062 -158 491960 369529 -159 607023 510100 -160 743920 887782 -161 447021 909489 -162 115264 674274 -163 952122 739642 -164 357594 767809 -165 790123 308563 -166 53513 715856 -167 978548 652290 -168 204285 913772 -169 123747 279079 -170 18587 802595 -171 278991 152541 -172 366506 349914 -173 547426 292742 -174 709110 583923 -175 680948 842383 -176 392716 335091 -177 541352 268609 -178 545966 582427 -179 854029 969890 -180 883851 357753 -181 134966 222255 -182 7482 761109 -183 123668 89531 -184 26365 117478 -185 378636 199217 -186 660418 23100 -187 829152 901537 -188 495066 111929 -189 368136 846654 -190 128308 369695 -191 885613 387704 -192 883957 432370 -193 173597 831257 -194 708374 217438 -195 528621 114754 -196 887407 6269 -197 900443 819369 -198 142177 255890 -199 323389 464969 -200 854847 435496 -201 924606 862456 -202 455615 311253 -203 956769 5011 -204 451133 108982 -205 95012 714709 -206 874053 636590 -207 441269 769097 -208 470345 128697 -209 321811 188112 -210 618932 867777 -211 766141 561396 -212 262630 943139 -213 274611 797962 -214 567485 517899 -215 944767 490054 -216 660795 259153 -217 751642 413598 -218 495641 249023 -219 946435 114859 -220 962024 44899 -221 509445 520261 -222 598505 660843 -223 348612 326011 -224 743638 957304 -225 544757 344214 -226 688405 805490 -227 520201 364792 -228 774700 665453 -229 110813 42016 -230 815611 707746 -231 58576 336274 -232 146958 50112 -233 205263 353792 -234 38084 780425 -235 420485 144334 -236 385058 364507 -237 643354 274718 -238 179372 960651 -239 41194 897838 -240 168438 609158 -241 687148 525469 -242 702287 237020 -243 431477 324521 -244 112195 701529 -245 208335 141849 -246 210938 166009 -247 694374 817689 -248 659840 144938 -249 876906 323787 -250 481470 387960 -251 284817 845800 -252 788865 935599 -253 657065 162935 -254 103202 283181 -255 933314 450179 -256 553257 409284 -257 833680 673762 -258 496808 892567 -259 641899 329268 -260 390271 60324 -261 476885 714296 -262 298955 32524 -263 859517 539258 -264 575641 707755 -265 617652 16436 -266 857449 758013 -267 964523 718259 -268 55900 277864 -269 691789 28525 -270 740212 538910 -271 682621 782926 -272 311257 148010 -273 664963 496641 -274 867053 661849 -275 958172 120582 -276 569296 169150 -277 706135 465033 -278 530510 526803 -279 881277 70606 -280 879699 379201 -281 471146 74676 -282 542968 394404 -283 767557 770331 -284 98026 685669 -285 8798 167 -286 25513 980050 -287 371095 269689 -288 491028 307735 -289 8161 833921 -290 502014 332713 -291 657981 505035 -292 828056 238450 -293 545290 782773 -294 215044 570118 -295 950702 285569 -296 179837 6119 -297 19727 740044 -298 513396 702571 -299 411830 41568 -300 656982 357228 -301 488480 33132 -302 159835 625459 -303 462601 64260 -304 341094 467684 -305 919743 747737 -306 311759 311158 -307 119903 594129 -308 199364 465026 -309 54948 802923 -310 881008 64985 -311 358500 56458 -312 28686 651570 -313 511686 992381 -314 820215 745636 -315 807087 426767 -316 966640 540476 -317 914177 754276 -318 20954 346822 -319 385132 233926 -320 39086 80263 -321 727824 412120 -322 689110 885716 -323 927068 123378 -324 977433 368157 -325 228358 521015 -326 882355 666194 -327 234480 230214 -328 390588 948003 -329 118954 405558 -330 604505 115778 -331 679128 107168 -332 428598 839479 -333 19912 899638 -334 622048 234187 -335 667061 75985 -336 487593 686791 -337 574565 359993 -338 182629 692306 -339 821978 798471 -340 260981 563683 -341 873427 414657 -342 141131 691309 -343 747639 859471 -344 111878 74698 -345 734364 121315 -346 724774 150944 -347 566864 963100 -348 592670 41033 -349 837554 329117 -350 703086 234762 -351 301400 178961 -352 107175 664674 -353 647588 467698 -354 525062 543171 -355 482575 531116 -356 836125 44256 -357 671193 483189 -358 391004 528183 -359 540303 465497 -360 246809 978879 -361 62083 292931 -362 531100 432299 -363 774584 308632 -364 579615 909891 -365 712405 714931 -366 167243 278806 -367 315896 729865 -368 727557 829170 -369 743463 657941 -370 300304 219455 -371 319168 646380 -372 827888 755485 -373 59231 431842 -374 619844 134565 -375 188516 284037 -376 461417 927663 -377 616711 988155 -378 67518 22593 -379 863064 377309 -380 937117 818417 -381 724235 867120 -382 242866 263119 -383 211947 189745 -384 216368 915708 -385 423958 959161 -386 908339 405738 -387 276981 962772 -388 517462 134420 -389 676219 542945 -390 364780 751991 -391 445567 202306 -392 972773 893987 -393 990695 300122 -394 35940 52776 -395 517917 537812 -396 612802 827754 -397 233983 341142 -398 311116 913430 -399 369399 865115 -400 238023 924811 -401 755623 76898 -402 769784 436524 -403 838470 259111 -404 954889 722724 -405 997459 767395 -406 986653 810288 -407 859391 399304 -408 205615 896480 -409 633137 901723 -410 952003 298517 -411 820346 351252 -412 677685 499146 -413 203538 831440 -414 635953 840456 -415 124301 258350 -416 179530 953495 -417 545485 597385 -418 765337 635279 -419 342914 573468 -420 688372 94215 -421 666985 134216 -422 100772 219400 -423 717466 261549 -424 150069 734837 -425 137604 479675 -426 277476 528942 -427 740813 979672 -428 745280 802128 -429 531412 172399 -430 671187 203191 -431 916196 381421 -432 34351 903244 -433 100410 852437 -434 709515 639991 -435 681838 944065 -436 746411 898300 -437 764119 472328 -438 21040 291403 -439 610438 698013 -440 753865 401409 -441 300027 463554 -442 169269 921110 -443 342153 798109 -444 919143 642241 -445 496975 912900 -446 925763 702922 -447 891630 744307 -448 347804 768685 -449 370096 628444 -450 198359 426062 -451 651111 452056 -452 980971 736194 -453 179647 813922 -454 359672 819703 -455 637518 947171 -456 882985 889171 -457 675424 758286 -458 277427 213274 -459 489790 290044 -460 555440 331724 -461 482340 81071 -462 441631 255775 -463 292953 294354 -464 917329 27924 -465 292681 207118 -466 931730 790735 -467 60494 806694 -468 518424 410856 -469 788130 410982 -470 643683 128879 -471 308318 629098 -472 86801 165250 -473 430560 844692 -474 261290 635854 -475 451353 53449 -476 851356 342172 -477 335763 991241 -478 494331 860376 -479 391561 174276 -480 217770 768791 -481 25792 948690 -482 176019 508638 -483 638853 253886 -484 802370 510173 -485 327726 432735 -486 951983 853936 -487 838691 501991 -488 480368 140167 -489 745307 450389 -490 761443 432577 -491 902793 743052 -492 810154 523889 -493 181325 983040 -494 614715 551855 -495 167841 264538 -496 608485 277956 -497 83256 210948 -498 176896 454382 -499 790407 584809 -500 684882 290392 -501 99384 810901 -502 874410 18775 -503 150655 108304 -504 532017 811873 -505 809916 511291 -506 245660 839705 -507 6435 953231 -508 160305 747836 -509 865433 965071 -510 331742 184470 -511 463478 217560 -512 825290 37333 -513 333350 141082 -514 979525 819915 -515 351335 372063 -516 608149 933434 -517 640473 250152 -518 186917 63087 -519 736617 856922 -520 363584 433489 -521 117607 649643 -522 220113 983370 -523 801059 390925 -524 452665 45923 -525 877597 35813 -526 474856 970491 -527 233474 312818 -528 682242 165949 -529 170427 624257 -530 831858 87567 -531 371687 675095 -532 448288 376426 -533 393683 596017 -534 619592 23064 -535 152172 769380 -536 295170 819510 -537 87474 295928 -538 709613 492986 -539 984086 512472 -540 355090 167400 -541 970654 355267 -542 988057 263767 -543 376 933054 -544 485145 264046 -545 856468 590900 -546 443495 713552 -547 704749 594204 -548 138319 298096 -549 562169 924184 -550 273756 308072 -551 406839 940193 -552 690832 710196 -553 65158 119589 -554 982434 969707 -555 312441 749188 -556 680804 906380 -557 767789 240958 -558 644072 658134 -559 970345 854084 -560 238107 95586 -561 33847 695408 -562 749308 844099 -563 362541 354977 -564 128678 377277 -565 230770 590024 -566 510552 14378 -567 200030 605902 -568 486596 64274 -569 587478 747099 -570 619695 342242 -571 962709 631060 -572 25939 760881 -573 557052 748087 -574 964244 860650 -575 231663 783341 -576 169418 184891 -577 693414 683731 -578 797520 392460 -579 740162 84929 -580 223600 884 -581 532110 235334 -582 350011 970199 -583 786478 49743 -584 880441 6907 -585 683905 895984 -586 693890 109695 -587 622421 454765 -588 926168 848955 -589 1993 65576 -590 46579 970080 -591 614814 43749 -592 153677 229885 -593 57914 275217 -594 664366 229830 -595 819424 436852 -596 183833 580571 -597 63193 723710 -598 648351 853013 -599 8638 571173 -600 834713 708096 -601 746093 898667 -602 814070 261582 -603 168527 125669 -604 31214 463528 -605 625817 522302 -606 728093 510332 -607 265505 786747 -608 817051 951538 -609 172140 626301 -610 321848 933464 -611 41104 309268 -612 172193 975809 -613 149890 795222 -614 295625 360839 -615 286238 800498 -616 817741 538464 -617 439758 524277 -618 241987 104482 -619 778243 366929 -620 412834 106375 -621 103076 38065 -622 907982 885959 -623 395748 874564 -624 408378 87384 -625 573819 853128 -626 557388 647799 -627 284934 34215 -628 890355 207628 -629 458909 289792 -630 19594 64044 -631 347425 664285 -632 50694 519442 -633 419225 690027 -634 602349 379546 -635 390999 942486 -636 543160 84756 -637 52482 468719 -638 376076 393305 -639 24333 137978 -640 85453 942261 -641 336313 5832 -642 341244 222194 -643 453072 153455 -644 487770 920316 -645 105052 551959 -646 502133 398697 -647 975930 470347 -648 563214 18320 -649 985583 439263 -650 509883 2442 -651 850231 270400 -652 1931 631557 -653 516815 551556 -654 306554 62740 -655 581142 736899 -656 719858 538592 -657 184740 467634 -658 561911 948728 -659 688355 580347 -660 956227 400904 -661 704444 163086 -662 869663 388556 -663 92255 272760 -664 82825 420924 -665 951903 824519 -666 86750 961593 -667 556835 348554 -668 222403 797721 -669 821092 873610 -670 660282 504344 -671 465100 907422 -672 964089 704148 -673 847515 339046 -674 529034 587373 -675 378092 290453 -676 935495 564658 -677 487359 685363 -678 915692 888637 -679 713564 653377 -680 409154 719094 -681 730463 432944 -682 402457 76797 -683 271798 812435 -684 574503 480591 -685 620119 222864 -686 419693 100981 -687 202255 665773 -688 465408 139785 -689 675726 184299 -690 472864 203618 -691 559196 429447 -692 11770 232809 -693 94056 653806 -694 559136 480037 -695 76755 409967 -696 223217 340500 -697 253490 437418 -698 84651 364119 -699 705167 298315 -700 238739 707729 -701 663320 344734 -702 114509 174473 -703 731257 506047 -704 552888 254550 -705 591306 261885 -706 329500 233526 -707 576621 999186 -708 495876 389962 -709 179453 965038 -710 992146 907679 -711 107268 276188 -712 241852 912389 -713 559544 74958 -714 986472 27781 -715 934516 959360 -716 586896 421176 -717 592993 210978 -718 874117 325669 -719 852826 12583 -720 736932 704093 -721 474352 594097 -722 487890 169076 -723 302699 947029 -724 98648 341101 -725 877874 9692 -726 377647 677386 -727 363799 279378 -728 120833 242144 -729 751741 903530 -730 300356 405588 -731 653221 540305 -732 517617 887213 -733 787533 735402 -734 745636 407545 -735 696487 548846 -736 291314 838352 -737 87164 982454 -738 530031 429881 -739 912388 962473 -740 791556 317400 -741 323217 82941 -742 727425 528927 -743 306139 46591 -744 903559 705779 -745 423445 138714 -746 580033 445280 -747 316095 188086 -748 412778 635999 -749 506932 505435 -750 639044 872817 -751 34640 136174 -752 549545 560473 -753 686474 294706 -754 949961 834871 -755 973239 74242 -756 338585 45961 -757 480085 161641 -758 825554 207940 -759 224210 329531 -760 474434 151533 -761 228470 240200 -762 768501 823670 -763 514206 155140 -764 288807 526690 -765 295980 235325 -766 479920 77517 -767 989234 717313 -768 978815 277256 -769 602855 565783 -770 703372 98198 -771 822380 574027 -772 231345 271911 -773 910244 339833 -774 676779 492425 -775 673879 257638 -776 347191 980241 -777 209455 403718 -778 392896 957123 -779 146940 832231 -780 581658 409218 -781 691850 384177 -782 131498 875041 -783 251861 764557 -784 814616 208174 -785 251397 485721 -786 531161 731784 -787 655651 216796 -788 804342 248228 -789 30744 364782 -790 430773 557082 -791 8200 456576 -792 945032 886761 -793 543475 95742 -794 946018 114192 -795 465452 214257 -796 462639 394680 -797 314386 217651 -798 567036 90596 -799 918376 14548 -800 467569 662016 -801 309088 311996 -802 61651 116797 -803 249437 890615 -804 35209 322693 -805 860243 297153 -806 11104 32747 -807 366779 367401 -808 946578 297169 -809 69791 913847 -810 308004 161265 -811 846181 800068 -812 740605 589381 -813 176632 219164 -814 670133 538853 -815 967358 913726 -816 213018 708050 -817 504539 133619 -818 545977 975452 -819 89797 577630 -820 940183 246305 -821 25950 32170 -822 806187 304186 -823 368076 662571 -824 654075 725004 -825 41018 347871 -826 420462 379522 -827 47189 553842 -828 448997 601038 -829 807456 928031 -830 570819 273985 -831 800818 457578 -832 382510 613937 -833 271203 978933 -834 226560 62592 -835 999324 284006 -836 643093 344787 -837 872829 960133 -838 740802 466658 -839 752878 186762 -840 140383 575594 -841 411708 742101 -842 968034 693373 -843 112908 755439 -844 325539 890602 -845 862416 567043 -846 748891 27205 -847 578306 656176 -848 603211 681163 -849 159341 846054 -850 563384 901418 -851 909693 467312 -852 584621 465423 -853 936163 678361 -854 411488 353816 -855 440934 693557 -856 275498 916854 -857 65615 3776 -858 525093 773613 -859 879272 726334 -860 10725 590039 -861 819592 380505 -862 499208 97906 -863 374313 175781 -864 760165 407405 -865 23969 62441 -866 55170 399062 -867 745828 446825 -868 300095 494854 -869 676486 964257 -870 168279 339295 -871 876167 599204 -872 879877 272377 -873 747451 368386 -874 527996 480399 -875 281862 427430 -876 920997 751936 -877 822084 500942 -878 846247 510630 -879 721483 137795 -880 165328 441309 -881 1874 447230 -882 185536 101638 -883 817390 676293 -884 36976 793237 -885 256324 156707 -886 245616 398872 -887 444472 583294 -888 669041 67655 -889 558420 998265 -890 251658 863683 -891 454297 622370 -892 242077 582660 -893 60567 607939 -894 213525 644190 -895 629435 623801 -896 457877 883248 -897 707932 11572 -898 93679 477294 -899 154732 296582 -900 603336 679795 -901 809965 529731 -902 228741 418179 -903 973133 298627 -904 509859 239424 -905 440375 238308 -906 297105 77293 -907 508359 541526 -908 983432 118625 -909 739297 173964 -910 7959 340095 -911 521561 740394 -912 189901 576529 -913 346742 715884 -914 170131 26293 -915 610161 370414 -916 557796 318996 -917 557890 13350 -918 566578 377004 -919 114010 700550 -920 599228 941941 -921 868642 39561 -922 636231 289339 -923 102240 717482 -924 693347 131403 -925 664829 377794 -926 307163 128439 -927 686420 232922 -928 121998 490968 -929 971841 215391 -930 851601 596129 -931 184616 809308 -932 640196 498434 -933 369666 257543 -934 441062 219019 -935 439285 265949 -936 425278 607893 -937 509134 630164 -938 32931 393122 -939 53973 956979 -940 454530 855773 -941 744043 954739 -942 174692 14032 -943 185797 748488 -944 678800 59455 -945 643684 309034 -946 935941 894991 -947 261264 333278 -948 516662 260748 -949 530426 6066 -950 256408 709118 -951 663508 736367 -952 676872 809056 -953 633750 352424 -954 953746 672388 -955 47124 373510 -956 812168 912385 -957 571707 542566 -958 660187 289625 -959 548043 306917 -960 981772 108918 -961 727117 434996 -962 962611 730167 -963 602440 688911 -964 931021 700078 -965 996414 680507 -966 439375 381585 -967 909855 81020 -968 43605 831658 -969 383032 632125 -970 353844 896172 -971 200444 371882 -972 77683 534765 -973 581916 500945 -974 932379 531097 -975 730837 827751 -976 329726 830347 -977 9652 575900 -978 269742 281923 -979 826512 595852 -980 765450 802092 -981 969391 321621 -982 318543 150951 -983 173065 440285 -984 834701 36941 -985 960650 159242 -986 357245 16946 -987 576079 154021 -988 779191 896770 -989 425344 386710 -990 460424 320516 -991 862398 335168 -992 934627 194321 -993 711116 117753 -994 63041 758903 -995 907955 603319 -996 996956 346091 -997 671474 194602 -998 538927 183498 -999 795802 923662 -1000 755574 685146 diff --git a/devel-examples/vignette-example/instances/1000-4.tsp b/devel-examples/vignette-example/instances/1000-4.tsp deleted file mode 100644 index 877731ef..00000000 --- a/devel-examples/vignette-example/instances/1000-4.tsp +++ /dev/null @@ -1,1006 +0,0 @@ -NAME : portgen-1000-4 -COMMENT : portgen N=1000, seed=4 -TYPE : TSP -DIMENSION : 1000 -EDGE_WEIGHT_TYPE : EUC_2D -NODE_COORD_SECTION -1 213622 224693 -2 545051 491543 -3 576725 33728 -4 814556 135630 -5 858986 874678 -6 961043 390416 -7 539534 72895 -8 127459 654894 -9 667768 290420 -10 159647 52546 -11 174813 331960 -12 97712 874332 -13 897116 988006 -14 598527 851992 -15 231870 710048 -16 744526 99853 -17 856331 242294 -18 641366 247446 -19 102397 859877 -20 845803 678106 -21 304530 600919 -22 594234 842552 -23 765480 967118 -24 967757 674667 -25 78967 175255 -26 19960 380746 -27 310817 955055 -28 390269 113768 -29 368362 302757 -30 850177 329279 -31 931331 954679 -32 289827 180879 -33 570148 360124 -34 796182 696982 -35 307415 160340 -36 687136 993101 -37 211452 984391 -38 32586 794066 -39 21142 142657 -40 484062 783347 -41 619644 295769 -42 1814 902591 -43 778716 789847 -44 810026 675451 -45 672145 281242 -46 451264 405414 -47 552461 685463 -48 990970 311428 -49 389466 609842 -50 809965 971413 -51 945976 825100 -52 190604 295619 -53 555611 724190 -54 378931 408226 -55 176338 600422 -56 303742 692910 -57 630611 568935 -58 878014 525917 -59 402217 604363 -60 189909 258719 -61 970657 186339 -62 887017 336002 -63 214363 862036 -64 802496 915832 -65 428780 308396 -66 415135 612915 -67 966318 883640 -68 479605 926733 -69 665158 432879 -70 24576 252798 -71 387629 205662 -72 485541 413426 -73 310585 264925 -74 518396 216458 -75 471099 128933 -76 508932 473634 -77 181062 501568 -78 556277 333060 -79 858781 306963 -80 816014 628877 -81 59031 946051 -82 383650 923540 -83 212793 98079 -84 207368 217185 -85 258349 613089 -86 7521 185758 -87 133264 60976 -88 749787 497022 -89 5276 385449 -90 779724 881303 -91 3254 495533 -92 99818 799903 -93 249364 469083 -94 229265 42777 -95 670847 381525 -96 719365 447973 -97 174529 411486 -98 245277 201870 -99 72397 424565 -100 663638 813562 -101 259648 132947 -102 436734 589795 -103 125678 13399 -104 373816 381159 -105 252203 87194 -106 103794 816003 -107 636116 434488 -108 909511 611058 -109 771522 972163 -110 678262 10922 -111 25681 782803 -112 553546 444787 -113 353441 874573 -114 749024 543468 -115 935298 736388 -116 123206 624117 -117 133245 692530 -118 777508 187251 -119 859416 665330 -120 890391 638306 -121 697561 257101 -122 364515 659924 -123 355844 936562 -124 894426 729742 -125 58045 370703 -126 452845 528929 -127 489267 927250 -128 690355 635530 -129 999702 744204 -130 812287 938427 -131 153982 708486 -132 490768 613897 -133 389633 846693 -134 451488 976192 -135 78644 972949 -136 716632 41779 -137 914118 307558 -138 558077 496752 -139 293535 626296 -140 754431 717910 -141 874871 4820 -142 731181 996870 -143 582405 414720 -144 133349 519347 -145 302896 930815 -146 735762 883224 -147 586685 917441 -148 921674 655781 -149 342982 56956 -150 101847 859091 -151 643026 268130 -152 975310 340134 -153 495832 448025 -154 797748 492396 -155 344844 275635 -156 502181 480354 -157 441307 881472 -158 202664 270758 -159 121800 573326 -160 692223 733852 -161 503710 394532 -162 874344 219552 -163 329922 448501 -164 66469 573984 -165 811726 110051 -166 699004 801138 -167 281451 478796 -168 215728 394517 -169 563512 849709 -170 794206 311647 -171 292920 560022 -172 827124 569044 -173 427104 341230 -174 8879 367133 -175 587518 473172 -176 589311 768998 -177 245229 991795 -178 160087 841888 -179 986678 496513 -180 124406 101315 -181 884512 948039 -182 698190 33197 -183 982714 942159 -184 653229 872263 -185 454367 861434 -186 261879 754666 -187 985807 219050 -188 144540 734711 -189 149302 882549 -190 59465 488034 -191 461822 569955 -192 449578 710411 -193 225539 750965 -194 102947 248253 -195 496081 273569 -196 741287 691249 -197 395341 932771 -198 49767 538253 -199 574214 608073 -200 424503 692393 -201 191927 126329 -202 307668 99484 -203 11349 19356 -204 319420 534818 -205 766256 409121 -206 738940 738424 -207 431 850836 -208 360027 193263 -209 552697 765419 -210 983430 444461 -211 367944 45155 -212 447759 761973 -213 669506 135550 -214 446998 886323 -215 207701 125183 -216 415290 614484 -217 116293 650343 -218 749093 723398 -219 569523 598741 -220 350383 32275 -221 198267 337528 -222 264823 51620 -223 905624 696131 -224 243489 633367 -225 263265 914217 -226 91254 687891 -227 400372 299320 -228 277102 577443 -229 10035 657324 -230 350390 287951 -231 449832 720679 -232 184434 733980 -233 210854 401411 -234 473601 948811 -235 945211 663896 -236 949774 919329 -237 502153 69212 -238 353207 680053 -239 644783 148439 -240 484871 92062 -241 125514 789674 -242 535933 919749 -243 675351 694611 -244 430050 382313 -245 439489 347682 -246 249796 620712 -247 653529 686487 -248 82501 278938 -249 835375 195611 -250 698412 225571 -251 51348 95049 -252 148496 171202 -253 788703 301580 -254 151958 480623 -255 623968 582491 -256 147393 627722 -257 217835 2708 -258 38154 829120 -259 67150 497946 -260 651478 931915 -261 566036 277989 -262 250398 719640 -263 612548 854724 -264 770832 330951 -265 280509 51627 -266 528095 164160 -267 524471 902379 -268 944669 497791 -269 571838 533225 -270 881594 846231 -271 627461 932103 -272 730835 507573 -273 781645 971807 -274 370313 933888 -275 73939 227777 -276 508105 504423 -277 915102 646785 -278 697476 887188 -279 570578 246117 -280 226533 290911 -281 729741 618733 -282 599028 777737 -283 955030 215290 -284 896887 710262 -285 221062 66347 -286 458807 133261 -287 424007 423700 -288 423810 61612 -289 362887 603613 -290 22164 725360 -291 284145 524715 -292 104418 989597 -293 321885 909362 -294 565131 746733 -295 947348 729379 -296 600904 861576 -297 312162 815247 -298 387424 494200 -299 508095 307134 -300 83762 720032 -301 608919 766700 -302 911724 348579 -303 943632 983389 -304 400005 925504 -305 324899 788114 -306 322056 823844 -307 298768 497154 -308 690007 868164 -309 306570 783780 -310 390313 460830 -311 707194 589752 -312 626499 501029 -313 457427 692106 -314 221536 75428 -315 480068 440420 -316 661607 437382 -317 278713 234050 -318 403303 460300 -319 225947 607264 -320 299590 453721 -321 602791 781350 -322 356420 486518 -323 22184 11152 -324 235077 811132 -325 357819 695317 -326 272663 432667 -327 827066 643342 -328 58425 171536 -329 487986 677674 -330 945275 483331 -331 757442 792741 -332 625914 871178 -333 185322 540705 -334 467424 812249 -335 474969 678855 -336 633086 495437 -337 425961 694996 -338 188166 274526 -339 762686 983156 -340 442604 285891 -341 204120 543861 -342 130152 996736 -343 682978 868865 -344 811468 407534 -345 48727 862598 -346 992876 413697 -347 132294 620735 -348 820634 107353 -349 355389 661424 -350 298351 747657 -351 248465 251920 -352 368528 71928 -353 491196 728801 -354 302514 830329 -355 960363 189559 -356 360067 80452 -357 628946 82676 -358 490455 343744 -359 660446 5178 -360 50544 77968 -361 185315 805999 -362 513898 727311 -363 430389 381165 -364 126909 354033 -365 203799 459364 -366 972012 932356 -367 22793 253045 -368 925823 123667 -369 914914 47475 -370 506280 339234 -371 208419 806289 -372 356990 970758 -373 677282 186876 -374 899799 404983 -375 190346 439468 -376 980443 1356 -377 457625 838986 -378 775645 316109 -379 229127 115482 -380 146104 367529 -381 813886 255038 -382 324048 621128 -383 981140 553777 -384 723461 658187 -385 405394 303579 -386 443944 255462 -387 814832 611075 -388 97524 183959 -389 348374 674911 -390 951665 114280 -391 152038 11426 -392 207928 836270 -393 645478 716973 -394 608307 401665 -395 271905 372045 -396 400206 256727 -397 642081 202701 -398 895289 952956 -399 991457 745914 -400 873234 493323 -401 838501 224888 -402 453318 76065 -403 287429 969016 -404 793428 621355 -405 193508 58672 -406 707801 827462 -407 843577 774059 -408 967322 557159 -409 612957 121347 -410 725838 28183 -411 562320 977547 -412 784953 912070 -413 465078 219056 -414 802144 738766 -415 323646 128507 -416 390530 727019 -417 481402 892992 -418 406478 324576 -419 167849 433868 -420 868947 88318 -421 104016 486960 -422 675826 243721 -423 809724 422658 -424 471773 730011 -425 737623 676233 -426 150811 252690 -427 422268 744726 -428 102793 111482 -429 743485 560325 -430 669586 962852 -431 801167 359560 -432 752408 105190 -433 954656 220841 -434 151635 599856 -435 964334 544663 -436 85386 882945 -437 383724 49605 -438 877372 309630 -439 555278 40227 -440 809277 353595 -441 475571 241819 -442 69179 360793 -443 327339 30970 -444 974611 376212 -445 938336 185637 -446 172941 567992 -447 469533 324283 -448 2931 221070 -449 103235 626221 -450 366348 500094 -451 867380 431546 -452 920734 384027 -453 200662 908992 -454 183510 61475 -455 417386 71823 -456 136870 367272 -457 621988 483949 -458 789911 233175 -459 890026 428124 -460 102258 733585 -461 117605 525414 -462 233507 464240 -463 677283 653839 -464 962211 999697 -465 848943 968380 -466 126119 493802 -467 622840 737454 -468 216725 108298 -469 619830 585230 -470 570881 94164 -471 140943 546486 -472 273954 204750 -473 68032 647526 -474 334484 5293 -475 646999 349092 -476 258859 103538 -477 777278 397968 -478 373975 373577 -479 808705 183280 -480 167302 92363 -481 289161 598280 -482 490594 323222 -483 930879 590384 -484 93318 417237 -485 415916 142384 -486 898690 884733 -487 781125 753166 -488 474726 14066 -489 748136 835539 -490 90264 303705 -491 845134 778931 -492 832394 756579 -493 679218 527838 -494 3208 299618 -495 806574 626340 -496 14979 202592 -497 169313 428496 -498 195473 256210 -499 765361 520787 -500 730024 53965 -501 899390 498944 -502 915028 343293 -503 503957 479927 -504 271143 20698 -505 718749 846137 -506 370369 509087 -507 376705 540961 -508 77383 86568 -509 428967 62097 -510 127748 674668 -511 825022 572530 -512 687213 361951 -513 242994 399745 -514 969704 437831 -515 249208 994798 -516 742923 727437 -517 116790 244127 -518 933336 336046 -519 402226 291433 -520 679195 592650 -521 98871 941110 -522 171870 131906 -523 801317 442448 -524 515378 807361 -525 185501 795727 -526 286506 327530 -527 271579 735225 -528 311042 171952 -529 382154 670901 -530 409957 167910 -531 77701 979710 -532 341502 126099 -533 747265 429258 -534 337216 244798 -535 739643 634934 -536 571189 621605 -537 876596 332020 -538 388161 497492 -539 300951 951988 -540 50909 71042 -541 17591 298802 -542 27874 81297 -543 917097 763212 -544 385935 990690 -545 496861 504078 -546 998830 157427 -547 551789 44261 -548 21460 477265 -549 64514 839849 -550 743744 303824 -551 141497 563390 -552 756452 114459 -553 778136 987704 -554 299655 190282 -555 818128 547829 -556 786017 391463 -557 174040 905878 -558 169080 920274 -559 427921 297241 -560 104638 269999 -561 364743 497367 -562 501053 435819 -563 493437 7799 -564 865153 762136 -565 553884 400457 -566 197837 110669 -567 133859 503080 -568 285025 626127 -569 124762 121995 -570 912216 996823 -571 335291 88694 -572 886052 226861 -573 139334 501462 -574 656373 115970 -575 550823 13661 -576 612112 302378 -577 285965 343287 -578 105987 30828 -579 429530 253372 -580 829434 152009 -581 862941 177660 -582 278065 821305 -583 212537 697322 -584 505411 947178 -585 766543 667618 -586 263900 311950 -587 746417 90977 -588 657887 62365 -589 211402 157766 -590 329831 462609 -591 578268 611781 -592 932702 401875 -593 537516 20176 -594 832603 312554 -595 290542 587702 -596 120715 177584 -597 355451 244597 -598 732922 23340 -599 342276 795074 -600 568974 76969 -601 290060 498607 -602 786138 88214 -603 435392 331 -604 369676 884089 -605 805771 85810 -606 198224 116975 -607 962829 241732 -608 31293 685357 -609 822208 33467 -610 88382 189197 -611 355046 710337 -612 378204 689574 -613 377558 765292 -614 525811 658202 -615 655585 657555 -616 692689 327312 -617 351995 244020 -618 264384 461292 -619 648951 690969 -620 370581 852158 -621 197968 799135 -622 224172 101345 -623 232656 410378 -624 799379 665877 -625 867039 967630 -626 497528 684074 -627 139489 911418 -628 384279 962747 -629 146612 542118 -630 823830 974100 -631 351380 678706 -632 513507 953612 -633 887842 399089 -634 892803 861484 -635 9076 620915 -636 885977 156330 -637 166428 120751 -638 691668 670971 -639 570848 466785 -640 305294 414811 -641 618679 983693 -642 834372 681484 -643 306175 13983 -644 813804 398382 -645 356178 865295 -646 568489 787466 -647 681893 749666 -648 966181 41637 -649 632706 103420 -650 409677 561684 -651 839530 332594 -652 360582 452228 -653 348950 513835 -654 849702 458004 -655 605243 370296 -656 148942 748230 -657 185940 958535 -658 405611 563913 -659 996812 763841 -660 987431 846204 -661 766382 789382 -662 451807 447392 -663 781385 553383 -664 795748 714200 -665 771800 177832 -666 821269 112843 -667 861542 934998 -668 265868 870449 -669 797753 875836 -670 275873 742262 -671 17170 49963 -672 410951 509973 -673 98912 779106 -674 335659 234501 -675 968281 412798 -676 245889 918506 -677 331620 231844 -678 740414 726686 -679 471052 425584 -680 186359 478501 -681 716082 973865 -682 182130 862980 -683 353125 98978 -684 337278 675966 -685 859622 626504 -686 228253 762310 -687 795559 574633 -688 600315 847875 -689 457762 219963 -690 706977 54698 -691 82330 370164 -692 527840 293299 -693 461750 847404 -694 930712 998561 -695 581872 166889 -696 533170 121786 -697 16213 649369 -698 514009 254860 -699 254404 836317 -700 909657 251036 -701 321344 115696 -702 527523 913582 -703 330467 875725 -704 390665 38320 -705 770093 893009 -706 795973 472490 -707 843711 19469 -708 945330 594295 -709 957651 532761 -710 348971 98265 -711 844574 500961 -712 766308 608585 -713 305160 112556 -714 234786 881977 -715 244166 724590 -716 457209 419441 -717 449869 813967 -718 258725 609840 -719 526452 508371 -720 347969 867455 -721 889752 397951 -722 649590 483607 -723 322315 32209 -724 355477 407628 -725 344208 401452 -726 20073 372426 -727 592151 185067 -728 793827 901902 -729 665827 713556 -730 654857 720627 -731 349272 882294 -732 690351 902638 -733 3257 398022 -734 822900 360103 -735 697154 913120 -736 238817 550023 -737 188553 947519 -738 78191 472147 -739 908809 581241 -740 814758 403257 -741 446729 521230 -742 227119 523539 -743 375317 574915 -744 729090 547230 -745 810710 860703 -746 786939 166348 -747 811216 434848 -748 628637 339728 -749 209398 702070 -750 405416 850167 -751 123400 774236 -752 592869 940950 -753 954722 498843 -754 145306 68612 -755 809749 218911 -756 172812 118596 -757 902845 794153 -758 933687 182923 -759 71077 255503 -760 274001 663528 -761 188624 120829 -762 954687 846987 -763 789720 464581 -764 957154 247602 -765 992797 579348 -766 326840 840196 -767 771492 595846 -768 230445 328132 -769 618385 432966 -770 589851 192394 -771 503837 473587 -772 273228 147182 -773 672079 666109 -774 211661 964229 -775 645127 164055 -776 382574 961795 -777 709273 826067 -778 523326 283203 -779 2743 997022 -780 710505 626589 -781 880457 712340 -782 478761 617355 -783 715074 699225 -784 845368 755663 -785 122074 267577 -786 971262 106848 -787 610375 109945 -788 280953 226828 -789 411556 128619 -790 323660 506517 -791 461837 960131 -792 537097 366207 -793 698890 614500 -794 361435 154137 -795 880772 531220 -796 482764 862721 -797 310891 322273 -798 221132 396988 -799 863212 163283 -800 866228 445250 -801 254553 83041 -802 640568 138610 -803 702218 422442 -804 424698 343066 -805 127177 908826 -806 921767 848606 -807 116250 179284 -808 143824 17736 -809 401448 156487 -810 396222 318085 -811 836012 682084 -812 889434 676823 -813 13024 888220 -814 466279 471764 -815 407727 858510 -816 802129 68490 -817 1442 414834 -818 584749 613230 -819 843881 357812 -820 222382 681154 -821 213051 204947 -822 757914 562686 -823 695207 800679 -824 973287 634068 -825 309249 332911 -826 930709 391448 -827 755555 7717 -828 643121 186063 -829 81599 225734 -830 553861 88987 -831 578561 66886 -832 120683 446022 -833 695774 716819 -834 90692 553564 -835 484076 343144 -836 44448 767380 -837 847238 63311 -838 387376 444564 -839 926529 881717 -840 33702 826960 -841 806621 240545 -842 917902 318740 -843 279949 735243 -844 947806 555420 -845 719059 867929 -846 522538 290316 -847 873735 879237 -848 636705 445670 -849 357709 694602 -850 175309 250642 -851 874150 91569 -852 600366 482288 -853 526290 690163 -854 473545 436815 -855 727768 907877 -856 238256 526179 -857 506674 685932 -858 566449 288244 -859 193150 241445 -860 809316 250104 -861 359110 396089 -862 378254 233434 -863 468994 952879 -864 167014 364949 -865 537021 697212 -866 971019 489714 -867 153948 125824 -868 588703 280442 -869 733870 231969 -870 752290 991704 -871 542093 706361 -872 746103 468955 -873 508819 126448 -874 912062 640301 -875 410243 683826 -876 278656 992759 -877 157581 478096 -878 279623 384436 -879 937620 474542 -880 893584 245847 -881 956293 241576 -882 684524 736063 -883 365784 531895 -884 780075 37719 -885 177113 440000 -886 376182 552848 -887 831201 125490 -888 971448 366350 -889 238508 900158 -890 953810 84558 -891 15258 692472 -892 471365 291173 -893 740919 729443 -894 805189 417885 -895 760039 56808 -896 500366 696151 -897 54856 312289 -898 615521 989244 -899 875159 620612 -900 497507 142468 -901 887939 11904 -902 686491 325685 -903 668568 586183 -904 521394 866407 -905 737176 550180 -906 579246 519735 -907 714502 836776 -908 745481 260141 -909 186719 372234 -910 120541 376540 -911 656735 159463 -912 540211 34644 -913 552061 364278 -914 866357 505516 -915 456921 385264 -916 844955 372100 -917 162981 403629 -918 505311 495522 -919 977969 634588 -920 545692 480778 -921 542723 432954 -922 297343 383498 -923 400072 340903 -924 155939 20212 -925 760228 251243 -926 122887 369642 -927 163690 112243 -928 297513 515839 -929 848923 282862 -930 820373 173046 -931 608214 886806 -932 320715 256398 -933 7457 146292 -934 222392 331004 -935 436703 404578 -936 104201 166507 -937 612006 869298 -938 253653 287092 -939 995772 427968 -940 737130 36221 -941 515355 583495 -942 685142 283875 -943 777050 958149 -944 51385 906582 -945 396172 359019 -946 273129 646965 -947 197884 141114 -948 376576 376216 -949 820948 428044 -950 129844 112979 -951 345130 727970 -952 283081 724006 -953 735888 539391 -954 684500 879815 -955 335192 339364 -956 464453 942340 -957 886689 461354 -958 899916 961248 -959 688921 179600 -960 879822 631241 -961 325343 794347 -962 201159 323724 -963 59447 376230 -964 883426 887999 -965 133409 714261 -966 602592 115957 -967 92775 397766 -968 571768 573014 -969 696805 223788 -970 383959 815801 -971 269227 871785 -972 26760 764930 -973 33675 478781 -974 445805 874160 -975 81667 345 -976 492789 932949 -977 294635 415583 -978 510386 229173 -979 635195 885314 -980 152238 162873 -981 842586 460712 -982 495855 519391 -983 70136 592667 -984 915579 121759 -985 427679 421134 -986 515442 814760 -987 97933 879476 -988 138451 392394 -989 499712 785576 -990 813337 830273 -991 741035 998111 -992 735761 970536 -993 871674 141880 -994 620101 573384 -995 327630 979100 -996 657435 575046 -997 796108 962824 -998 300358 454467 -999 773852 147284 -1000 626479 641280 diff --git a/devel-examples/vignette-example/instances/1000-5.tsp b/devel-examples/vignette-example/instances/1000-5.tsp deleted file mode 100644 index 610a19a8..00000000 --- a/devel-examples/vignette-example/instances/1000-5.tsp +++ /dev/null @@ -1,1006 +0,0 @@ -NAME : portgen-1000-5 -COMMENT : portgen N=1000, seed=5 -TYPE : TSP -DIMENSION : 1000 -EDGE_WEIGHT_TYPE : EUC_2D -NODE_COORD_SECTION -1 310938 549569 -2 934232 356709 -3 379044 972178 -4 460173 909711 -5 770867 699563 -6 937861 47743 -7 681396 360355 -8 528493 883040 -9 825609 46141 -10 197751 546743 -11 144878 40457 -12 59610 725881 -13 721679 790281 -14 727299 462006 -15 731163 716345 -16 237932 130728 -17 556534 207052 -18 922169 901286 -19 69674 76776 -20 867825 924935 -21 517025 884226 -22 472786 68248 -23 564503 602377 -24 981030 114850 -25 779861 960449 -26 815667 703367 -27 846667 851 -28 468967 180209 -29 993034 727180 -30 434539 477758 -31 902503 383397 -32 41886 845931 -33 182538 53635 -34 574957 613148 -35 795851 926116 -36 902010 710758 -37 266280 237302 -38 731075 441511 -39 193789 58759 -40 256913 541469 -41 797247 119 -42 27466 253405 -43 813842 854535 -44 88841 710602 -45 24514 868534 -46 326328 456526 -47 280924 941708 -48 22924 806266 -49 617945 235484 -50 337172 122992 -51 408587 922271 -52 857937 238391 -53 163201 815548 -54 675900 593262 -55 187009 614432 -56 91368 282432 -57 702665 566004 -58 151429 445976 -59 102472 100177 -60 823007 376271 -61 435689 339473 -62 275975 672859 -63 517528 979739 -64 852821 27889 -65 74100 915526 -66 765611 600526 -67 871750 642480 -68 450101 514815 -69 297453 461461 -70 101975 367865 -71 752063 988663 -72 887595 648242 -73 432844 986855 -74 180551 608065 -75 424180 43185 -76 953445 590056 -77 161383 421645 -78 357381 808060 -79 50520 215456 -80 788289 648386 -81 518095 214438 -82 491286 819143 -83 862369 102704 -84 394836 54423 -85 133160 164573 -86 265425 494406 -87 675997 779821 -88 422826 845633 -89 178089 854329 -90 315477 709467 -91 929218 637364 -92 239599 425713 -93 397431 551172 -94 109240 52606 -95 780111 347397 -96 119978 243029 -97 328301 937402 -98 102440 257656 -99 312666 107773 -100 225415 587211 -101 808765 326221 -102 292587 714712 -103 113966 316080 -104 350457 735669 -105 24213 806208 -106 698820 997914 -107 435345 440892 -108 528408 275065 -109 886136 553884 -110 716703 604713 -111 790037 287063 -112 829007 545948 -113 355808 939203 -114 201819 961285 -115 665855 106746 -116 495176 442419 -117 830116 509269 -118 10647 931304 -119 202019 798706 -120 897305 122366 -121 665035 555355 -122 335903 175398 -123 557359 832915 -124 414021 782353 -125 581594 163236 -126 55836 351381 -127 441917 118669 -128 92035 366345 -129 496105 783318 -130 704064 182661 -131 114060 551750 -132 838364 901847 -133 141173 143464 -134 662011 259946 -135 883533 695493 -136 861043 103783 -137 972290 553467 -138 548876 438656 -139 845145 710338 -140 453913 989462 -141 443098 418500 -142 257220 483193 -143 992685 943425 -144 604055 928268 -145 368095 867182 -146 269293 942072 -147 915173 201812 -148 261322 561251 -149 583065 782435 -150 626521 118702 -151 987770 703683 -152 328440 592131 -153 720137 637336 -154 94161 958724 -155 125983 148609 -156 762290 567837 -157 415222 836881 -158 913368 171987 -159 636576 636552 -160 640525 579921 -161 560399 879575 -162 633425 764830 -163 707722 157360 -164 775343 380158 -165 833329 911540 -166 344495 886420 -167 584354 305303 -168 227171 875261 -169 3277 420338 -170 569824 820698 -171 306848 967503 -172 287742 788174 -173 306783 389717 -174 308647 150343 -175 494089 103961 -176 785907 202906 -177 949106 714981 -178 774207 101349 -179 119328 23136 -180 364960 844876 -181 634058 673822 -182 388899 305285 -183 841760 794787 -184 280094 627048 -185 530098 523650 -186 863340 486233 -187 142463 536564 -188 794013 357493 -189 930468 918444 -190 990622 606373 -191 38032 752207 -192 15198 988452 -193 277481 670672 -194 497521 279069 -195 463542 432384 -196 595166 376228 -197 890239 46174 -198 957357 820615 -199 825040 751178 -200 994160 949290 -201 459249 390202 -202 159720 887715 -203 65929 33700 -204 187707 960653 -205 437499 103534 -206 603827 840259 -207 559594 932575 -208 249709 257829 -209 783582 342725 -210 347927 21145 -211 969747 528915 -212 632888 580808 -213 64401 473137 -214 326512 254747 -215 470634 760313 -216 169785 969815 -217 480945 887087 -218 2546 197773 -219 192612 82623 -220 738743 19651 -221 887089 154796 -222 931141 442396 -223 462637 66251 -224 743340 309431 -225 981772 484220 -226 494103 570292 -227 280543 233847 -228 779504 489434 -229 909257 272633 -230 885169 868156 -231 841088 105084 -232 221910 417847 -233 216445 449031 -234 909117 117197 -235 469938 183458 -236 514489 474151 -237 360952 863706 -238 527041 399454 -239 248372 399041 -240 801303 574967 -241 563879 53878 -242 369578 602478 -243 919225 64700 -244 747905 63097 -245 670642 553515 -246 288655 75415 -247 612684 555285 -248 505162 412938 -249 793843 67434 -250 915355 63182 -251 817879 344299 -252 508127 406805 -253 920341 440224 -254 200714 678065 -255 314622 714804 -256 741529 846160 -257 601990 331654 -258 579500 765672 -259 492400 666624 -260 912684 803506 -261 655187 841682 -262 201842 406756 -263 365579 170189 -264 966023 954147 -265 943365 86817 -266 198740 570306 -267 84419 86499 -268 833438 717719 -269 451888 37924 -270 22977 153552 -271 572300 81280 -272 150412 867135 -273 898327 446972 -274 873572 205928 -275 189705 334972 -276 446914 839696 -277 124068 828537 -278 864441 247572 -279 259880 421628 -280 573367 202621 -281 988336 162789 -282 655087 161069 -283 142503 660249 -284 695747 734854 -285 433326 132527 -286 892100 286471 -287 476919 577711 -288 356592 815490 -289 717613 373304 -290 542314 118006 -291 910309 544395 -292 380780 740744 -293 98481 35950 -294 915219 923349 -295 943995 173189 -296 21971 717033 -297 604597 890449 -298 261451 285828 -299 604360 572700 -300 510543 82836 -301 729358 500267 -302 663614 71699 -303 424663 902518 -304 458916 383324 -305 730056 828490 -306 332353 336531 -307 477632 400178 -308 180649 271302 -309 558192 764637 -310 899618 856675 -311 55888 123046 -312 224311 350489 -313 403168 391832 -314 622857 405220 -315 153048 454073 -316 356574 334289 -317 643248 713823 -318 785653 573778 -319 66763 980601 -320 560094 827178 -321 477758 150581 -322 23730 87320 -323 117301 898925 -324 492722 254107 -325 487281 869619 -326 662971 199140 -327 419652 56469 -328 726262 395069 -329 857019 949790 -330 286045 850884 -331 835755 478314 -332 823230 902877 -333 350732 181772 -334 312800 390312 -335 282877 281724 -336 778580 304084 -337 277356 29999 -338 193703 856747 -339 703394 167841 -340 624227 8099 -341 534813 673066 -342 119174 302163 -343 618318 878259 -344 511059 740371 -345 363090 603881 -346 260978 676450 -347 697724 278370 -348 48598 173674 -349 873224 993731 -350 893616 260553 -351 195530 324880 -352 629880 479181 -353 334805 989904 -354 79966 117488 -355 438150 848002 -356 884010 116648 -357 586699 682164 -358 589906 159304 -359 780589 544860 -360 854278 177058 -361 308547 319068 -362 496695 22323 -363 86193 453699 -364 674203 798175 -365 695193 203798 -366 776780 585906 -367 729690 776224 -368 124089 418165 -369 86366 437010 -370 712257 459013 -371 97670 966199 -372 886092 186032 -373 295333 941909 -374 179755 675401 -375 192176 594899 -376 499470 75049 -377 298538 689817 -378 483773 609624 -379 595189 853655 -380 355092 916640 -381 903538 642956 -382 405231 979137 -383 750332 917810 -384 230555 400667 -385 386830 647997 -386 979549 105187 -387 352683 259379 -388 677587 233497 -389 20529 806877 -390 538549 476569 -391 858510 820547 -392 443082 778552 -393 300260 133823 -394 180675 750553 -395 25892 206278 -396 187609 685699 -397 50179 64259 -398 479463 992482 -399 613515 626713 -400 508445 61836 -401 921379 372877 -402 136851 715607 -403 736389 678922 -404 631967 519986 -405 389557 349949 -406 428949 844635 -407 827763 148814 -408 729030 217839 -409 592776 340971 -410 499673 757850 -411 304295 603841 -412 892221 324994 -413 726617 606672 -414 968336 637076 -415 522990 998664 -416 601530 500543 -417 417319 188600 -418 47620 13874 -419 992784 294268 -420 49521 82421 -421 541046 839703 -422 250880 268041 -423 901982 583768 -424 793477 725185 -425 337641 872791 -426 24146 976438 -427 103723 509780 -428 460305 420835 -429 955557 948251 -430 667986 722514 -431 686138 337699 -432 470464 307136 -433 808902 589245 -434 593755 559721 -435 246831 145262 -436 424361 867591 -437 3329 626881 -438 733704 327856 -439 500118 382441 -440 864688 305781 -441 651114 20084 -442 969089 800475 -443 312525 263831 -444 30079 110183 -445 379697 458374 -446 420118 433062 -447 47436 904259 -448 658059 673455 -449 836374 623998 -450 534337 574126 -451 83649 411036 -452 860497 31859 -453 221676 4061 -454 7349 303247 -455 197254 196474 -456 390756 845373 -457 568553 209611 -458 302395 253076 -459 290262 566205 -460 649076 135446 -461 752870 969757 -462 25384 672704 -463 61613 13324 -464 7093 971469 -465 405205 729642 -466 320507 196870 -467 185186 668213 -468 915025 805740 -469 451531 759478 -470 498079 59449 -471 973569 463874 -472 461107 244251 -473 705504 450361 -474 407678 374732 -475 842646 644735 -476 666361 864904 -477 218792 804694 -478 253619 886779 -479 225850 192283 -480 116834 415935 -481 552530 247871 -482 805168 137805 -483 222905 926882 -484 384266 324302 -485 504107 852034 -486 845398 915529 -487 723559 4341 -488 469085 887966 -489 750964 220689 -490 419085 174833 -491 787474 814810 -492 854635 989269 -493 177112 72636 -494 391701 47381 -495 445308 988143 -496 421473 127228 -497 255370 646044 -498 214051 58039 -499 740315 456765 -500 775165 817537 -501 699396 186988 -502 955647 667812 -503 857260 851551 -504 10269 229523 -505 627582 180983 -506 495077 178468 -507 746975 128691 -508 994461 425301 -509 992500 159123 -510 923754 164865 -511 186567 927501 -512 549136 686569 -513 152638 658409 -514 959882 55746 -515 147081 617533 -516 877696 521441 -517 593106 238101 -518 679756 609006 -519 67835 725943 -520 994807 751810 -521 80135 232577 -522 123627 280442 -523 801575 493972 -524 578092 568800 -525 493405 555641 -526 98156 684568 -527 309684 157632 -528 939841 177955 -529 593882 717545 -530 988055 248254 -531 783716 284325 -532 234715 875771 -533 100847 262499 -534 54841 466532 -535 327115 500489 -536 847209 423700 -537 665717 368112 -538 66709 501999 -539 617816 391504 -540 746728 974683 -541 64527 242337 -542 67690 898827 -543 833817 593370 -544 286725 717334 -545 137254 417256 -546 554164 601302 -547 398828 494318 -548 904601 656434 -549 566859 755514 -550 213733 299576 -551 876155 186588 -552 822072 518722 -553 491113 855818 -554 616877 410857 -555 323814 346470 -556 891229 876547 -557 580291 570799 -558 694089 182413 -559 885496 740397 -560 971170 444412 -561 695639 299327 -562 252799 27538 -563 624333 660621 -564 601628 146995 -565 876998 210890 -566 885121 206959 -567 67689 400257 -568 83453 187980 -569 662046 496891 -570 204737 651404 -571 707873 546327 -572 746164 692841 -573 721617 254837 -574 348503 371290 -575 869984 243980 -576 54806 419864 -577 878515 2842 -578 414455 669196 -579 118898 421814 -580 435268 303133 -581 193772 119985 -582 206119 672410 -583 638596 344902 -584 130382 887449 -585 849181 439252 -586 833909 514206 -587 870413 727189 -588 389606 275775 -589 420811 249956 -590 613082 955137 -591 541722 179813 -592 711727 573865 -593 17118 765135 -594 839 395278 -595 761660 738551 -596 57598 774596 -597 647709 765485 -598 817494 193667 -599 675914 18975 -600 303235 445842 -601 834026 98547 -602 758207 914847 -603 702258 874993 -604 708137 304650 -605 985724 649319 -606 668356 723619 -607 660154 696716 -608 245535 419175 -609 472275 440634 -610 854916 444929 -611 668987 111406 -612 584214 403339 -613 605226 735362 -614 755998 955565 -615 24931 514612 -616 567637 116160 -617 264231 963763 -618 286781 818102 -619 519659 15010 -620 328329 597942 -621 292860 560205 -622 540362 316731 -623 69564 946191 -624 190381 244369 -625 160259 82132 -626 437669 720348 -627 994044 788622 -628 878204 717865 -629 834316 794444 -630 628066 884155 -631 355334 693127 -632 976320 387782 -633 356459 108150 -634 183257 343422 -635 627152 299343 -636 228794 227905 -637 280375 772784 -638 7260 948638 -639 117362 795592 -640 525135 887360 -641 901045 961554 -642 327499 140775 -643 159278 874510 -644 139839 876448 -645 607304 178630 -646 634845 176236 -647 387857 28985 -648 369147 64954 -649 279830 767578 -650 309471 120925 -651 828828 394788 -652 719234 272898 -653 181086 476114 -654 392849 853268 -655 629343 3693 -656 578026 957867 -657 187140 449436 -658 249310 179097 -659 305269 947335 -660 18635 291505 -661 828320 415678 -662 33951 506227 -663 470514 834005 -664 508671 7476 -665 591697 531145 -666 555789 264093 -667 166248 521442 -668 309334 943178 -669 774414 878063 -670 891464 980180 -671 569241 192504 -672 857813 315798 -673 350310 219166 -674 142284 881629 -675 558470 535142 -676 556282 272354 -677 175880 778325 -678 565136 564735 -679 228540 197791 -680 963564 237908 -681 701700 514786 -682 961804 649163 -683 434452 385521 -684 100054 871341 -685 99126 30144 -686 36813 423639 -687 388864 483493 -688 735222 555966 -689 239798 255626 -690 941091 905779 -691 605465 310880 -692 43911 353789 -693 829444 41002 -694 302289 517085 -695 86990 923812 -696 843123 903072 -697 778937 861319 -698 943367 145601 -699 803640 374320 -700 580576 794343 -701 979368 886658 -702 940538 652691 -703 929676 245402 -704 228442 822091 -705 948880 524133 -706 262446 711454 -707 110801 39752 -708 394784 798628 -709 735849 100484 -710 705796 288851 -711 581881 725734 -712 290765 304782 -713 50776 150155 -714 483101 736172 -715 553816 489820 -716 327523 417707 -717 306745 416957 -718 643333 894010 -719 200078 4159 -720 959005 30816 -721 305152 201804 -722 811289 798139 -723 341931 117389 -724 612307 474154 -725 810543 793212 -726 662499 67467 -727 820504 90755 -728 466820 561660 -729 579913 523581 -730 9357 35665 -731 45323 224283 -732 863085 918064 -733 218981 60641 -734 900165 312661 -735 697821 277394 -736 186320 261694 -737 289941 912583 -738 626351 514413 -739 905230 200009 -740 837961 489115 -741 570241 959519 -742 726814 518150 -743 444496 103239 -744 554621 388680 -745 197976 582692 -746 993845 887417 -747 306337 681610 -748 844496 43458 -749 911863 898705 -750 171788 827517 -751 212159 412298 -752 636193 321427 -753 222970 702980 -754 340652 302353 -755 646258 363581 -756 7039 191232 -757 325605 426665 -758 41820 157906 -759 917945 181474 -760 73568 175522 -761 148778 1459 -762 140873 870303 -763 65234 774022 -764 625500 968513 -765 689613 923371 -766 173761 602876 -767 553750 474379 -768 482076 379009 -769 633914 300149 -770 476330 286590 -771 185293 373146 -772 315112 22453 -773 433914 992386 -774 746543 436033 -775 616375 70473 -776 417957 943349 -777 209091 248417 -778 653756 609283 -779 858547 161813 -780 839351 843960 -781 69065 40503 -782 826023 359668 -783 178287 633892 -784 876119 303151 -785 992751 49434 -786 411362 481911 -787 565098 3094 -788 757565 205428 -789 792367 892456 -790 216547 455951 -791 915474 463687 -792 129162 845653 -793 854305 133257 -794 776852 194081 -795 296091 848183 -796 502889 330762 -797 307397 426896 -798 875227 703381 -799 808047 312017 -800 264887 228485 -801 200018 854086 -802 219486 160424 -803 154998 954270 -804 814187 363438 -805 394111 520498 -806 832430 664465 -807 865721 991167 -808 341070 738302 -809 733105 399127 -810 484441 474906 -811 825844 564101 -812 38264 764265 -813 849415 557276 -814 262425 404674 -815 848096 803295 -816 391240 428929 -817 498345 696049 -818 623520 251009 -819 597965 137994 -820 504582 116003 -821 400151 377725 -822 709640 821185 -823 22338 938787 -824 292498 543131 -825 577480 317951 -826 440955 403373 -827 463921 461592 -828 837245 771088 -829 355741 523436 -830 536903 903989 -831 356305 676193 -832 858855 278107 -833 120346 454705 -834 954824 44536 -835 968829 402282 -836 445803 189974 -837 821647 166489 -838 33950 422471 -839 100180 576671 -840 927020 78327 -841 201534 738989 -842 867770 944106 -843 446989 715047 -844 570073 220237 -845 575702 168815 -846 296184 553428 -847 169164 102299 -848 670199 210177 -849 556077 543151 -850 787234 599867 -851 838607 715827 -852 616111 499153 -853 116416 701966 -854 535603 519814 -855 14602 122198 -856 201014 135503 -857 947734 368088 -858 607804 802876 -859 507028 756555 -860 607908 910169 -861 898628 411673 -862 257301 368961 -863 563675 729976 -864 573863 322494 -865 50073 331984 -866 886867 580365 -867 562068 804822 -868 877312 66030 -869 791255 499681 -870 336302 644113 -871 208018 813517 -872 612328 665533 -873 270186 884510 -874 296127 800203 -875 538624 940222 -876 636314 233583 -877 493077 455250 -878 712999 258241 -879 153758 811288 -880 621841 50385 -881 910711 35921 -882 183512 370489 -883 914179 387497 -884 523174 282200 -885 97901 723294 -886 506748 706825 -887 217931 667685 -888 273854 665044 -889 918595 802050 -890 655961 305433 -891 576218 762574 -892 700652 999687 -893 421272 850946 -894 396853 191579 -895 890642 489814 -896 542855 509054 -897 401779 613007 -898 137364 501193 -899 595586 944642 -900 391678 605142 -901 965914 494077 -902 144241 233191 -903 364003 873740 -904 532930 493390 -905 33978 862053 -906 861388 962178 -907 920645 132026 -908 507530 401657 -909 634141 570505 -910 233124 412985 -911 791910 578531 -912 890522 492759 -913 757379 12671 -914 562583 984739 -915 303682 400114 -916 132113 425205 -917 768072 793908 -918 444044 614040 -919 841928 568626 -920 492156 19615 -921 216804 826348 -922 958455 477657 -923 697904 964323 -924 618532 909020 -925 855627 124692 -926 938610 610846 -927 640960 991564 -928 473028 540709 -929 726004 350333 -930 789146 749963 -931 31811 964304 -932 1233 14362 -933 645248 35040 -934 3722 442988 -935 434121 543206 -936 783124 725121 -937 714877 108431 -938 474375 181063 -939 937571 898957 -940 19731 216670 -941 286666 212250 -942 195593 553719 -943 368302 167809 -944 423971 753709 -945 148660 409004 -946 610317 398939 -947 134504 948950 -948 236490 491683 -949 111470 850023 -950 3281 516841 -951 26752 719574 -952 889289 638957 -953 838026 726359 -954 415253 87241 -955 623261 305218 -956 116738 972295 -957 201672 380142 -958 139645 632872 -959 829799 52283 -960 777871 153564 -961 923570 153699 -962 439707 917280 -963 516453 63550 -964 835831 75919 -965 270405 748015 -966 765809 850329 -967 275696 714512 -968 99932 314371 -969 10577 815450 -970 414073 735430 -971 338010 371687 -972 975837 995095 -973 485433 456618 -974 959232 217224 -975 432496 172939 -976 655852 35550 -977 579618 255265 -978 751031 176422 -979 443877 174776 -980 539025 523654 -981 715782 599802 -982 673167 887831 -983 967207 745050 -984 996457 206576 -985 894708 683026 -986 673639 612574 -987 619786 604932 -988 497711 888019 -989 574080 184441 -990 166249 340030 -991 619673 661054 -992 536894 746751 -993 32233 166007 -994 177161 387865 -995 747305 354881 -996 317914 804000 -997 920742 731046 -998 61790 725436 -999 751901 370905 -1000 497383 597414 diff --git a/devel-examples/vignette-example/instances/1000-6.tsp b/devel-examples/vignette-example/instances/1000-6.tsp deleted file mode 100644 index 5c016383..00000000 --- a/devel-examples/vignette-example/instances/1000-6.tsp +++ /dev/null @@ -1,1006 +0,0 @@ -NAME : portgen-1000-6 -COMMENT : portgen N=1000, seed=6 -TYPE : TSP -DIMENSION : 1000 -EDGE_WEIGHT_TYPE : EUC_2D -NODE_COORD_SECTION -1 408254 874445 -2 323414 221874 -3 181362 910628 -4 105789 683792 -5 682747 524449 -6 914679 705071 -7 823257 647814 -8 929527 111187 -9 983450 801863 -10 235855 40939 -11 114943 748954 -12 21508 577430 -13 546242 592557 -14 856071 72019 -15 230455 722642 -16 731338 161603 -17 256737 171811 -18 202973 555125 -19 36952 293674 -20 889846 171763 -21 729520 167533 -22 351338 293944 -23 363527 237635 -24 994303 555034 -25 480754 745642 -26 611375 25987 -27 382517 46647 -28 547666 246651 -29 617707 151602 -30 18901 626236 -31 873675 812114 -32 793946 510984 -33 794928 747146 -34 353732 529313 -35 284287 691892 -36 116884 428416 -37 321108 490213 -38 429564 88956 -39 366436 974861 -40 29764 299591 -41 974849 704468 -42 53118 604219 -43 848967 919223 -44 367656 745753 -45 376882 455826 -46 201393 507639 -47 9387 197954 -48 54879 301104 -49 846424 861125 -50 864379 274571 -51 871198 19442 -52 525270 181163 -53 770792 906906 -54 972869 778298 -55 197679 628442 -56 878994 871953 -57 774720 563074 -58 424843 366035 -59 802727 595991 -60 456104 493824 -61 900722 492607 -62 664933 9715 -63 820693 97442 -64 903145 139945 -65 719420 522657 -66 116087 588138 -67 777182 401321 -68 420597 102896 -69 929748 490043 -70 179375 482932 -71 116496 771665 -72 289649 883058 -73 555104 708785 -74 842706 999672 -75 377261 957437 -76 397958 706478 -77 141704 341722 -78 158484 283060 -79 242259 123949 -80 760565 667896 -81 977158 482825 -82 598923 714747 -83 511945 107328 -84 582304 891661 -85 7970 716057 -86 523329 803055 -87 218730 498667 -88 95865 194243 -89 350902 323210 -90 851230 537632 -91 855182 779196 -92 379380 51524 -93 545498 633261 -94 989215 62435 -95 889375 313268 -96 520591 38086 -97 482073 463317 -98 959602 313441 -99 552934 790981 -100 787192 360860 -101 357883 519495 -102 148441 839628 -103 102254 618761 -104 327098 90180 -105 796224 525222 -106 293845 179824 -107 234573 447296 -108 147304 939072 -109 751 135605 -110 755144 198504 -111 554393 791323 -112 104469 647109 -113 358174 3834 -114 654613 379101 -115 396413 477104 -116 867145 260722 -117 526986 326008 -118 243786 675357 -119 544623 932083 -120 904219 606425 -121 632509 853609 -122 307290 690871 -123 758874 729268 -124 933616 834963 -125 105143 955768 -126 658827 173833 -127 394568 310088 -128 493715 97160 -129 992508 822433 -130 595842 426896 -131 74138 395014 -132 185960 189798 -133 892712 440236 -134 872533 543701 -135 688422 418036 -136 5455 165787 -137 30462 799375 -138 539676 380560 -139 396755 794380 -140 153394 261014 -141 11325 832180 -142 783259 969516 -143 402965 472130 -144 74761 337188 -145 433295 803550 -146 802823 921 -147 243661 486182 -148 600970 466721 -149 823147 507915 -150 151195 378314 -151 332513 139236 -152 681569 844129 -153 944443 826647 -154 390574 425051 -155 907122 21584 -156 22399 655319 -157 389138 792291 -158 624072 73216 -159 151353 699777 -160 588827 425990 -161 617088 364618 -162 392506 310107 -163 85523 866219 -164 484218 186333 -165 854932 713028 -166 989986 971703 -167 887258 131809 -168 238614 356005 -169 443042 990967 -170 345443 329749 -171 320777 374983 -172 748360 7304 -173 186461 438205 -174 608415 933553 -175 400659 734750 -176 982503 636814 -177 652983 438166 -178 388327 360809 -179 251978 549759 -180 605514 588437 -181 383604 399606 -182 79608 577373 -183 700806 647415 -184 906959 381833 -185 605829 185867 -186 464800 217799 -187 299118 854077 -188 443487 980274 -189 711634 954339 -190 921779 724713 -191 614241 934458 -192 580819 266494 -193 329424 590380 -194 892095 309884 -195 431002 591199 -196 449046 61208 -197 385137 159576 -198 864948 102978 -199 75865 894283 -200 563816 206186 -201 726570 654076 -202 11773 675946 -203 120509 48045 -204 55994 386489 -205 108742 797947 -206 468714 942093 -207 118757 14314 -208 139391 322396 -209 14468 920031 -210 712425 597828 -211 571550 12675 -212 818017 399643 -213 459296 810724 -214 206025 623171 -215 733568 395442 -216 924280 325145 -217 845596 123832 -218 255998 672148 -219 815701 566504 -220 127103 7027 -221 575911 972063 -222 597459 833173 -223 19649 436370 -224 243191 985494 -225 700280 54223 -226 896952 452693 -227 160714 168374 -228 281906 401424 -229 808479 887941 -230 419948 448360 -231 232343 489490 -232 259385 101714 -233 222035 496650 -234 344633 285583 -235 994664 703020 -236 79204 28973 -237 219751 658201 -238 700876 118856 -239 851960 649642 -240 117736 57871 -241 2244 318083 -242 203223 285207 -243 163098 434789 -244 65759 743881 -245 901796 759347 -246 327514 530118 -247 571839 424082 -248 927823 546938 -249 752312 939258 -250 132297 900793 -251 584410 593548 -252 867758 642408 -253 51978 578868 -254 249470 875507 -255 5275 847116 -256 335664 64598 -257 986145 660600 -258 120846 702225 -259 917650 835303 -260 173891 675097 -261 744338 405375 -262 153286 93871 -263 118610 485655 -264 161214 577343 -265 606222 122007 -266 869386 976453 -267 644367 270619 -268 722207 937646 -269 331937 542623 -270 164360 460873 -271 517139 230456 -272 569990 226698 -273 15009 922138 -274 376831 477968 -275 305472 442167 -276 385723 174969 -277 333035 10289 -278 31407 607956 -279 949181 597139 -280 920201 114331 -281 246930 706846 -282 711146 544402 -283 329977 105207 -284 494607 759447 -285 645590 198708 -286 325393 439682 -287 529830 731723 -288 289374 569368 -289 72340 142996 -290 62464 510653 -291 536473 564075 -292 657141 491891 -293 875076 162539 -294 265306 99964 -295 940642 616999 -296 443039 572490 -297 897032 965652 -298 135479 77457 -299 700626 838267 -300 937323 445640 -301 849798 233835 -302 415503 794818 -303 905694 821647 -304 517827 841144 -305 135212 868867 -306 342650 849217 -307 656497 303202 -308 671292 674440 -309 809813 745494 -310 408923 252519 -311 404581 656340 -312 822123 199949 -313 348909 91558 -314 24178 735011 -315 826029 467727 -316 51541 231195 -317 7783 193596 -318 168002 687256 -319 907578 353939 -320 820598 200636 -321 352725 519811 -322 691041 688122 -323 212417 786698 -324 750366 697083 -325 616742 43921 -326 53279 965614 -327 12237 469596 -328 394099 618602 -329 226051 221907 -330 626815 218437 -331 914069 163887 -332 20545 934576 -333 516142 822838 -334 158176 968374 -335 90784 884593 -336 924073 112730 -337 128752 365001 -338 199240 438967 -339 644102 352526 -340 805850 730307 -341 865506 802271 -342 108195 607591 -343 553658 887653 -344 210649 73207 -345 677454 345164 -346 529079 939203 -347 263154 936004 -348 276563 239994 -349 391059 326039 -350 488881 773449 -351 142595 397839 -352 891233 886435 -353 178414 251008 -354 857418 404646 -355 915938 506445 -356 407952 152843 -357 544453 281651 -358 689357 974865 -359 900732 84541 -360 658012 276147 -361 431779 832137 -362 479493 317335 -363 741998 526233 -364 221497 242317 -365 186587 948232 -366 581548 239456 -367 436588 299404 -368 322354 712662 -369 257818 826544 -370 918233 578792 -371 986920 126108 -372 415195 401306 -373 913385 696942 -374 459710 945819 -375 194006 750330 -376 18497 148742 -377 139452 540649 -378 191900 903139 -379 961251 591828 -380 564080 465752 -381 993189 30874 -382 486413 337146 -383 519524 281843 -384 737648 143146 -385 368266 992415 -386 515154 954912 -387 890534 907682 -388 257650 283036 -389 692685 938843 -390 125434 838858 -391 564981 629668 -392 678236 720834 -393 955042 550674 -394 753042 99442 -395 779879 40511 -396 975013 114671 -397 458277 925818 -398 63637 32008 -399 235573 507512 -400 143656 630348 -401 4257 520866 -402 820385 355148 -403 185348 388828 -404 470505 418617 -405 585606 641226 -406 150096 861809 -407 811949 523569 -408 490738 878518 -409 572596 560595 -410 273508 487516 -411 46269 230136 -412 999489 737917 -413 988157 994288 -414 134527 535386 -415 722334 868821 -416 812530 274068 -417 353237 484208 -418 688761 703172 -419 817719 154667 -420 230096 76524 -421 978077 192447 -422 825933 292362 -423 994241 744877 -424 115181 720359 -425 937660 69349 -426 897480 700187 -427 785178 274834 -428 817818 730189 -429 167629 336177 -430 666386 482176 -431 571109 315838 -432 188520 509082 -433 663148 957649 -434 35875 519587 -435 529327 745861 -436 763336 852236 -437 622934 204158 -438 590036 346082 -439 444958 724655 -440 920099 257968 -441 826658 798349 -442 869000 240157 -443 297711 496692 -444 85547 844154 -445 821059 731112 -446 667296 298132 -447 625340 484235 -448 313187 125840 -449 569513 621775 -450 702326 648158 -451 299918 390526 -452 800260 679692 -453 242691 99130 -454 831187 545020 -455 977122 321125 -456 644642 323475 -457 515118 935274 -458 814880 272977 -459 690497 704285 -460 195894 537307 -461 388136 414100 -462 817260 881169 -463 445942 372809 -464 51976 943242 -465 961467 490905 -466 514895 899937 -467 747532 598973 -468 613325 503183 -469 283231 933725 -470 425277 24734 -471 806194 381261 -472 648260 283751 -473 342975 253195 -474 480871 744171 -475 38292 940377 -476 73864 626271 -477 660307 211421 -478 133263 399980 -479 642994 201287 -480 66366 739508 -481 815899 897461 -482 119742 952388 -483 514931 263380 -484 675215 231366 -485 592298 561684 -486 792105 946326 -487 665993 255516 -488 463443 761865 -489 753793 605838 -490 747905 45961 -491 729815 850689 -492 876875 221959 -493 675005 617433 -494 780195 795143 -495 84042 349945 -496 827968 51864 -497 341427 863592 -498 232628 859867 -499 715268 392743 -500 820307 581110 -501 499402 875032 -502 996265 992330 -503 210562 223175 -504 749395 438348 -505 536415 515829 -506 619785 847850 -507 117244 716420 -508 911540 764034 -509 556034 256150 -510 719759 655063 -511 548111 282471 -512 411059 11187 -513 62282 917073 -514 950061 673662 -515 44954 240268 -516 12469 315444 -517 69423 232076 -518 426175 881965 -519 733444 160454 -520 310418 910971 -521 61399 524045 -522 75384 428978 -523 801834 545496 -524 640805 330239 -525 801310 315555 -526 909806 41606 -527 347789 580039 -528 568641 183957 -529 805609 764188 -530 566154 328597 -531 489730 588940 -532 127929 625444 -533 454429 95740 -534 772466 688266 -535 914586 366043 -536 123228 225795 -537 454839 404204 -538 745257 506505 -539 934682 831020 -540 442546 878324 -541 111463 185872 -542 107507 716356 -543 750537 423528 -544 187515 443979 -545 777646 330435 -546 109498 45178 -547 245868 944375 -548 787742 835603 -549 69205 671179 -550 683721 295329 -551 610814 809785 -552 887692 922985 -553 204091 723933 -554 934098 631432 -555 829501 145112 -556 996442 361630 -557 986542 235719 -558 219098 444552 -559 343072 183554 -560 837701 618825 -561 26535 101286 -562 4544 619257 -563 755229 313442 -564 338103 531854 -565 200112 21323 -566 572406 303249 -567 1518 297434 -568 881882 749833 -569 199330 871787 -570 497258 305985 -571 80456 3960 -572 606277 158820 -573 303899 8212 -574 40633 626610 -575 189145 474300 -576 497500 537350 -577 471066 662398 -578 722923 307564 -579 808266 590257 -580 41102 454258 -581 524603 62311 -582 134173 523516 -583 64655 992481 -584 755352 827721 -585 931819 210885 -586 403919 716461 -587 994408 363401 -588 121325 489184 -589 630220 342146 -590 896334 447665 -591 505176 747845 -592 490751 745854 -593 496719 510094 -594 169076 478002 -595 232778 889401 -596 994480 371608 -597 939967 286372 -598 902065 363994 -599 9552 242876 -600 37495 814714 -601 377992 698487 -602 730276 741480 -603 969123 749655 -604 46598 725211 -605 165678 212828 -606 138487 330263 -607 357478 151701 -608 459777 152994 -609 122343 847800 -610 621450 700661 -611 982929 512476 -612 790225 117105 -613 832893 705431 -614 986185 252928 -615 394277 371669 -616 442586 905008 -617 176467 683506 -618 309177 174912 -619 390366 339050 -620 286077 343725 -621 387751 321275 -622 856552 532117 -623 906471 482004 -624 581383 822862 -625 453479 196633 -626 377809 756623 -627 848599 665825 -628 372128 472984 -629 522019 46770 -630 432302 794211 -631 359288 707548 -632 439134 821953 -633 825076 817212 -634 473711 825361 -635 245229 977772 -636 571611 299480 -637 394321 424816 -638 322852 226305 -639 663876 124399 -640 744976 359909 -641 183412 939415 -642 820626 600065 -643 12381 735038 -644 465874 354514 -645 858429 491965 -646 701201 565005 -647 93820 308305 -648 772114 88270 -649 926953 431736 -650 209265 680166 -651 818127 456983 -652 77885 93569 -653 13221 438394 -654 935997 248533 -655 653444 637090 -656 7109 167505 -657 188340 940336 -658 93010 794282 -659 613727 130828 -660 49838 736805 -661 890258 41975 -662 616096 565062 -663 159644 114627 -664 221595 300751 -665 411594 884458 -666 290308 415343 -667 470955 107886 -668 352799 15906 -669 751074 880289 -670 507055 218098 -671 121311 335045 -672 304675 121624 -673 601707 659225 -674 948909 528757 -675 148660 657486 -676 866675 626201 -677 20141 324806 -678 389858 402784 -679 986027 969998 -680 740769 997314 -681 687319 55707 -682 741478 435345 -683 515778 672064 -684 862829 66716 -685 338629 433784 -686 845373 84969 -687 982168 392352 -688 870129 264056 -689 21833 291289 -690 175204 756860 -691 128599 251596 -692 559982 414279 -693 197138 234600 -694 673865 35609 -695 592107 680734 -696 153076 684358 -697 541660 73270 -698 372725 36341 -699 352876 912322 -700 251494 337650 -701 637391 657620 -702 353553 391800 -703 528886 615079 -704 66219 605861 -705 127668 155257 -706 728918 950418 -707 377890 60034 -708 844238 2960 -709 514047 668207 -710 62620 479436 -711 319188 950507 -712 815221 978 -713 796392 187753 -714 731415 590368 -715 863466 255050 -716 197836 415972 -717 163621 19946 -718 27941 178181 -719 873705 499947 -720 570041 194178 -721 720553 5658 -722 972988 112670 -723 361546 202569 -724 869137 540681 -725 276877 184972 -726 304926 762508 -727 48856 996444 -728 139814 221419 -729 493999 333606 -730 363858 350704 -731 741374 566272 -732 35819 933489 -733 434704 723260 -734 977429 265219 -735 698487 641668 -736 133823 973365 -737 391329 877648 -738 174510 556679 -739 901650 818777 -740 861163 574973 -741 693754 397809 -742 226509 512761 -743 513675 631563 -744 380152 230131 -745 585241 304680 -746 200751 608485 -747 801459 928373 -748 60354 747187 -749 614328 95340 -750 938160 804867 -751 300919 50359 -752 679518 701904 -753 491217 907116 -754 535998 536094 -755 482768 508250 -756 841266 263867 -757 748364 59177 -758 149952 132889 -759 764813 107446 -760 873134 687517 -761 108931 882089 -762 327059 893620 -763 340749 83463 -764 293847 689424 -765 386430 267393 -766 20681 365555 -767 336008 352912 -768 733706 429886 -769 649444 167332 -770 362808 380786 -771 866749 272706 -772 356996 897724 -773 195748 318662 -774 281425 907838 -775 587623 976891 -776 453340 924903 -777 708909 670766 -778 784185 935363 -779 714351 326605 -780 968197 61331 -781 257672 368666 -782 173286 101982 -783 641501 568560 -784 906871 850640 -785 863428 831290 -786 851463 856975 -787 519822 896243 -788 234177 184028 -789 173179 656293 -790 109434 405385 -791 369112 967242 -792 721227 325099 -793 9721 652015 -794 192269 234026 -795 711411 165146 -796 523014 798803 -797 303903 531518 -798 529322 9774 -799 752883 460752 -800 663547 11720 -801 145482 625132 -802 798403 182238 -803 607778 486097 -804 203676 383810 -805 661045 132170 -806 743093 480324 -807 615193 803051 -808 538316 458869 -809 64762 641767 -810 572659 631726 -811 815676 446118 -812 187093 851708 -813 685807 226331 -814 58572 337584 -815 288464 748079 -816 980351 789369 -817 995248 977264 -818 662292 888787 -819 352048 918175 -820 786782 550852 -821 587252 550502 -822 661367 79685 -823 349469 76896 -824 611710 452195 -825 845711 302991 -826 951202 415299 -827 172287 915467 -828 31369 356113 -829 629883 821139 -830 519945 718991 -831 134048 285500 -832 597028 110193 -833 544918 192591 -834 818957 535508 -835 453581 461420 -836 847158 612567 -837 796056 269668 -838 680524 400377 -839 273831 271625 -840 820338 329693 -841 596447 237432 -842 817638 569473 -843 614030 694850 -844 192340 885054 -845 432345 469700 -846 69830 816540 -847 464594 325361 -848 703693 974684 -849 754445 391699 -850 399160 949092 -851 803064 340084 -852 631856 516017 -853 706543 713769 -854 597660 602813 -855 301436 336518 -856 163773 744828 -857 388793 50245 -858 649160 317508 -859 820906 271666 -860 406500 570234 -861 438145 427257 -862 136347 504489 -863 658355 507074 -864 980711 280038 -865 563124 966755 -866 802716 671017 -867 970188 483820 -868 165920 851619 -869 848639 767393 -870 920313 296522 -871 873944 920674 -872 478554 862111 -873 31554 642572 -874 680192 960105 -875 667005 196618 -876 993972 474407 -877 828574 432405 -878 146375 132047 -879 369895 148035 -880 350097 854924 -881 865129 830266 -882 682500 4914 -883 462573 243098 -884 266273 526682 -885 18690 6587 -886 637315 860801 -887 604660 209881 -888 576261 963738 -889 598683 703942 -890 358113 526308 -891 137179 832676 -892 929940 708200 -893 101625 972449 -894 988516 965274 -895 21246 922821 -896 585345 321957 -897 748703 913725 -898 659207 13142 -899 316013 268673 -900 285849 67815 -901 43889 976250 -902 601991 140697 -903 59438 161296 -904 544466 120374 -905 330779 173925 -906 143530 404620 -907 126788 427276 -908 269579 543172 -909 81563 768775 -910 345707 449431 -911 927084 997600 -912 240832 950874 -913 962698 661065 -914 258809 463963 -915 150442 414964 -916 419272 478309 -917 373163 184187 -918 382777 732558 -919 705887 502664 -920 438621 558452 -921 890886 219741 -922 619567 571815 -923 995736 587744 -924 81124 797828 -925 951026 998142 -926 754333 852050 -927 118230 870885 -928 648543 565579 -929 603086 417804 -930 757919 326880 -931 455409 41801 -932 681752 772326 -933 283039 923789 -934 785053 554973 -935 431539 681834 -936 462047 283734 -937 817748 347564 -938 695097 75034 -939 879370 369947 -940 302331 397118 -941 57978 841005 -942 706043 823562 -943 959555 377470 -944 796556 600836 -945 901148 458988 -946 947505 150913 -947 71124 756786 -948 96404 607151 -949 401992 272002 -950 876718 920702 -951 708374 711177 -952 495497 553907 -953 940163 913327 -954 146006 294668 -955 911330 271072 -956 769023 2250 -957 516655 298930 -958 379374 304495 -959 970676 924966 -960 675921 675887 -961 521796 513050 -962 678255 510837 -963 973460 750870 -964 788236 263840 -965 407401 781769 -966 929027 584701 -967 458617 31258 -968 628095 55728 -969 324349 407113 -970 444188 655059 -971 406793 871590 -972 924914 225260 -973 937192 434455 -974 472658 560287 -975 783326 345534 -976 818915 138152 -977 864601 94948 -978 991675 123672 -979 252560 464238 -980 925812 884435 -981 588978 738893 -982 850480 256270 -983 864278 897432 -984 77335 291394 -985 361737 944919 -986 831837 410389 -987 141639 330387 -988 856972 383644 -989 648449 583307 -990 519161 849788 -991 498310 323998 -992 338028 522966 -993 192791 190133 -994 734221 202346 -995 166980 730662 -996 978393 32954 -997 45375 499268 -998 823222 996404 -999 729950 594527 -1000 368288 553548 diff --git a/devel-examples/vignette-example/instances/1000-7.tsp b/devel-examples/vignette-example/instances/1000-7.tsp deleted file mode 100644 index 2efb9769..00000000 --- a/devel-examples/vignette-example/instances/1000-7.tsp +++ /dev/null @@ -1,1006 +0,0 @@ -NAME : portgen-1000-7 -COMMENT : portgen N=1000, seed=7 -TYPE : TSP -DIMENSION : 1000 -EDGE_WEIGHT_TYPE : EUC_2D -NODE_COORD_SECTION -1 505570 199321 -2 712595 87039 -3 983680 849077 -4 751405 457873 -5 594628 349334 -6 891497 362398 -7 965118 935274 -8 330561 339334 -9 141292 557585 -10 273960 535135 -11 85008 457451 -12 983406 428979 -13 370805 394832 -14 984843 682033 -15 729748 728940 -16 224744 192477 -17 956941 136569 -18 483776 208964 -19 4230 510573 -20 911868 418592 -21 942015 450840 -22 229890 519640 -23 162551 872893 -24 7576 995218 -25 181648 530835 -26 407082 348607 -27 918367 92442 -28 626364 313092 -29 242379 576025 -30 603263 774715 -31 844846 240832 -32 546005 176036 -33 407318 440657 -34 132507 445478 -35 772722 457668 -36 331758 146073 -37 375937 743124 -38 128052 736400 -39 539083 890963 -40 802614 57713 -41 152452 408817 -42 78770 955033 -43 884093 983912 -44 646472 780904 -45 729251 43118 -46 76457 558752 -47 737850 454200 -48 86834 795941 -49 74902 486766 -50 391587 426150 -51 333810 116612 -52 192603 123934 -53 378383 998265 -54 269837 963334 -55 208349 642452 -56 666620 461475 -57 846774 560144 -58 698258 286094 -59 502981 91805 -60 89202 611376 -61 365754 645741 -62 53891 346571 -63 123857 215145 -64 953470 252002 -65 364740 129787 -66 466563 575749 -67 682614 160161 -68 391093 690977 -69 562042 518625 -70 256775 597998 -71 480930 554666 -72 691702 117874 -73 677364 430716 -74 504861 391278 -75 330342 871688 -76 842471 822900 -77 122025 261799 -78 959587 758060 -79 433998 32441 -80 732841 687406 -81 436222 751211 -82 706559 610350 -83 161522 111953 -84 769772 728900 -85 882780 267541 -86 781233 111703 -87 761463 217513 -88 768904 542854 -89 523715 792091 -90 386983 365796 -91 781146 921028 -92 519161 677334 -93 693565 715351 -94 869189 72263 -95 998639 279140 -96 921204 833142 -97 635845 989233 -98 816765 369227 -99 793203 474189 -100 348969 134509 -101 907000 712769 -102 4294 964545 -103 90542 921442 -104 303739 444690 -105 568234 244236 -106 888871 361735 -107 33801 453701 -108 766201 603079 -109 115366 717326 -110 793584 792294 -111 318749 295583 -112 379930 748270 -113 360541 68464 -114 107408 796917 -115 126970 847461 -116 239114 79025 -117 223857 142747 -118 476925 419411 -119 887226 65460 -120 911133 90485 -121 599984 151863 -122 278678 206345 -123 960390 625621 -124 453212 887574 -125 628692 748301 -126 261819 996285 -127 347218 501507 -128 895395 827975 -129 488911 861547 -130 487619 671131 -131 34216 238279 -132 533557 477748 -133 644252 737007 -134 83056 827456 -135 493310 140579 -136 149867 227792 -137 88634 45283 -138 530475 322464 -139 948364 878422 -140 852875 532565 -141 579552 245860 -142 309298 455839 -143 813245 835 -144 545467 746109 -145 498494 739918 -146 336354 59770 -147 572149 770553 -148 940618 372192 -149 63229 233395 -150 675869 637925 -151 677256 574789 -152 34699 96126 -153 168748 15958 -154 686987 891379 -155 688261 894559 -156 282507 742802 -157 363053 747701 -158 334776 974445 -159 666129 763003 -160 537130 272060 -161 673778 849661 -162 151586 855385 -163 463323 575078 -164 193092 992507 -165 876534 514517 -166 635477 56985 -167 190161 958316 -168 250058 836749 -169 882807 561597 -170 121062 838800 -171 334706 782464 -172 208978 226434 -173 66140 486693 -174 908183 716763 -175 307230 365539 -176 179099 70721 -177 356860 161352 -178 2448 620270 -179 384627 76382 -180 846068 331999 -181 133150 125389 -182 770316 849461 -183 559853 500043 -184 533823 136619 -185 681561 848083 -186 66261 949365 -187 455773 171590 -188 92960 603056 -189 492801 990234 -190 852937 843052 -191 190450 116709 -192 146439 544535 -193 381366 510087 -194 286669 340700 -195 398463 750014 -196 302926 746188 -197 880035 272978 -198 772538 385340 -199 326690 37388 -200 133473 463083 -201 993891 917949 -202 863826 464177 -203 175088 62389 -204 924281 812324 -205 779986 492360 -206 333601 43927 -207 677919 96053 -208 29072 386962 -209 245353 497337 -210 76922 174512 -211 173352 496435 -212 3145 218477 -213 854191 148312 -214 85538 991595 -215 996502 30571 -216 678774 680476 -217 210247 360576 -218 509450 146523 -219 438790 50386 -220 515462 994404 -221 264733 789331 -222 263777 223950 -223 576661 806490 -224 743042 661558 -225 418787 624226 -226 299801 335094 -227 40886 102901 -228 784308 313415 -229 707701 503250 -230 954726 28565 -231 623599 873895 -232 296861 785582 -233 227626 544270 -234 780150 453969 -235 519391 222582 -236 643919 583795 -237 78550 452695 -238 874711 838257 -239 455549 900244 -240 434168 540776 -241 440610 582288 -242 36868 967936 -243 406972 804879 -244 383614 424665 -245 132949 965180 -246 366372 984821 -247 530994 292880 -248 350484 680938 -249 710781 811081 -250 349239 738404 -251 350941 842798 -252 227389 878011 -253 183616 717513 -254 298226 72949 -255 695929 979429 -256 929800 283035 -257 370300 989546 -258 662192 638778 -259 342900 3981 -260 435097 546688 -261 833488 969068 -262 104729 780987 -263 871641 801121 -264 356405 200539 -265 269079 157197 -266 540031 382599 -267 204314 454739 -268 610975 157574 -269 211987 47322 -270 305743 768194 -271 461978 379633 -272 989567 586260 -273 131691 397304 -274 880091 750007 -275 421239 549363 -276 324532 510242 -277 542002 192041 -278 198372 968341 -279 638483 772650 -280 267035 26041 -281 505525 250903 -282 767206 927735 -283 517450 550166 -284 293467 784040 -285 857854 264888 -286 758686 592892 -287 582742 885734 -288 222156 323246 -289 427066 912688 -290 582614 903300 -291 162637 583755 -292 933503 243037 -293 651672 289127 -294 615393 276579 -295 937288 60809 -296 864106 427947 -297 189468 40854 -298 9507 869086 -299 796891 103833 -300 364104 808445 -301 970237 967403 -302 167393 517938 -303 386725 740776 -304 576738 298964 -305 540369 909244 -306 352947 361903 -307 835361 206226 -308 161935 77578 -309 61435 726351 -310 918228 648364 -311 753275 189634 -312 419935 49409 -313 294650 791283 -314 425499 64803 -315 499009 481380 -316 746508 128102 -317 372319 673369 -318 550352 800734 -319 748393 727277 -320 81102 574094 -321 227692 889042 -322 358351 288924 -323 307533 674471 -324 8011 140058 -325 746203 218223 -326 443586 732087 -327 604823 882723 -328 61936 842135 -329 595084 494023 -330 967586 585990 -331 992382 849460 -332 217861 966274 -333 681552 463905 -334 3552 546437 -335 898692 487463 -336 69566 921377 -337 980147 700004 -338 204777 21187 -339 584810 537211 -340 987472 452515 -341 196199 931475 -342 97217 913018 -343 488997 897047 -344 910240 406044 -345 991817 86447 -346 797181 201956 -347 828584 593639 -348 504527 306315 -349 908894 658346 -350 84146 286345 -351 89660 470799 -352 152585 293688 -353 22023 512111 -354 634870 691805 -355 393726 164889 -356 931894 189039 -357 502206 881139 -358 788809 790426 -359 20875 624222 -360 461746 375236 -361 555010 345205 -362 462290 612347 -363 397802 598766 -364 768791 686458 -365 677981 692666 -366 386317 893005 -367 143485 822583 -368 520620 7159 -369 429269 216078 -370 124209 698571 -371 876171 286018 -372 944297 616580 -373 531436 451976 -374 739666 216237 -375 195836 905760 -376 537523 222435 -377 980365 391480 -378 900028 196655 -379 327314 330001 -380 773067 14863 -381 82841 418792 -382 567595 695154 -383 288717 645876 -384 244742 885625 -385 349702 336832 -386 50759 804637 -387 428385 555986 -388 837712 332575 -389 364840 70809 -390 712318 201147 -391 271452 438789 -392 913391 663117 -393 609824 967525 -394 325410 448330 -395 533866 874744 -396 762417 543644 -397 866375 787376 -398 647811 71534 -399 857632 388311 -400 778867 198861 -401 87135 668856 -402 503919 994689 -403 634307 98734 -404 309044 317247 -405 781655 932503 -406 871244 878983 -407 796135 898323 -408 252446 539197 -409 552416 780218 -410 47343 217183 -411 788244 856431 -412 106758 150841 -413 249697 381903 -414 300718 433696 -415 921678 738978 -416 23530 47592 -417 289154 779815 -418 329902 392469 -419 642654 15067 -420 410671 70627 -421 415108 545191 -422 400987 316683 -423 86499 905986 -424 436885 715534 -425 537679 265907 -426 770815 423935 -427 466632 39889 -428 175330 39543 -429 379702 724103 -430 664786 241838 -431 456080 293976 -432 906576 711028 -433 517394 326053 -434 477995 479452 -435 811824 346460 -436 102311 836882 -437 242539 781435 -438 446367 364308 -439 389798 66869 -440 975510 210154 -441 2201 576615 -442 768910 679840 -443 282898 729553 -444 141015 578125 -445 262420 3849 -446 914473 163201 -447 203243 64210 -448 968315 578225 -449 302651 619551 -450 870315 722190 -451 516187 370016 -452 740023 327525 -453 263706 194200 -454 655025 786792 -455 756990 445777 -456 898527 801576 -457 461682 660937 -458 327364 292878 -459 90733 842366 -460 742712 939169 -461 23401 858443 -462 609136 89633 -463 830272 732294 -464 96858 915014 -465 517729 252167 -466 709283 603005 -467 309878 529733 -468 311626 200625 -469 114932 107973 -470 352475 990019 -471 638819 298649 -472 835413 323251 -473 980447 56029 -474 554065 113609 -475 233938 236020 -476 481366 387637 -477 101822 618148 -478 12906 913182 -479 60138 210290 -480 15898 63080 -481 79268 547052 -482 434317 766971 -483 806957 599878 -484 966163 138431 -485 680489 271334 -486 738813 977123 -487 608427 506691 -488 457802 635764 -489 756621 990988 -490 76726 917089 -491 672156 886568 -492 899115 454648 -493 172899 162231 -494 168688 542906 -495 722775 711748 -496 234462 976501 -497 427483 81140 -498 251206 661696 -499 690222 328721 -500 865449 344683 -501 299408 563076 -502 36883 316849 -503 563864 594798 -504 488521 647173 -505 445248 850675 -506 744494 517231 -507 487514 304150 -508 828618 102767 -509 119568 353176 -510 515765 145261 -511 909655 637441 -512 272981 335805 -513 971926 175736 -514 940240 291577 -515 942827 863003 -516 147242 109448 -517 545739 226051 -518 172595 154924 -519 399053 594965 -520 626030 70132 -521 42663 815512 -522 27141 577514 -523 802092 597020 -524 703519 91677 -525 109214 75469 -526 721456 398644 -527 385894 2445 -528 197440 189960 -529 17336 810832 -530 144253 408940 -531 195745 893555 -532 21142 375116 -533 808011 928981 -534 490090 910000 -535 502057 231598 -536 399247 27890 -537 243961 440296 -538 423804 511011 -539 251547 270536 -540 138365 781966 -541 158400 129407 -542 147324 533886 -543 667258 253686 -544 88305 170623 -545 418039 243613 -546 664833 489053 -547 92907 394432 -548 670884 14773 -549 571550 586844 -550 153710 291081 -551 345472 432983 -552 953312 327248 -553 917069 592048 -554 251320 852007 -555 335187 943753 -556 101655 846713 -557 392793 900640 -558 744106 706692 -559 800648 626710 -560 704232 793238 -561 357431 903246 -562 756290 210976 -563 886125 966264 -564 74578 916713 -565 523227 831756 -566 259690 399540 -567 935348 194612 -568 680311 311686 -569 736614 246683 -570 789779 960566 -571 453038 461594 -572 466390 624800 -573 886182 761587 -574 732763 881931 -575 508306 704619 -576 940194 654836 -577 63617 321953 -578 31390 945932 -579 497634 758700 -580 646936 605382 -581 855434 4636 -582 62227 374621 -583 490714 640060 -584 380323 767992 -585 14458 982519 -586 973928 918717 -587 118404 999612 -588 853043 702594 -589 839628 434336 -590 179585 940193 -591 468630 315877 -592 269776 917844 -593 976321 255054 -594 337312 560726 -595 703897 40250 -596 931363 968621 -597 232225 807260 -598 986637 534321 -599 343190 466777 -600 771756 183587 -601 921958 298427 -602 702345 568112 -603 235989 624317 -604 385059 145772 -605 345632 776336 -606 608619 936907 -607 54803 606686 -608 674019 886812 -609 772410 254967 -610 387984 956393 -611 296870 913545 -612 996236 830870 -613 60561 675501 -614 216372 550291 -615 763623 228726 -616 317534 693856 -617 88703 403249 -618 331574 531722 -619 261074 663090 -620 243825 89508 -621 482643 82345 -622 172742 747503 -623 743379 17818 -624 972384 401355 -625 746699 311135 -626 317949 792898 -627 703154 543029 -628 866052 228102 -629 209723 299096 -630 236538 704267 -631 363242 721969 -632 901947 256123 -633 293693 526273 -634 764165 307299 -635 863306 656201 -636 914428 371055 -637 508268 76848 -638 638444 503972 -639 210391 453207 -640 964817 832458 -641 465778 917275 -642 313753 59356 -643 865484 595565 -644 791908 832580 -645 109555 805300 -646 767557 953774 -647 799784 587624 -648 175080 111587 -649 574077 95893 -650 109059 239407 -651 807426 519177 -652 436537 914240 -653 845357 400673 -654 479144 643797 -655 677545 270486 -656 436193 377143 -657 189540 431237 -658 936710 409467 -659 922184 314322 -660 81042 182106 -661 952196 668271 -662 198240 623898 -663 848774 395250 -664 934518 594027 -665 231491 237770 -666 24827 566593 -667 775661 694331 -668 396265 88635 -669 727735 882516 -670 122646 456016 -671 673381 477585 -672 751537 927449 -673 853104 99285 -674 755534 175886 -675 738849 779830 -676 177069 980049 -677 864401 871288 -678 214580 240833 -679 743515 742206 -680 517975 756721 -681 672937 596628 -682 521151 221528 -683 597105 958607 -684 625605 262090 -685 578132 837424 -686 653933 746298 -687 575472 301212 -688 5036 972146 -689 803869 326952 -690 409318 607941 -691 651734 192312 -692 76052 474769 -693 564833 428198 -694 45441 554133 -695 97225 437657 -696 463029 465644 -697 304383 285221 -698 802083 927082 -699 902112 450325 -700 922412 880957 -701 295415 428582 -702 766568 130908 -703 128095 984756 -704 903996 389631 -705 306455 786381 -706 195391 189382 -707 644980 80317 -708 293692 207293 -709 292245 235930 -710 419445 670022 -711 56494 175280 -712 339677 697174 -713 542008 225351 -714 979730 444564 -715 173116 20279 -716 68149 414237 -717 20497 622936 -718 412549 462352 -719 547331 995734 -720 181077 357539 -721 135953 809511 -722 134688 427202 -723 381162 287749 -724 125966 607208 -725 743212 576732 -726 947352 457548 -727 277208 902132 -728 812808 881177 -729 408084 143631 -730 718359 665743 -731 437425 908261 -732 208553 948915 -733 650428 385879 -734 54694 217778 -735 699154 5942 -736 81326 685036 -737 492717 842713 -738 722670 598945 -739 898071 437545 -740 884366 660831 -741 817266 836098 -742 726204 507372 -743 582854 159887 -744 205683 71581 -745 972507 26669 -746 407658 329553 -747 296580 175135 -748 276213 450916 -749 316794 291975 -750 704532 782217 -751 389678 688421 -752 722842 82381 -753 759465 111253 -754 731344 769835 -755 319278 652920 -756 675493 336503 -757 171124 691689 -758 258085 107871 -759 611681 33418 -760 672701 199511 -761 69085 762719 -762 513245 916937 -763 616263 392904 -764 962193 410336 -765 83247 611416 -766 867601 128235 -767 118266 231446 -768 985337 480763 -769 664973 34515 -770 249287 474982 -771 548206 172265 -772 398880 772995 -773 957583 644939 -774 816307 379642 -775 558872 883308 -776 488722 906457 -777 208727 93115 -778 914615 261443 -779 570154 491396 -780 97044 278702 -781 446280 696829 -782 520548 844295 -783 104714 503228 -784 937623 398129 -785 734105 613146 -786 291564 232039 -787 474546 789392 -788 710789 162628 -789 553991 420130 -790 2322 354820 -791 822749 470797 -792 313292 804544 -793 165136 170772 -794 607686 273971 -795 126731 482109 -796 543139 266844 -797 300409 636140 -798 183418 316167 -799 697719 609487 -800 62206 794955 -801 90947 396177 -802 377320 204051 -803 60559 17925 -804 593165 404182 -805 927979 743842 -806 653756 296183 -807 364664 614934 -808 735562 179435 -809 396420 884407 -810 660877 788547 -811 805508 328135 -812 335922 939150 -813 522198 895387 -814 854718 270494 -815 728833 692863 -816 569462 149808 -817 492151 258479 -818 701063 526566 -819 106132 698357 -820 68982 985700 -821 774352 723279 -822 613093 338184 -823 676600 215004 -824 930921 361258 -825 113942 288030 -826 461448 427224 -827 880653 369342 -828 225493 941139 -829 904026 118841 -830 502987 533992 -831 911792 894808 -832 335200 942278 -833 969490 930477 -834 683089 26480 -835 938333 520558 -836 248513 35161 -837 770465 372846 -838 327098 378283 -839 447482 966580 -840 713657 581059 -841 991361 735876 -842 767506 194840 -843 781071 674654 -844 814607 549872 -845 288988 770586 -846 843476 79652 -847 760023 548423 -848 737186 739191 -849 952813 240247 -850 11085 298317 -851 767521 964341 -852 647601 532882 -853 296669 725571 -854 659717 685813 -855 588271 550838 -856 126531 354153 -857 829853 732401 -858 690516 832139 -859 134784 786776 -860 205091 230299 -861 977663 442841 -862 15394 640016 -863 753036 284172 -864 387560 237582 -865 76176 601526 -866 718565 761669 -867 378309 162818 -868 454528 637207 -869 906023 35105 -870 504324 948931 -871 539869 27830 -872 344780 58689 -873 792922 400634 -874 64258 120006 -875 795387 453014 -876 351631 715230 -877 164071 409559 -878 579751 5852 -879 586032 484782 -880 78354 659462 -881 819548 624612 -882 181488 639339 -883 10968 98700 -884 9372 771163 -885 939478 289881 -886 767881 14777 -887 991389 752077 -888 878668 262433 -889 278770 605835 -890 60265 747183 -891 698140 902777 -892 159228 416713 -893 781978 93953 -894 580180 738969 -895 151850 355828 -896 627834 134859 -897 95626 214443 -898 181049 525092 -899 36441 592703 -900 180021 530489 -901 121863 458422 -902 59741 48203 -903 754874 448853 -904 556002 747357 -905 627580 485798 -906 425672 847063 -907 332931 722526 -908 31628 684688 -909 528985 967045 -910 458289 485876 -911 62259 416669 -912 591142 408989 -913 168017 309458 -914 955035 943186 -915 997203 429814 -916 706431 531413 -917 978254 574466 -918 321511 851077 -919 569845 436702 -920 385085 97290 -921 564967 613135 -922 280679 665974 -923 293568 211165 -924 543717 686637 -925 46425 871591 -926 570056 93254 -927 595500 750206 -928 824058 590450 -929 480168 485275 -930 726692 903797 -931 879007 119299 -932 362271 530290 -933 920830 812537 -934 566383 666957 -935 428957 820463 -936 140971 842348 -937 920619 586698 -938 915819 969004 -939 821168 840936 -940 584931 577567 -941 829290 469760 -942 216494 93405 -943 550807 587131 -944 169141 447963 -945 653636 508973 -946 284693 902888 -947 7744 564622 -948 956319 722619 -949 692515 693981 -950 750154 324564 -951 389996 702780 -952 101705 468858 -953 42301 100295 -954 876760 502094 -955 199398 236926 -956 421308 32205 -957 831638 217718 -958 619103 976119 -959 111554 797648 -960 573971 198210 -961 120022 872402 -962 916803 104393 -963 430467 438190 -964 740642 451761 -965 544397 815523 -966 92244 319074 -967 641537 348004 -968 156258 797085 -969 638122 998776 -970 474302 574688 -971 475576 371493 -972 873991 455425 -973 388951 412291 -974 986084 903351 -975 134155 518128 -976 981977 240753 -977 149584 934630 -978 232319 70921 -979 61242 753701 -980 312599 245216 -981 462173 877984 -982 27792 624710 -983 761349 49815 -984 158213 376212 -985 828766 206811 -986 990034 208203 -987 663493 55842 -988 216232 879268 -989 722817 982172 -990 872073 359545 -991 376947 986941 -992 139161 299180 -993 353350 214260 -994 291282 16827 -995 586655 106443 -996 638871 261909 -997 170009 267490 -998 584653 267373 -999 707999 818148 -1000 239193 509682 diff --git a/devel-examples/vignette-example/instances/1000-8.tsp b/devel-examples/vignette-example/instances/1000-8.tsp deleted file mode 100644 index 21fc1933..00000000 --- a/devel-examples/vignette-example/instances/1000-8.tsp +++ /dev/null @@ -1,1006 +0,0 @@ -NAME : portgen-1000-8 -COMMENT : portgen N=1000, seed=8 -TYPE : TSP -DIMENSION : 1000 -EDGE_WEIGHT_TYPE : EUC_2D -NODE_COORD_SECTION -1 602886 524197 -2 101777 952204 -3 785998 787527 -4 397021 231954 -5 506509 174219 -6 868316 19725 -7 106980 222733 -8 731596 567481 -9 299133 313307 -10 312064 29332 -11 55073 165948 -12 945304 280528 -13 195368 197107 -14 113615 292047 -15 229041 735237 -16 718150 223352 -17 657144 101328 -18 764579 862804 -19 971508 727471 -20 933889 665420 -21 154510 734147 -22 108443 745336 -23 961575 508152 -24 20849 435401 -25 882541 316029 -26 202790 671228 -27 454217 138238 -28 705063 379533 -29 867052 448 -30 187625 923194 -31 816018 669549 -32 298065 841088 -33 19708 134168 -34 911282 361643 -35 261158 223444 -36 546631 863731 -37 430765 996035 -38 826541 383845 -39 711730 807065 -40 575465 815835 -41 330055 113166 -42 104422 305847 -43 919219 48600 -44 925287 816055 -45 81619 630410 -46 951521 609865 -47 466313 710445 -48 118788 290779 -49 303381 112407 -50 918794 577730 -51 796421 213783 -52 859936 66706 -53 985974 89623 -54 566806 148370 -55 219019 656462 -56 454245 50996 -57 918829 557214 -58 971672 206153 -59 203236 587619 -60 722299 728928 -61 830787 798875 -62 442849 683428 -63 427022 332847 -64 3794 364059 -65 10061 736918 -66 817038 563360 -67 588046 919002 -68 361589 279058 -69 194337 547207 -70 334174 713065 -71 845364 337668 -72 93756 352690 -73 799623 152646 -74 167015 782885 -75 283423 785940 -76 286984 939322 -77 102346 181876 -78 760691 233061 -79 625737 940934 -80 705116 706915 -81 895285 19598 -82 814195 505953 -83 811098 116577 -84 957240 566138 -85 757590 819025 -86 39137 420351 -87 304196 936359 -88 441944 891464 -89 696528 260972 -90 922736 193961 -91 707110 62860 -92 658942 303145 -93 841632 797440 -94 749164 82092 -95 107904 245011 -96 321817 628199 -97 789616 515148 -98 673928 425013 -99 33471 157397 -100 910746 908159 -101 456117 906043 -102 860148 89461 -103 78830 224124 -104 280380 799201 -105 340244 963250 -106 483896 543645 -107 833030 460105 -108 385097 267086 -109 229981 299046 -110 832025 386085 -111 83105 799843 -112 655391 849431 -113 362907 133094 -114 560202 214734 -115 857528 217819 -116 611084 897327 -117 920728 959486 -118 710064 163464 -119 229830 198836 -120 918047 574545 -121 567458 450117 -122 250066 721818 -123 161905 521974 -124 972807 940185 -125 152240 540833 -126 864810 818737 -127 299868 692926 -128 297074 558790 -129 985314 900662 -130 379396 915366 -131 994293 81543 -132 881153 765698 -133 395792 33778 -134 293579 111211 -135 298199 863123 -136 294279 289796 -137 146805 291191 -138 521274 264368 -139 499974 962465 -140 552356 804117 -141 147779 659540 -142 835337 942162 -143 223526 529541 -144 16174 155029 -145 563693 676285 -146 869885 118619 -147 900637 54924 -148 280266 277662 -149 303311 958874 -150 200543 897537 -151 21999 10341 -152 387828 348123 -153 393054 205269 -154 983399 357706 -155 469400 767533 -156 542616 830285 -157 336968 703111 -158 45480 875674 -159 180906 826229 -160 485432 118129 -161 730467 334704 -162 910667 400662 -163 841123 283937 -164 901967 798682 -165 898137 316005 -166 280968 142268 -167 493064 784822 -168 261501 317494 -169 322572 132226 -170 896681 347851 -171 348634 189944 -172 669596 445564 -173 945818 535180 -174 207952 499974 -175 213800 996328 -176 375695 504629 -177 60737 884538 -178 616568 879731 -179 517277 603006 -180 86622 75560 -181 882696 851173 -182 461025 121549 -183 418899 352671 -184 160688 891404 -185 757292 510299 -186 667722 680932 -187 612428 489104 -188 742434 225838 -189 273967 26128 -190 784094 961391 -191 766660 298961 -192 712060 822577 -193 433308 429795 -194 681242 371515 -195 365923 908829 -196 156805 431168 -197 374934 386381 -198 680129 667702 -199 577516 180492 -200 703129 719980 -201 261213 181823 -202 715879 252408 -203 229668 76733 -204 792568 238160 -205 451229 186773 -206 198488 145762 -207 237082 177792 -208 918754 451528 -209 476239 74643 -210 441419 751196 -211 775155 980195 -212 188274 37312 -213 249086 485899 -214 965052 360019 -215 259435 665700 -216 433269 35807 -217 574899 597320 -218 762903 620898 -219 61878 534268 -220 903822 981780 -221 953555 606599 -222 930095 614727 -223 133674 176610 -224 242893 337621 -225 137294 194229 -226 702650 217496 -227 921057 37428 -228 286710 225405 -229 606923 118559 -230 489505 608770 -231 14854 258300 -232 334337 469449 -233 233217 591889 -234 215666 622354 -235 44117 742144 -236 208634 138617 -237 937349 247190 -238 48545 557659 -239 59138 150845 -240 750601 23680 -241 878975 846493 -242 870514 650665 -243 650845 174968 -244 701469 105449 -245 364103 171013 -246 405231 439523 -247 490150 161678 -248 773145 814938 -249 669250 682905 -250 566181 576015 -251 117471 92047 -252 587020 113614 -253 315253 856157 -254 346982 270391 -255 386582 111742 -256 523935 501473 -257 754456 318492 -258 203538 575330 -259 768150 172659 -260 696304 418279 -261 922639 532761 -262 56173 468102 -263 624672 116587 -264 551596 823735 -265 931936 192388 -266 210677 788746 -267 764262 638859 -268 499744 377502 -269 92037 552021 -270 447126 75514 -271 406818 528810 -272 409145 945823 -273 248373 872469 -274 383350 22047 -275 537005 656558 -276 263341 845515 -277 750968 373793 -278 365338 328725 -279 327784 948161 -280 613869 937751 -281 764120 794960 -282 823265 311067 -283 704923 995125 -284 92328 808632 -285 70118 331069 -286 191980 746103 -287 635653 39746 -288 154938 77124 -289 781792 682379 -290 102764 295946 -291 788802 603435 -292 209865 994184 -293 428267 415716 -294 965480 453194 -295 933935 504619 -296 285173 283404 -297 481903 116057 -298 883534 660714 -299 893156 369399 -300 790884 171249 -301 90677 700970 -302 919282 241058 -303 867756 659905 -304 635649 756784 -305 945525 949621 -306 363244 874589 -307 14226 109250 -308 652578 480715 -309 313057 707208 -310 427533 44208 -311 101969 722928 -312 17748 898869 -313 240391 491009 -314 826820 394595 -315 171989 495033 -316 441475 25008 -317 736854 153142 -318 932702 914212 -319 589209 100615 -320 341606 947552 -321 102659 258272 -322 25661 889726 -323 402649 562245 -324 265656 583033 -325 875665 392525 -326 833894 498561 -327 197409 295851 -328 729773 65668 -329 964116 766139 -330 308356 953543 -331 70696 535033 -332 415177 997973 -333 846962 104972 -334 848928 124499 -335 706600 90332 -336 215059 730023 -337 831542 35007 -338 210314 603408 -339 525519 721896 -340 169095 174723 -341 526892 60680 -342 86238 218445 -343 424337 906441 -344 609831 738880 -345 306180 827730 -346 65283 464709 -347 394015 251273 -348 732492 372635 -349 426729 990653 -350 679411 799241 -351 36725 543759 -352 413938 700941 -353 865632 773214 -354 412322 978963 -355 871514 823332 -356 455837 225235 -357 459959 480626 -358 888260 605986 -359 141018 163903 -360 265481 474326 -361 678242 858274 -362 445087 907358 -363 53606 671300 -364 316085 130600 -365 169375 437100 -366 191085 546555 -367 850382 345763 -368 718886 301657 -369 600721 605612 -370 330185 818350 -371 765422 445927 -372 473399 831854 -373 149487 207009 -374 19621 486656 -375 197666 61191 -376 56550 296128 -377 821278 242311 -378 608155 490170 -379 693376 68174 -380 982055 563975 -381 172492 806710 -382 648777 53163 -383 57909 9909 -384 751835 628104 -385 331138 681250 -386 586365 654362 -387 966237 204289 -388 417775 382113 -389 36995 202776 -390 299203 563436 -391 977924 247910 -392 148545 605399 -393 264607 384375 -394 897777 797219 -395 287853 708976 -396 549821 972616 -397 274473 648935 -398 231985 111059 -399 479690 269110 -400 414078 767373 -401 170013 816845 -402 187452 634230 -403 83267 808640 -404 147583 215878 -405 977704 223780 -406 592392 896157 -407 780321 273078 -408 14154 199876 -409 532236 999842 -410 821178 946849 -411 530219 482725 -412 214026 563764 -413 511237 769519 -414 466909 332006 -415 121022 609135 -416 234530 821116 -417 225071 75423 -418 971043 81767 -419 467589 875467 -420 591245 64730 -421 852138 897935 -422 976041 341003 -423 178757 67095 -424 758589 710708 -425 137697 462465 -426 644150 147683 -427 148087 804943 -428 532843 348896 -429 591774 112029 -430 663186 1499 -431 341051 272115 -432 624633 912974 -433 371641 694457 -434 920115 439317 -435 94320 947058 -436 441286 821528 -437 862144 358712 -438 302699 382535 -439 334638 409083 -440 30921 162340 -441 177745 354880 -442 668820 119522 -443 268084 962414 -444 196483 312096 -445 703782 276586 -446 161651 28271 -447 781146 644186 -448 623443 30610 -449 35790 617328 -450 38304 796222 -451 732457 349506 -452 679786 975357 -453 284720 289269 -454 478863 28565 -455 536859 570428 -456 152413 279677 -457 408247 386599 -458 839848 312779 -459 490969 980446 -460 289530 341030 -461 658666 302787 -462 401012 298097 -463 214601 91779 -464 141741 886786 -465 73991 13429 -466 903671 306073 -467 872223 460492 -468 9926 898067 -469 946632 282220 -470 279673 955304 -471 471445 216037 -472 22566 362751 -473 617919 858864 -474 627258 483048 -475 429585 531663 -476 888869 149003 -477 543337 24874 -478 892550 426384 -479 477282 219294 -480 965430 386652 -481 342637 196642 -482 748891 581554 -483 98983 936376 -484 257111 45495 -485 768680 980984 -486 685520 7920 -487 550861 757866 -488 452160 509663 -489 759449 376138 -490 405547 788217 -491 614496 922447 -492 921356 687338 -493 670792 707029 -494 557181 290669 -495 361509 73550 -496 640956 901137 -497 513540 298689 -498 269783 463524 -499 665176 264699 -500 910591 108256 -501 99414 251120 -502 77501 641367 -503 917166 966422 -504 227647 855998 -505 354081 185521 -506 869202 186613 -507 857784 891879 -508 745696 441499 -509 683101 450202 -510 311771 635458 -511 271200 992412 -512 134904 660424 -513 881570 434400 -514 930418 909493 -515 840700 485738 -516 282016 903451 -517 22056 220025 -518 919015 427883 -519 64662 29476 -520 941642 229292 -521 23927 106979 -522 978898 726050 -523 802350 648543 -524 766232 853116 -525 417118 835382 -526 533106 755682 -527 423999 424852 -528 826239 195962 -529 229064 857476 -530 722352 489283 -531 901759 198171 -532 914356 124789 -533 161593 762223 -534 207715 131733 -535 89529 97152 -536 675267 829985 -537 33083 476388 -538 102352 515517 -539 568413 710051 -540 834184 685607 -541 205336 72942 -542 187140 351416 -543 583978 83845 -544 989094 897267 -545 58431 156791 -546 220167 932928 -547 939947 844489 -548 554025 193942 -549 73895 502509 -550 623698 286833 -551 80130 56180 -552 18932 731510 -553 630046 460163 -554 568542 72582 -555 840874 742394 -556 206867 331796 -557 799044 565560 -558 269115 968831 -559 258224 69867 -560 570763 967651 -561 688327 705205 -562 508035 802695 -563 17021 619086 -564 811053 301572 -565 846341 642188 -566 946975 495830 -567 869177 91789 -568 478740 873539 -569 273898 621579 -570 82301 615147 -571 825620 919227 -572 326503 90779 -573 468464 514962 -574 424892 137251 -575 827467 934938 -576 382888 772322 -577 656168 981509 -578 339858 584300 -579 187002 927143 -580 252770 756507 -581 186265 946962 -582 990281 225726 -583 916773 287639 -584 5293 708264 -585 97096 754153 -586 543938 120972 -587 242399 635824 -588 584762 916004 -589 49037 526525 -590 462837 432721 -591 432083 883909 -592 48801 89834 -593 455923 13 -594 505549 643450 -595 175015 191100 -596 868245 565633 -597 524483 328147 -598 71208 704648 -599 676827 690678 -600 506016 552460 -601 465924 898367 -602 674414 394745 -603 502855 498978 -604 723521 566333 -605 525585 339845 -606 78750 543551 -607 752127 61670 -608 888261 620631 -609 422478 662133 -610 154518 212125 -611 610812 314614 -612 202247 544635 -613 288228 645571 -614 446558 847654 -615 132969 85783 -616 192483 482703 -617 940 122992 -618 353971 888532 -619 131782 987131 -620 201573 835292 -621 577534 843415 -622 488932 962889 -623 580287 553631 -624 363386 979847 -625 39918 425637 -626 258089 829173 -627 557709 420233 -628 359976 983220 -629 897426 551421 -630 40774 614322 -631 367196 736390 -632 364760 690293 -633 762310 235335 -634 54619 789238 -635 481382 334629 -636 257244 442630 -637 622214 728881 -638 954036 781638 -639 756905 782014 -640 184659 305007 -641 748144 895136 -642 806880 518646 -643 718587 456093 -644 117943 310646 -645 360681 118636 -646 833913 342544 -647 505748 866943 -648 578047 134904 -649 221200 760051 -650 8853 798648 -651 796725 581372 -652 795188 734911 -653 677492 362952 -654 22292 39062 -655 701645 903883 -656 865277 586780 -657 190740 922137 -658 780409 24652 -659 230641 497816 -660 112246 627406 -661 14134 294567 -662 780384 682733 -663 537904 675872 -664 647441 887303 -665 51388 591083 -666 759346 717843 -667 80368 280775 -668 439730 161363 -669 704396 884742 -670 738237 693934 -671 225451 620126 -672 198399 733275 -673 104501 539345 -674 562159 823014 -675 329039 902174 -676 487462 333897 -677 708662 417769 -678 39302 78882 -679 501003 514413 -680 295180 516128 -681 658556 137549 -682 300825 7711 -683 678432 245150 -684 388380 457465 -685 817635 241064 -686 462493 407627 -687 168777 210071 -688 139943 680236 -689 585905 362615 -690 643431 459021 -691 174869 133028 -692 592123 535260 -693 932527 621796 -694 417018 72657 -695 602342 194580 -696 772982 246930 -697 67106 497172 -698 231441 817823 -699 451349 988327 -700 593331 424264 -701 953439 199543 -702 179582 870017 -703 727305 354434 -704 741773 173402 -705 485242 417506 -706 661864 428346 -707 912070 100600 -708 743145 411625 -709 70443 803652 -710 776269 860608 -711 793801 400053 -712 864134 393370 -713 287625 262949 -714 228044 298759 -715 482766 785509 -716 938463 412503 -717 877373 225925 -718 797157 746522 -719 220957 491522 -720 792114 520901 -721 551353 613365 -722 296387 741734 -723 400778 372929 -724 382796 673735 -725 209547 968492 -726 589778 152589 -727 505561 807821 -728 485801 540936 -729 322170 953657 -730 72860 980782 -731 133476 250251 -732 381288 964340 -733 866152 48499 -734 131958 170336 -735 699821 370216 -736 28829 396707 -737 594105 807777 -738 270830 641211 -739 894491 56312 -740 907569 746688 -741 940778 274387 -742 225899 501984 -743 652033 688211 -744 31215 913032 -745 359772 748658 -746 614564 50621 -747 791701 421898 -748 492072 154646 -749 19259 488609 -750 470904 759567 -751 478437 326483 -752 766166 462858 -753 27713 315390 -754 926689 3577 -755 155787 797589 -756 509721 409138 -757 593884 324201 -758 366218 82854 -759 458549 959389 -760 472268 711506 -761 29239 643348 -762 699432 940254 -763 891778 702345 -764 630540 131247 -765 780064 955439 -766 714521 890914 -767 900525 109979 -768 236967 531639 -769 680503 901697 -770 135766 569178 -771 229662 71825 -772 440763 648266 -773 719418 971215 -774 351189 851447 -775 530120 789726 -776 524105 888011 -777 708545 515464 -778 45045 587523 -779 425958 656187 -780 225890 496073 -781 634887 24991 -782 867811 586609 -783 567927 437895 -784 968374 945617 -785 604783 395002 -786 731665 607102 -787 429269 682541 -788 187400 141228 -789 934803 183967 -790 895209 304254 -791 276387 974352 -792 905357 283990 -793 320551 689530 -794 23103 313915 -795 542051 799072 -796 563264 734885 -797 296914 740763 -798 837513 622559 -799 642555 758222 -800 460865 578190 -801 36412 167222 -802 956237 225865 -803 513339 549752 -804 982654 424555 -805 194913 355515 -806 564419 112042 -807 114136 426818 -808 932808 900002 -809 728077 127047 -810 749095 945367 -811 795340 210152 -812 484752 26592 -813 358590 564442 -814 650864 203404 -815 169202 637648 -816 158573 510248 -817 989053 539693 -818 739835 164344 -819 860216 478539 -820 351181 420549 -821 961452 896056 -822 564820 596683 -823 3731 353112 -824 250133 270322 -825 382173 273070 -826 971695 439150 -827 589019 823217 -828 419616 526164 -829 178168 416544 -830 486029 348994 -831 689535 504115 -832 73373 774363 -833 394062 668363 -834 547221 517452 -835 423086 579696 -836 649868 457754 -837 744874 476024 -838 973672 356189 -839 621132 661534 -840 606975 832425 -841 386274 234320 -842 717375 820207 -843 948112 654458 -844 436874 214689 -845 145631 71472 -846 617122 342764 -847 55452 771485 -848 770680 503698 -849 151181 88795 -850 623011 647542 -851 731979 588598 -852 663346 549747 -853 886796 737374 -854 721775 768812 -855 875105 765158 -856 89289 963478 -857 270912 414557 -858 731871 346771 -859 448662 301887 -860 3683 890364 -861 517181 458426 -862 894441 775544 -863 847717 61270 -864 794408 195127 -865 589228 236297 -866 634414 852320 -867 786429 841817 -868 743136 422795 -869 963407 302817 -870 88335 601340 -871 205795 134987 -872 211006 255267 -873 554290 158696 -874 448323 279908 -875 923768 709410 -876 709289 956054 -877 499567 386713 -878 13127 879658 -879 802169 821529 -880 806610 464000 -881 773966 418957 -882 680476 273764 -883 559363 954301 -884 752471 15645 -885 860267 573174 -886 898447 168753 -887 378119 294272 -888 181074 561127 -889 958858 507727 -890 762416 968058 -891 259100 972879 -892 388516 125227 -893 462331 215456 -894 171844 512664 -895 282454 788834 -896 670323 947762 -897 442550 515160 -898 702892 37041 -899 756868 916733 -900 74192 993162 -901 199838 940595 -902 517491 955709 -903 450309 736409 -904 567537 374340 -905 924382 797670 -906 707814 289505 -907 539074 17776 -908 793677 826203 -909 976407 165315 -910 570872 522321 -911 197433 835738 -912 941452 867104 -913 373336 957852 -914 651262 422409 -915 843963 444665 -916 993589 584517 -917 583345 964745 -918 260244 969595 -919 433804 370740 -920 331550 636127 -921 239049 6528 -922 941791 760132 -923 591400 834585 -924 6309 575445 -925 141824 745040 -926 385779 334458 -927 72770 629527 -928 999572 615320 -929 357250 552746 -930 695464 480714 -931 302605 196797 -932 42790 288254 -933 558621 701285 -934 347713 778942 -935 426375 959091 -936 819894 400961 -937 23491 825831 -938 136541 862975 -939 762967 311925 -940 867531 758015 -941 600601 98515 -942 726944 363249 -943 142059 796792 -944 541727 295090 -945 406124 558958 -946 621881 654862 -947 944364 372458 -948 816233 838087 -949 983037 115960 -950 623591 728425 -951 71617 694384 -952 707914 383808 -953 144438 287263 -954 607513 709521 -955 487467 202780 -956 73593 62159 -957 146621 136506 -958 858832 647742 -959 252432 670331 -960 472021 720533 -961 718248 231753 -962 155351 697949 -963 887473 125509 -964 693047 639682 -965 681392 849278 -966 255461 53446 -967 824458 664751 -968 684422 538441 -969 951894 590438 -970 504416 494317 -971 544359 871395 -972 823068 685591 -973 840710 390128 -974 499511 246415 -975 484985 690723 -976 145040 343355 -977 434567 774313 -978 472963 18171 -979 869925 43163 -980 699386 605997 -981 335369 17074 -982 205104 993150 -983 658420 202197 -984 239091 461030 -985 295795 468703 -986 148231 6017 -987 185346 781298 -988 575492 374893 -989 797186 381037 -990 224986 869302 -991 255584 649884 -992 940295 75395 -993 513908 238386 -994 848342 831308 -995 6330 482224 -996 299350 490863 -997 294643 35712 -998 346085 538342 -999 686049 41770 -1000 110098 465817 diff --git a/devel-examples/vignette-example/instances/1000-9.tsp b/devel-examples/vignette-example/instances/1000-9.tsp deleted file mode 100644 index 9dc0350a..00000000 --- a/devel-examples/vignette-example/instances/1000-9.tsp +++ /dev/null @@ -1,1006 +0,0 @@ -NAME : portgen-1000-9 -COMMENT : portgen N=1000, seed=9 -TYPE : TSP -DIMENSION : 1000 -EDGE_WEIGHT_TYPE : EUC_2D -NODE_COORD_SECTION -1 700201 849072 -2 490958 817370 -3 588316 725976 -4 42637 6036 -5 418389 999104 -6 845134 677053 -7 248841 510193 -8 132630 795628 -9 456974 69028 -10 350169 523528 -11 25138 874444 -12 907202 132077 -13 19931 999382 -14 242387 902061 -15 728333 741534 -16 211556 254227 -17 357347 66086 -18 45382 516643 -19 938786 944370 -20 955911 912249 -21 367006 17454 -22 986995 971032 -23 760599 143410 -24 34122 875585 -25 583435 101222 -26 998498 993848 -27 990067 184034 -28 783761 445974 -29 491725 424871 -30 771987 71672 -31 787190 98267 -32 50124 506140 -33 632098 827679 -34 690057 277809 -35 749594 989219 -36 761505 581388 -37 485593 248946 -38 525030 31289 -39 884377 723167 -40 348315 573957 -41 507657 817516 -42 130073 656660 -43 954344 113289 -44 204102 851207 -45 433988 217702 -46 826586 660977 -47 194776 966691 -48 150743 785617 -49 531860 738048 -50 446002 729309 -51 259032 310954 -52 527269 9477 -53 593565 180981 -54 863774 333406 -55 229689 670472 -56 241871 640518 -57 990883 554284 -58 245086 126212 -59 903490 83433 -60 355397 846481 -61 295819 952009 -62 831807 20284 -63 730187 450550 -64 54119 476115 -65 655381 344048 -66 167514 550971 -67 493478 677842 -68 332085 867139 -69 826632 575789 -70 411574 828132 -71 209798 120669 -72 495809 587506 -73 921883 874576 -74 829170 174491 -75 236504 700192 -76 731498 55744 -77 82667 101953 -78 561794 708061 -79 817476 849426 -80 677392 726425 -81 354349 287985 -82 921832 401557 -83 460674 121202 -84 144708 403376 -85 632400 370509 -86 297041 728999 -87 846929 655204 -88 114983 240074 -89 869341 729853 -90 458490 22125 -91 633073 204692 -92 798723 928955 -93 989699 879529 -94 629139 91920 -95 217168 210882 -96 722431 423255 -97 943388 41064 -98 531090 480798 -99 273740 840604 -100 472523 681808 -101 5235 99317 -102 716001 214378 -103 67118 526805 -104 257021 153711 -105 112254 682264 -106 78922 725556 -107 632258 466509 -108 3994 931093 -109 344596 880767 -110 870466 979875 -111 847462 304103 -112 930853 950592 -113 365274 197724 -114 12997 632550 -115 588085 588177 -116 983053 715630 -117 617599 776225 -118 943203 907517 -119 572434 332213 -120 924961 58605 -121 534933 748371 -122 221454 237292 -123 363420 418327 -124 492402 992796 -125 675789 333366 -126 467801 641189 -127 252518 884345 -128 698754 289605 -129 481718 939776 -130 271174 159600 -131 954371 924807 -132 228749 53649 -133 147331 330550 -134 504101 394965 -135 103088 585666 -136 438691 351800 -137 204977 537100 -138 512074 206272 -139 51584 46507 -140 251837 75669 -141 716006 73220 -142 361376 428484 -143 633806 58246 -144 486880 563950 -145 628893 612653 -146 403416 177468 -147 229124 339294 -148 619914 183132 -149 543393 684354 -150 725218 157148 -151 366743 445894 -152 740958 600120 -153 617360 394580 -154 279812 824034 -155 250539 640508 -156 802724 917768 -157 310883 658520 -158 756184 776903 -159 695682 889455 -160 433735 964199 -161 787156 819747 -162 669747 945940 -163 218923 992796 -164 610841 604857 -165 919740 117493 -166 926459 227550 -167 795967 611329 -168 272944 798238 -169 762337 702855 -170 672300 856902 -171 362563 597425 -172 130215 664694 -173 825496 583668 -174 507720 283184 -175 120371 627117 -176 572290 938536 -177 764613 607724 -178 230688 139192 -179 649927 129629 -180 327176 819121 -181 632242 576957 -182 151733 393636 -183 277945 205299 -184 787552 646189 -185 833023 172516 -186 269183 412498 -187 769083 806617 -188 391908 848619 -189 55133 62023 -190 715251 79731 -191 342869 481212 -192 277680 100618 -193 485251 349502 -194 75816 402330 -195 333384 67644 -196 10685 116148 -197 869832 499783 -198 587719 950065 -199 828341 323597 -200 272785 976877 -201 528534 445696 -202 567932 40639 -203 284248 91078 -204 660855 663995 -205 122473 881186 -206 63375 247596 -207 796244 259531 -208 808436 516094 -209 707124 651950 -210 805917 327879 -211 376958 463955 -212 373403 856146 -213 643981 823486 -214 844565 728443 -215 522369 300829 -216 187764 391138 -217 939550 834065 -218 16355 95273 -219 684967 18149 -220 292182 969156 -221 642377 423866 -222 596413 5504 -223 690686 546729 -224 742744 13685 -225 855802 764232 -226 105499 99897 -227 801228 971955 -228 789112 137396 -229 506145 733867 -230 24284 188975 -231 406110 642705 -232 371813 153316 -233 238808 639509 -234 651182 790740 -235 568844 261706 -236 773350 693439 -237 796147 41684 -238 222380 277060 -239 662727 401447 -240 67033 506585 -241 317341 110697 -242 704159 333394 -243 894719 545058 -244 19324 786233 -245 595256 376846 -246 444090 894226 -247 449305 30475 -248 195806 948937 -249 627718 554728 -250 783123 413626 -251 884002 341297 -252 946651 349216 -253 446891 994801 -254 395738 467834 -255 77236 244054 -256 118071 719911 -257 138611 647438 -258 744885 511883 -259 193400 341337 -260 957510 289870 -261 11790 96454 -262 7616 155218 -263 377703 432052 -264 746788 446930 -265 594793 227578 -266 881322 194893 -267 324210 822979 -268 388513 597429 -269 972086 56720 -270 588509 382835 -271 351657 677987 -272 828723 305385 -273 365055 347635 -274 886610 294086 -275 652772 763753 -276 202149 180787 -277 959935 555545 -278 532303 689109 -279 17086 123672 -280 960703 849461 -281 22715 339017 -282 879324 694400 -283 892397 440084 -284 891188 833225 -285 282382 397249 -286 625273 899313 -287 688565 193757 -288 87720 831002 -289 136519 452071 -290 622914 688593 -291 414966 623115 -292 486227 745331 -293 204863 542305 -294 315568 629810 -295 930581 948429 -296 706241 138861 -297 774338 191259 -298 757562 452343 -299 989421 634965 -300 217664 534053 -301 211116 434538 -302 671172 964178 -303 348787 579034 -304 694560 214603 -305 350681 989998 -306 373541 387276 -307 193090 12274 -308 143220 883853 -309 564678 688065 -310 936838 440053 -311 450663 256222 -312 615560 748329 -313 186132 190734 -314 228141 724386 -315 844970 508686 -316 136442 921915 -317 101389 632916 -318 315051 27690 -319 430024 473953 -320 602110 321009 -321 977626 627502 -322 692971 490528 -323 497766 450018 -324 523300 26009 -325 5126 566827 -326 224202 265034 -327 789995 708978 -328 397610 289201 -329 333148 38256 -330 649126 321097 -331 149009 220606 -332 612493 29672 -333 12372 746038 -334 694304 702562 -335 514508 693202 -336 360553 538669 -337 682938 370010 -338 215851 185628 -339 466227 906581 -340 350718 896931 -341 857585 189885 -342 75260 523873 -343 359676 915835 -344 309421 71717 -345 620543 569012 -346 333385 727462 -347 959445 908908 -348 960456 438956 -349 944564 322961 -350 274677 312137 -351 983791 616718 -352 675290 108195 -353 709241 34317 -354 189774 266122 -355 349301 481775 -356 979779 261431 -357 417712 80113 -358 987711 421547 -359 261161 703585 -360 69215 573415 -361 801474 371343 -362 427885 202370 -363 709411 743834 -364 863379 574742 -365 660769 181533 -366 995854 200105 -367 557280 868942 -368 917152 596154 -369 772172 995146 -370 536162 938129 -371 654673 605836 -372 2502 47128 -373 767538 962042 -374 299577 757074 -375 199496 216622 -376 575577 369821 -377 662192 93143 -378 316283 783685 -379 59438 806347 -380 191043 113087 -381 262144 194627 -382 729959 411172 -383 827101 373942 -384 258929 370584 -385 312575 25668 -386 121970 504087 -387 504088 852593 -388 997838 431652 -389 709151 334742 -390 886087 925725 -391 684395 57031 -392 383699 547682 -393 919389 801226 -394 470145 146107 -395 41840 543209 -396 337225 401588 -397 682571 510493 -398 816159 150585 -399 101748 149908 -400 49289 335886 -401 252891 964834 -402 870986 273771 -403 532226 518546 -404 986121 114509 -405 173753 515056 -406 313540 913330 -407 764507 647833 -408 775861 860555 -409 512056 219466 -410 595013 676516 -411 272193 109020 -412 321294 976688 -413 772776 157135 -414 633101 230316 -415 320366 479292 -416 445530 594641 -417 160988 371030 -418 612185 771064 -419 292523 735866 -420 771820 58833 -421 289169 250678 -422 551094 365324 -423 271016 228204 -424 80294 705882 -425 737716 659024 -426 517484 871432 -427 829542 569997 -428 890355 658250 -429 803846 499955 -430 661586 761161 -431 226022 250254 -432 342689 114920 -433 225887 62861 -434 362236 399182 -435 376816 547657 -436 780261 806173 -437 481749 935989 -438 159031 400761 -439 279478 751297 -440 86332 114526 -441 353288 133145 -442 568730 559204 -443 253270 195275 -444 251951 46067 -445 145143 549323 -446 408828 893341 -447 359049 224162 -448 278571 482995 -449 768929 615105 -450 206293 870254 -451 948726 328996 -452 619550 623190 -453 305735 384339 -454 302701 270337 -455 316727 695080 -456 406298 757778 -457 354812 112262 -458 352332 332680 -459 891205 118526 -460 836348 742891 -461 293932 747130 -462 192889 506562 -463 598931 451264 -464 186623 858559 -465 630253 774692 -466 98060 9140 -467 434569 391252 -468 708227 595510 -469 778333 456468 -470 206871 920589 -471 304070 133424 -472 209719 402252 -473 255391 661698 -474 700452 852487 -475 625231 827306 -476 296372 910369 -477 984851 431601 -478 772194 939585 -479 894427 228297 -480 914962 710225 -481 606005 846233 -482 63465 396137 -483 391009 272874 -484 548059 952559 -485 856870 690634 -486 632228 38717 -487 493295 9042 -488 446519 383562 -489 762278 761287 -490 734367 659345 -491 556837 958326 -492 943596 920028 -493 168686 251827 -494 945674 38431 -495 243 435353 -496 47450 825773 -497 599597 516237 -498 288360 265353 -499 640129 200677 -500 955732 871828 -501 899420 939164 -502 118120 965886 -503 270469 338045 -504 966773 64823 -505 262915 520367 -506 993910 855995 -507 228054 479609 -508 662774 780232 -509 246635 547228 -510 107776 125656 -511 632744 347382 -512 996827 985042 -513 791214 693064 -514 920597 527408 -515 738572 108473 -516 416789 697454 -517 498372 214000 -518 665434 700842 -519 730271 463986 -520 257253 388453 -521 5191 398446 -522 930655 874586 -523 802609 700067 -524 828946 614555 -525 725022 595296 -526 344756 112720 -527 462104 847259 -528 455039 201965 -529 440791 904119 -530 300451 569626 -531 607774 502786 -532 807569 874461 -533 515175 595464 -534 925340 353467 -535 677000 962707 -536 951286 632080 -537 822205 512480 -538 780900 520023 -539 885278 149567 -540 530002 589249 -541 252272 16477 -542 226957 168946 -543 500698 914003 -544 889884 623911 -545 698824 69970 -546 775502 376803 -547 786986 294546 -548 437166 373111 -549 576240 418174 -550 93686 282585 -551 814789 679378 -552 84552 135773 -553 343024 328278 -554 885763 293157 -555 346560 541036 -556 312080 816880 -557 205295 230481 -558 794124 230970 -559 715799 513023 -560 437294 142064 -561 19223 507165 -562 259780 394414 -563 147918 271908 -564 547528 686431 -565 169455 452621 -566 634259 592121 -567 803007 988966 -568 277168 435392 -569 811182 996476 -570 374822 269728 -571 198203 376861 -572 186616 556759 -573 50747 268337 -574 117022 392571 -575 146628 165258 -576 825583 889809 -577 248718 641065 -578 648325 222667 -579 876370 95585 -580 858604 907631 -581 517096 889287 -582 918335 76832 -583 342832 935219 -584 630263 648535 -585 179734 525787 -586 113948 323228 -587 366395 272036 -588 316481 129414 -589 258446 618715 -590 746088 925250 -591 395537 451942 -592 827826 261824 -593 935525 744972 -594 673785 726175 -595 646133 341949 -596 805128 162646 -597 816741 849035 -598 155779 874975 -599 10465 914579 -600 240277 921332 -601 9890 498306 -602 646482 221377 -603 769720 373640 -604 61982 986894 -605 705539 903353 -606 548882 150195 -607 449452 516655 -608 102503 354449 -609 72545 69300 -610 921052 467857 -611 924753 715684 -612 408258 258401 -613 515896 615641 -614 676745 145017 -615 502315 942840 -616 67431 271551 -617 913176 842734 -618 376368 245342 -619 2489 311171 -620 159321 581075 -621 672426 604485 -622 805122 178275 -623 417195 89444 -624 754388 558340 -625 333138 540138 -626 198229 865447 -627 412264 297436 -628 853901 738339 -629 585130 803747 -630 845009 524378 -631 371150 750811 -632 827573 124464 -633 230927 944396 -634 345073 271176 -635 99459 13058 -636 600061 514205 -637 736161 380913 -638 269628 59305 -639 303419 110821 -640 404500 777557 -641 30510 872997 -642 300007 977937 -643 571690 316620 -644 443978 788712 -645 611807 431971 -646 900269 731313 -647 211711 146262 -648 981013 158220 -649 868324 424208 -650 908647 357890 -651 786024 643566 -652 153840 555581 -653 509627 325231 -654 565440 434326 -655 725746 537280 -656 294360 796418 -657 191940 413038 -658 624109 639837 -659 539099 681310 -660 143450 72706 -661 76072 920864 -662 362529 741569 -663 227033 956495 -664 360365 180579 -665 871285 944396 -666 493865 869093 -667 385075 867219 -668 483196 234092 -669 681056 886969 -670 353827 931852 -671 777521 762667 -672 645261 539100 -673 355899 979404 -674 368784 470142 -675 919228 24518 -676 797855 687744 -677 552922 964250 -678 864024 916931 -679 258491 286620 -680 72385 275535 -681 644175 678470 -682 80498 793893 -683 759759 531693 -684 151156 652840 -685 57139 644704 -686 271053 68956 -687 762081 118931 -688 274851 388327 -689 367941 398278 -690 877544 310102 -691 698003 73744 -692 108193 595750 -693 300221 815394 -694 788594 591181 -695 107460 951502 -696 82935 28216 -697 829830 709123 -698 660799 708564 -699 585 526330 -700 264249 967571 -701 611463 970505 -702 592597 609126 -703 326514 724111 -704 579550 957172 -705 664029 48630 -706 128336 667310 -707 179160 120883 -708 192599 615958 -709 848640 371375 -710 133094 51194 -711 531107 624826 -712 388590 89567 -713 33241 300547 -714 476359 152955 -715 792416 550739 -716 808776 410768 -717 734248 828914 -718 181765 30693 -719 894584 987310 -720 403150 684263 -721 966753 417218 -722 458086 56265 -723 420394 458109 -724 639626 740262 -725 675881 360251 -726 232204 847630 -727 733913 713509 -728 158795 200694 -729 236256 763682 -730 427360 295821 -731 829527 592240 -732 554022 979766 -733 81876 711118 -734 209223 122894 -735 700488 734489 -736 976332 108378 -737 695493 772842 -738 818989 683477 -739 890912 675080 -740 930771 832546 -741 64290 712677 -742 725594 496595 -743 721212 216536 -744 856746 754482 -745 747038 470647 -746 821470 771689 -747 286822 668660 -748 707931 858375 -749 721724 685244 -750 237276 736917 -751 567197 964545 -752 809491 843335 -753 295961 519527 -754 122035 237318 -755 992297 942259 -756 343948 481774 -757 16644 956713 -758 474350 57837 -759 305417 885361 -760 271835 223500 -761 989393 523978 -762 885618 963571 -763 167292 11786 -764 298887 852158 -765 476881 299461 -766 561442 653594 -767 682783 988512 -768 488598 582516 -769 696032 768880 -770 22244 663374 -771 911118 971384 -772 482647 523537 -773 481253 297491 -774 886071 323251 -775 501368 696144 -776 559488 869565 -777 208363 937814 -778 175474 913603 -779 281762 820978 -780 354736 713444 -781 823494 353154 -782 215073 328922 -783 31140 372563 -784 999126 493106 -785 475460 176858 -786 171765 982166 -787 383993 575690 -788 664012 119828 -789 315614 947804 -790 788096 253689 -791 730024 477908 -792 497422 763436 -793 475966 208287 -794 438520 353860 -795 957371 116035 -796 583390 202926 -797 293420 845385 -798 491609 928952 -799 587391 906956 -800 859524 361425 -801 981877 938267 -802 535155 247678 -803 966119 81580 -804 372143 444927 -805 461847 967187 -806 475082 927901 -807 863607 238701 -808 130054 620568 -809 59734 369687 -810 837313 102188 -811 785172 92169 -812 633581 114034 -813 194981 233498 -814 447010 136315 -815 609570 582432 -816 747684 870687 -817 485956 820908 -818 778607 802123 -819 614300 258720 -820 633381 855398 -821 148553 68833 -822 516547 855183 -823 330862 491220 -824 569344 179385 -825 650404 258110 -826 481941 451076 -827 297385 277092 -828 613740 111189 -829 452310 714246 -830 469071 163996 -831 467279 113422 -832 811545 606449 -833 818634 406249 -834 411354 8424 -835 907838 638834 -836 51223 880348 -837 719283 579202 -838 620246 334096 -839 794783 356488 -840 500294 83792 -841 781187 732764 -842 667243 445574 -843 115152 634261 -844 59142 879507 -845 2274 372357 -846 390769 605876 -847 350882 994547 -848 804174 268204 -849 349549 937344 -850 234936 996766 -851 696436 212855 -852 679091 566611 -853 476923 749177 -854 783832 851811 -855 161939 979478 -856 52047 572803 -857 711971 96713 -858 773227 861403 -859 762540 816998 -860 802274 550429 -861 56699 474010 -862 773487 911071 -863 942397 838368 -864 201257 152671 -865 102279 871069 -866 550263 942972 -867 194549 520815 -868 31744 208383 -869 20792 570529 -870 672347 253749 -871 871721 242143 -872 77232 451845 -873 315658 916758 -874 832388 439810 -875 52149 965806 -876 66947 196877 -877 835064 363867 -878 446503 753464 -879 18306 158275 -880 534867 268539 -881 728385 213303 -882 179464 908190 -883 107757 809903 -884 495570 260126 -885 781055 856468 -886 29014 322730 -887 764848 836468 -888 483481 859821 -889 638945 409620 -890 464568 188933 -891 820061 42981 -892 617804 833740 -893 142684 336960 -894 763507 286358 -895 413057 221841 -896 712813 760665 -897 789473 815878 -898 224735 548991 -899 477295 240764 -900 968363 455836 -901 277813 422768 -902 975242 863215 -903 145744 23966 -904 579073 1323 -905 221183 109543 -906 989956 731947 -907 745218 313026 -908 555726 967719 -909 423829 363585 -910 683454 558766 -911 332608 254806 -912 291763 325219 -913 578655 606245 -914 347488 901633 -915 690723 459515 -916 280748 637621 -917 188436 355024 -918 198977 88113 -919 297763 304778 -920 278014 174964 -921 913130 399922 -922 602904 854291 -923 889232 458006 -924 468902 464254 -925 237223 618489 -926 201502 575662 -927 550040 508848 -928 175087 640191 -929 234332 620217 -930 664237 57631 -931 726202 274295 -932 723309 46219 -933 196412 590034 -934 129043 890926 -935 423793 97719 -936 498817 959575 -937 126362 64965 -938 357263 756945 -939 704766 782914 -940 150131 938463 -941 371913 727271 -942 237395 633092 -943 733312 6453 -944 914312 142217 -945 158612 608943 -946 959069 406836 -947 880984 180294 -948 676147 953555 -949 273559 537938 -950 497027 132287 -951 753239 685987 -952 314122 298759 -953 246576 474231 -954 338266 916947 -955 775535 168634 -956 725878 92114 -957 461604 55293 -958 98561 319366 -959 393310 543014 -960 370071 242856 -961 316474 591104 -962 393898 291506 -963 344480 812829 -964 645452 827602 -965 818388 883032 -966 418678 787818 -967 7379 981497 -968 212585 279798 -969 265666 182101 -970 534531 413946 -971 613142 371298 -972 772145 915756 -973 292469 367965 -974 12937 589478 -975 835814 863317 -976 308103 445956 -977 719550 613995 -978 713608 965420 -979 678608 332625 -980 86174 966778 -981 208564 156165 -982 382416 361589 -983 555491 354580 -984 319968 545848 -985 762824 730596 -986 306428 803831 -987 707199 506753 -988 934753 870518 -989 871554 779903 -990 577898 379059 -991 134221 312827 -992 741428 851610 -993 674467 262513 -994 405402 645789 -995 426005 858005 -996 959829 719818 -997 419276 803934 -998 107517 809310 -999 664098 265391 -1000 981002 421951 diff --git a/devel-examples/vignette-example/scenario.txt b/devel-examples/vignette-example/scenario.txt index 16796d96..99c66a9a 100644 --- a/devel-examples/vignette-example/scenario.txt +++ b/devel-examples/vignette-example/scenario.txt @@ -2,11 +2,11 @@ ## Scenario setup for Iterated Race (irace). ############################################################################ logFile = "./irace-acotsp.Rdata" -trainInstancesDir = "./instances" -trainInstancesFile = "instances.txt" +trainInstancesDir = "./acotsp/instances/" +trainInstancesFile = "./tsp-2000-50-training.txt" maxExperiments = 1000 -testInstancesDir = "./instances" -testInstancesFile = "instances-test.txt" +testInstancesDir = "./acotsp/instances/" +testInstancesFile = "./tsp-2000-50-testing.txt" configurationsFile = "default.txt" testNbElites = 5 testIterationElites = 1 diff --git a/devel-examples/vignette-example/target-runner b/devel-examples/vignette-example/target-runner index d06519cc..90569f5d 100755 --- a/devel-examples/vignette-example/target-runner +++ b/devel-examples/vignette-example/target-runner @@ -28,7 +28,7 @@ CONFIG_PARAMS=$* # End of parsing # Path to the ACOTSP software: -BINDIR=~/bin +BINDIR=./acotsp EXE=${BINDIR}/acotsp EXE_PARAMS="--tries 1 --quiet -i $INSTANCE --seed $SEED ${CONFIG_PARAMS}" diff --git a/devel-examples/vignette-example/tsp-2000-50-testing.txt b/devel-examples/vignette-example/tsp-2000-50-testing.txt new file mode 100644 index 00000000..ea3326ff --- /dev/null +++ b/devel-examples/vignette-example/tsp-2000-50-testing.txt @@ -0,0 +1,50 @@ +2000-101.tsp +2000-102.tsp +2000-103.tsp +2000-104.tsp +2000-105.tsp +2000-106.tsp +2000-107.tsp +2000-108.tsp +2000-109.tsp +2000-110.tsp +2000-201.tsp +2000-202.tsp +2000-203.tsp +2000-204.tsp +2000-205.tsp +2000-206.tsp +2000-207.tsp +2000-208.tsp +2000-209.tsp +2000-210.tsp +2000-301.tsp +2000-302.tsp +2000-303.tsp +2000-304.tsp +2000-305.tsp +2000-306.tsp +2000-307.tsp +2000-308.tsp +2000-309.tsp +2000-310.tsp +2000-401.tsp +2000-402.tsp +2000-403.tsp +2000-404.tsp +2000-405.tsp +2000-406.tsp +2000-407.tsp +2000-408.tsp +2000-409.tsp +2000-410.tsp +2000-501.tsp +2000-502.tsp +2000-503.tsp +2000-504.tsp +2000-505.tsp +2000-506.tsp +2000-507.tsp +2000-508.tsp +2000-509.tsp +2000-510.tsp diff --git a/devel-examples/vignette-example/tsp-2000-50-training.txt b/devel-examples/vignette-example/tsp-2000-50-training.txt new file mode 100644 index 00000000..cb7ba405 --- /dev/null +++ b/devel-examples/vignette-example/tsp-2000-50-training.txt @@ -0,0 +1,50 @@ +2000-511.tsp +2000-512.tsp +2000-513.tsp +2000-514.tsp +2000-515.tsp +2000-516.tsp +2000-517.tsp +2000-518.tsp +2000-519.tsp +2000-520.tsp +2000-611.tsp +2000-612.tsp +2000-613.tsp +2000-614.tsp +2000-615.tsp +2000-616.tsp +2000-617.tsp +2000-618.tsp +2000-619.tsp +2000-620.tsp +2000-711.tsp +2000-712.tsp +2000-713.tsp +2000-714.tsp +2000-715.tsp +2000-716.tsp +2000-717.tsp +2000-718.tsp +2000-719.tsp +2000-720.tsp +2000-811.tsp +2000-812.tsp +2000-813.tsp +2000-814.tsp +2000-815.tsp +2000-816.tsp +2000-817.tsp +2000-818.tsp +2000-819.tsp +2000-820.tsp +2000-911.tsp +2000-912.tsp +2000-913.tsp +2000-914.tsp +2000-915.tsp +2000-916.tsp +2000-917.tsp +2000-918.tsp +2000-919.tsp +2000-920.tsp diff --git a/devel-tests/random/instances.txt b/devel-tests/random/instances.txt index 97b3d1a5..3fb3367c 100644 --- a/devel-tests/random/instances.txt +++ b/devel-tests/random/instances.txt @@ -1,15 +1,15 @@ 1 -2 -3 -4 -5 -6 -7 -8 -9 -10 -11 -12 -13 -14 -15 +# 2 +# 3 +# 4 +# 5 +# 6 +# 7 +# 8 +# 9 +# 10 +# 11 +# 12 +# 13 +# 14 +# 15 diff --git a/devel-tests/random/parameters.txt b/devel-tests/random/parameters.txt index 6de92c95..dcf52196 100644 --- a/devel-tests/random/parameters.txt +++ b/devel-tests/random/parameters.txt @@ -1,2 +1,6 @@ -nants "" c (5,10,20) +nants "" i (1,20) lsize "" c (5,10,20) +rest_t "--restart_temp_ratio=" r,log (0.00001, 0.9) + +[global] +digits = 5 diff --git a/devel-tests/random/scenario.txt b/devel-tests/random/scenario.txt index c0b4618a..36e71de9 100644 --- a/devel-tests/random/scenario.txt +++ b/devel-tests/random/scenario.txt @@ -1,5 +1,7 @@ trainInstancesFile = "instances.txt" -maxExperiments = 1000 -nbConfigurations = 9 -logFile = "" +maxExperiments = 100 +#maxTime = 100 +#boundMax= 10 +#nbConfigurations = 9 +#logFile = "" # targetRunnerTimeout = 1 diff --git a/devel-tests/random/target-runner b/devel-tests/random/target-runner index b2350a6a..557782d4 100755 --- a/devel-tests/random/target-runner +++ b/devel-tests/random/target-runner @@ -1,4 +1,4 @@ #!/bin/bash -sleep "0.$RANDOM" -sleep "0.$RANDOM" +#sleep "0.$RANDOM" +#sleep "0.$RANDOM" echo $(perl -e "print $5*$6*0.$RANDOM") diff --git a/inst/examples/acotsp/README b/inst/examples/acotsp/README index 20f78a65..3a513b37 100644 --- a/inst/examples/acotsp/README +++ b/inst/examples/acotsp/README @@ -1,17 +1,20 @@ Example configuration scenario: ACOTSP -===================================== +====================================== This example scenario shows how irace can be used to tune an optimization algorithm. ACOTSP is a software that implements various Ant Colony algorithms -for the symmetric Traveling Salesman Problem. -The ACOTSP software is available at: +for the symmetric Traveling Salesman Problem. -http://www.aco-metaheuristic.org/aco-code/ +We will use the version provided here: + +https://github.com/MLopez-Ibanez/ACOTSPQAP/archive/refs/heads/master.zip TSP instances and more information is available at: https://iridia.ulb.ac.be/supp/IridiaSupp2016-003/index.html +You should use the instances from this file: +https://iridia.ulb.ac.be/supp/IridiaSupp2016-003/scenarios/acotsp/instances.tar.gz diff --git a/inst/examples/acotsp/parameters-acotsp.txt b/inst/examples/acotsp/parameters-acotsp.txt index d8e01df3..6dfd9d48 100644 --- a/inst/examples/acotsp/parameters-acotsp.txt +++ b/inst/examples/acotsp/parameters-acotsp.txt @@ -5,7 +5,7 @@ localsearch "--localsearch " c (0, 1, 2, 3) alpha "--alpha " r (0.00, 5.00) beta "--beta " r (0.00, 10.00) rho "--rho " r (0.01, 1.00) -ants "--ants " i (5, 100) +ants "--ants " i (5, 100) q0 "--q0 " r (0.0, 1.0) | algorithm == "acs" rasrank "--rasranks " i (1, 100) | algorithm == "ras" elitistants "--elitistants " i (1, 750) | algorithm == "eas" diff --git a/inst/examples/acotsp/scenario.txt b/inst/examples/acotsp/scenario.txt index ac676bfd..5bde3ff9 100644 --- a/inst/examples/acotsp/scenario.txt +++ b/inst/examples/acotsp/scenario.txt @@ -13,7 +13,7 @@ execDir = "./acotsp-arena" ## Directory where tuning instances are located, either absolute path or ## relative to current directory. -trainInstancesDir = "./Instances" +trainInstancesDir = "./instances" ## The maximum number of runs (invocations of targetRunner) that will performed. It ## determines the (maximum) budget of experiments for the tuning. @@ -21,7 +21,7 @@ maxExperiments = 5000 ## File that contains a set of initial configurations. If empty or NULL, ## all initial configurations are randomly generated. -# configurationsFile = "" +# configurationsFile = "default.txt" ## A value of 0 silences all debug messages. Higher values provide ## more verbose debug messages. @@ -29,4 +29,3 @@ maxExperiments = 5000 ## END of scenario file ############################################################################ - diff --git a/inst/examples/acotsp/target-runner b/inst/examples/acotsp/target-runner index 95bc163f..414d0303 100755 --- a/inst/examples/acotsp/target-runner +++ b/inst/examples/acotsp/target-runner @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh ############################################################################### # This script is to tune the ACOTSP software. # @@ -17,14 +17,9 @@ error() { exit 1 } - # Path to the ACOTSP executable (this path is relative to the execution directory `execDir` specified in the scenario). # EXE="~/bin/acotsp" -EXE="../ACOTSP-1.03/acotsp" - -# Fixed parameters that should be always passed to ACOTSP. -# The time to be used is always 10 seconds, and we want only one run: -FIXED_PARAMS=" --tries 1 --time 10 --quiet " +EXE="../ACOTSPQAP-master/acotsp" CONFIG_ID="$1" INSTANCE_ID="$2" @@ -34,38 +29,37 @@ INSTANCE="$4" shift 4 || error "Not enough parameters" CONFIG_PARAMS=$* -STDOUT=c${CONFIG_ID}-${INSTANCE_ID}-${SEED}.stdout -STDERR=c${CONFIG_ID}-${INSTANCE_ID}-${SEED}.stderr +# Fixed parameters that should be always passed to ACOTSP. +# The time to be used is always 10 seconds, and we want only one run: +EXE_PARAMS=" --tries 1 --time 10 --quiet -i $INSTANCE --seed $SEED ${CONFIG_PARAMS}" if [ ! -x "${EXE}" ]; then error "${EXE}: not found or not executable (pwd: $(pwd))" fi # Now we can call ACOTSP by building a command line with all parameters for it -$EXE ${FIXED_PARAMS} -i $INSTANCE --seed $SEED ${CONFIG_PARAMS} 1> $STDOUT 2> $STDERR +$EXE ${EXE_PARAMS} 1> $STDOUT 2> $STDERR -# The output of the candidate $CONFIG_ID should be written in the file +# The output of the candidate $CONFIG_ID should be written in the file # c${CONFIG_ID}.stdout (see target runner for ACOTSP). # Does this file exist? if [ ! -s "${STDOUT}" ]; then - # In this case, the file does not exist. Let's exit with a value + # In this case, the file does not exist. Let's exit with a value # different from 0. In this case irace will stop with an error. error "${STDOUT}: No such file or directory" fi # Ok, the file exist. It contains the whole output written by ACOTSP. -# This script should return a single numerical value, the best objective +# This script should return a single numerical value, the best objective # value found by this run of ACOTSP. The following line is to extract # this value from the file containing ACOTSP output. -COST=$(cat ${STDOUT} | grep -o -E 'Best [-+0-9.e]+' | cut -d ' ' -f2) -if ! [[ "$COST" =~ ^[-+0-9.e]+$ ]] ; then +COST=$(grep -o -E 'Best [-+0-9.e]+' "${STDOUT}" | cut -d ' ' -f2) +if [ -z "$COST" ]; then + cat $STDERR error "${STDOUT}: Output is not a number" fi - # Print it! echo "$COST" - # We are done with our duty. Clean files and exit with 0 (no error). -rm -f "${STDOUT}" "${STDERR}" -rm -f best.* stat.* cmp.* +rm -f "${STDOUT}" "${STDERR}" best.* stat.* cmp.* exit 0 diff --git a/inst/examples/slurm/job.sh b/inst/examples/slurm/job.sh index 384087db..5465c456 100644 --- a/inst/examples/slurm/job.sh +++ b/inst/examples/slurm/job.sh @@ -28,5 +28,16 @@ # To load some software (you can show the list with 'module avail'): module load r # You may need to load more modules depending on your application. + +## Option 1. Call the irace executable: +~/bin/irace --exec-dir=~/local/irace-execdir/ --parallel $((SLURM_NTASKS_PER_NODE)) +# I usually add a symbolic link from the location where the irace executable is installed to ~/bin/irace. Otherwise, replace ~/bin/irace with the path to the irace executable. It will be something like: ~/local/R_libs/4.1/irace/bin/irace. How to find this path is explained in the user guide and in the quick-install intro: https://mlopez-ibanez.github.io/irace/#installing-the-irace-package +# You can add additional irace options to the line above (see the user-guide), for example --scenario "tuning-scenario.txt". I added --exec-dir above but you can define the execDir in the scenario file instead. It is important that it is a location that is available for writing from the computing nodes. You may also want to redirect the output of job.sh to some file you can read later. How to do that should be explained in the documentation of your cluster. Also, make sure that you are using the same R version when installing irace than the version that you are loading within job.sh. Otherwise, you may get strange errors. + +# MPI: If you want to use mpi, you may need to add something like: +mpirun -np 1 ~/bin/irace --exec-dir=~/local/irace-execdir/ --parallel $((SLURM_NTASKS-1)) --mpi 1 +# $SLURM_NTASKS is expanded automatically to the value you give to --ntasks=. We subtract one from that value because irace itself consumes one task. + +## Option 2: Using an R script cd /path_to_irace_R_script Rscript ./run_irace.R diff --git a/inst/examples/slurm/run_irace.R b/inst/examples/slurm/run_irace.R index f2d77e3a..8e11ee4f 100644 --- a/inst/examples/slurm/run_irace.R +++ b/inst/examples/slurm/run_irace.R @@ -1,25 +1,42 @@ +# This is an example of launching irace from an R script. library(irace) -# initial set up +# Initial set up. Please check in the documentation of your cluster whether +# $SLURM_NTASKS_PER_NODE is the correct environment variable to get the number +# of tasks that can be run in parallel. ncores <- as.numeric(Sys.getenv("SLURM_NTASKS_PER_NODE", parallel::detectCores())) -parameter_table <- '...' +# This needs to be customized by the user. Instead of defining +# 'parameter_table' you could use a file 'parameters.txt'. +parameter_table <- ' +param1 "--param1 " i (1,100) +param2 "--param2=" c (0,1) +' parameters <- readParameters(text = parameter_table) +# This needs to be customized by the user. Instead of defining targetRunner as +# an R function 'my_target_runner' you could use an executable file +# 'my_target_runner'. my_target_runner <- function(experiment, scenario) { instance <- experiment$instance configuration <- experiment$configuration seed <- experiment$seed - return(list(cost = my_algorithm(instance, configuration, seed))) + cost <- my_algorithm(instance, configuration, seed) + # You could also return 'time' if you are using 'maxTime' in the scenario. + return(list(cost = cost)) } -instances <- load_instances() # Some function that loads or defines the instances. You could also use trainInstancesFile or trainInstancesDir in the scenario. + + +# Some function that loads or defines the instances. You could also use +# trainInstancesFile or trainInstancesDir in the scenario. +instances <- load_instances() scenario <- list(targetRunner = my_target_runner, instances = instances, maxExperiments = 200, - parallel = ncores) + parallel = ncores, parameters = parameters) # check if the scenario is valid (comment out if taking too much time) -checkIraceScenario(scenario, parameters = parameters) +checkIraceScenario(scenario) # run irace -irace.main(scenario = scenario, parameters = parameters) +irace_main(scenario = scenario) diff --git a/inst/exdata/irace-acotsp.Rdata b/inst/exdata/irace-acotsp.Rdata index 90726667..1096fab8 100644 Binary files a/inst/exdata/irace-acotsp.Rdata and b/inst/exdata/irace-acotsp.Rdata differ diff --git a/inst/exdata/log-ablation.Rdata b/inst/exdata/log-ablation.Rdata index a7e8634c..1a496691 100644 Binary files a/inst/exdata/log-ablation.Rdata and b/inst/exdata/log-ablation.Rdata differ diff --git a/inst/exdata/sann.rda b/inst/exdata/sann.rda index 2cf18684..6797ee19 100644 Binary files a/inst/exdata/sann.rda and b/inst/exdata/sann.rda differ diff --git a/inst/templates/scenario.txt.tmpl b/inst/templates/scenario.txt.tmpl index ec85ff94..87149f86 100644 --- a/inst/templates/scenario.txt.tmpl +++ b/inst/templates/scenario.txt.tmpl @@ -61,11 +61,13 @@ ## a multiple of `eachTest`. # firstTest = 5 -## Number of training instances, that make up a `block' in +## Number of training instances, that make up a 'block' in ## `trainInstancesFile`. Elimination of configurations will only be performed ## after evaluating a complete block and never in the middle of a block. Each ## block typically contains one instance from each instance class (type or -## family) and the block size is the number of classes. +## family) and the block size is the number of classes. The value of +## `blockSize` will multiply `firstTest`, `eachTest` and +## `elitistNewInstances`. # blockSize = 1 ## Number of instances evaluated between elimination tests. @@ -158,7 +160,7 @@ # quiet = 0 ## Debug level of the output of `irace`. Set this to 0 to silence all debug -## messages. Higher values provide more verbose debug messages. +## messages. Higher values (1, 2 or 3) provide more verbose debug messages. # debugLevel = 0 ## Seed of the random number generator (by default, generate a random seed). @@ -191,6 +193,10 @@ ## `elitist` is active, `maxTime > 0` and `boundMax > 0`. # capping = NA +## If set to 1, elimination due to capping only happens after `firstTest` +## instances are seen. +# cappingAfterFirstTest = 0 + ## Measure used to obtain the execution bound from the performance of the ## elite configurations: median, mean, worst, best. # cappingType = "median" @@ -244,4 +250,3 @@ ## END of scenario file ############################################################################ - diff --git a/man/ablation.Rd b/man/ablation.Rd index 172ea1de..97ec197e 100644 --- a/man/ablation.Rd +++ b/man/ablation.Rd @@ -20,32 +20,32 @@ ablation( \arguments{ \item{iraceResults}{\code{list()}|\code{character(1)}\cr Object created by \pkg{irace} and typically saved in the log file \code{irace.Rdata}. If a character string is given, then it is interpreted as the path to the log file from which the \code{iraceResults} object will be loaded.} -\item{src, target}{(\code{integer(1)}) Source and target configuration IDs. By default, the first configuration ever evaluated (ID 1) is used as \code{src} and the best configuration found by irace is used as target.} +\item{src, target}{\code{integer(1)|character(1)}\cr Source and target configuration IDs. By default, the first configuration ever evaluated (ID 1) is used as \code{src} and the best configuration found by irace is used as target. If the argument is a string, it is interpreted as the path to a file, with the format specified by \code{\link[=readConfigurationsFile]{readConfigurationsFile()}}, that contains the configuration.} -\item{ab_params}{Specific parameter names to be used for the ablation. They must be in \code{parameters$names}. By default, use all parameters.} +\item{ab_params}{\code{character()}\cr Specific parameter names to be used for the ablation. They must be in \code{parameters$names}. By default, use all parameters.} -\item{type}{Type of ablation to perform: \code{"full"} will execute each configuration on all \code{n_instances} to determine the best-performing one; \code{"racing"} will apply racing to find the best configurations.} +\item{type}{\code{"full"|"racing"}\cr Type of ablation to perform: \code{"full"} will execute each configuration on all \code{n_instances} to determine the best-performing one; \code{"racing"} will apply racing to find the best configurations.} -\item{nrep}{(\code{integer(1)}) Number of replications per instance used in \code{"full"} ablation. When \code{nrep > 1}, each configuration will be executed \code{nrep} times on each instance with different random seeds.} +\item{nrep}{\code{integer(1)}\cr Number of replications per instance used in \code{"full"} ablation. When \code{nrep > 1}, each configuration will be executed \code{nrep} times on each instance with different random seeds.} -\item{seed}{(\code{integer(1)}) Integer value to use as seed for the random number generation.} +\item{seed}{\code{integer(1)}\cr Integer value to use as seed for the random number generation.} -\item{ablationLogFile}{(\code{character(1)}) Log file to save the ablation log. If \code{NULL}, the results are not saved to a file.} +\item{ablationLogFile}{\code{character(1)}\cr Log file to save the ablation log. If \code{NULL}, the results are not saved to a file.} -\item{instancesFile}{(\code{character(1)}) Instances file used for ablation: \code{'train'}, \code{'test'} or a filename containing the list of instances.} +\item{instancesFile}{\code{character(1)}\cr Instances file used for ablation: \code{'train'}, \code{'test'} or a filename containing the list of instances.} \item{...}{Further arguments to override scenario settings, e.g., \code{debugLevel}, \code{parallel}, etc.} } \value{ A list containing the following elements: \describe{ -\item{configurations}{Configurations tested in the ablation.} -\item{instances}{A matrix with the instances used in the experiments. First column has the -instances IDs from \code{iraceResults$scenario$instances}, second column the seed assigned to the instance.} +\item{allConfigurations}{Configurations tested in the ablation.} +\item{state}{State of the ablation process.} \item{experiments}{A matrix with the results of the experiments (columns are configurations, rows are instances).} \item{scenario}{Scenario object with the settings used for the experiments.} \item{trajectory}{IDs of the best configurations at each step of the ablation.} \item{best}{Best configuration found in the experiments.} +\item{complete}{\code{TRUE} if the ablation process was completed.} } } \description{ diff --git a/man/ablation_cmdline.Rd b/man/ablation_cmdline.Rd index b1ad84ae..61d28d88 100644 --- a/man/ablation_cmdline.Rd +++ b/man/ablation_cmdline.Rd @@ -14,13 +14,13 @@ provided on the R command line as a character vector, e.g., \value{ A list containing the following elements: \describe{ -\item{configurations}{Configurations tested in the ablation.} -\item{instances}{A matrix with the instances used in the experiments. First column has the -instances IDs from \code{iraceResults$scenario$instances}, second column the seed assigned to the instance.} +\item{allConfigurations}{Configurations tested in the ablation.} +\item{state}{State of the ablation process.} \item{experiments}{A matrix with the results of the experiments (columns are configurations, rows are instances).} \item{scenario}{Scenario object with the settings used for the experiments.} \item{trajectory}{IDs of the best configurations at each step of the ablation.} \item{best}{Best configuration found in the experiments.} +\item{complete}{\code{TRUE} if the ablation process was completed.} } } \description{ @@ -35,9 +35,11 @@ List of command-line options: \if{html}{\out{
}}\preformatted{-l,--log-file Path to the (.Rdata) file created by irace from which the "iraceResults" object will be loaded. --S,--src Source configuration ID. Default: 1. --T,--target Target configuration ID. By default the best - configuration found by irace. +-S,--src Source configuration ID or the path to a file + containing the configuration. Default: 1. +-T,--target Target configuration ID (by default the best + configuration found by irace) or the path to a file + containing the configuration. -P,--params Specific parameter names to be used for the ablation (separated with commas). By default use all -t,--type Type of ablation to perform: "full" will execute each diff --git a/man/checkParameters.Rd b/man/checkParameters.Rd index 8e646706..0d4eb785 100644 --- a/man/checkParameters.Rd +++ b/man/checkParameters.Rd @@ -7,7 +7,7 @@ checkParameters(parameters) } \arguments{ -\item{parameters}{(\code{list()}) \cr Data structure containing the parameter +\item{parameters}{\code{ParameterSpace}\cr Data structure containing the parameter space definition. The data structure has to similar to the one returned by the function \code{\link{readParameters}}.} } diff --git a/man/configurations_print_command.Rd b/man/configurations_print_command.Rd index e817c7ea..0d8faf76 100644 --- a/man/configurations_print_command.Rd +++ b/man/configurations_print_command.Rd @@ -10,7 +10,7 @@ configurations_print_command(configurations, parameters) \item{configurations}{\code{data.frame}\cr Parameter configurations of the target algorithm (one per row).} -\item{parameters}{(\code{list()}) \cr Data structure containing the parameter +\item{parameters}{\code{ParameterSpace}\cr Data structure containing the parameter space definition. The data structure has to similar to the one returned by the function \code{\link{readParameters}}.} } diff --git a/man/defaultScenario.Rd b/man/defaultScenario.Rd index a4457141..4f146876 100644 --- a/man/defaultScenario.Rd +++ b/man/defaultScenario.Rd @@ -27,7 +27,7 @@ The scenario list contains the following elements: \item{\code{execDir}}{Directory where the programs will be run. (Default: \code{"./"})} \item{\code{logFile}}{File to save tuning results as an R dataset, either absolute path or relative to execDir. (Default: \code{"./irace.Rdata"})} \item{\code{quiet}}{Reduce the output generated by irace to a minimum. (Default: \code{0})} -\item{\code{debugLevel}}{Debug level of the output of \code{irace}. Set this to 0 to silence all debug messages. Higher values provide more verbose debug messages. (Default: \code{0})} +\item{\code{debugLevel}}{Debug level of the output of \code{irace}. Set this to 0 to silence all debug messages. Higher values (1, 2 or 3) provide more verbose debug messages. (Default: \code{0})} \item{\code{seed}}{Seed of the random number generator (by default, generate a random seed). (Default: \code{NA})} \item{\code{repairConfiguration}}{User-defined R function that takes a configuration generated by irace and repairs it. (Default: \code{""})} \item{\code{postselection}}{Perform a postselection race after the execution of irace to consume all remaining budget. Value 0 disables the postselection race. (Default: \code{1})} @@ -59,7 +59,7 @@ The scenario list contains the following elements: \describe{ \item{\code{targetRunner}}{Executable called for each configuration that executes the target algorithm to be tuned. See the templates and examples provided. (Default: \code{"./target-runner"})} \item{\code{targetRunnerLauncher}}{Executable that will be used to launch the target runner, when \code{targetRunner} cannot be executed directly (e.g., a Python script in Windows). (Default: \code{""})} -\item{\code{targetCmdline}}{Command-line arguments provided to \code{targetRunner} (or \code{targetRunnerLauncher} if defined). The substrings \code{\{configurationID\}}, \code{\{instanceID\}}, \code{\{seed\}}, \code{\{instance\}}, and \code{\{bound\}} will be replaced by their corresponding values. The substring \code{\{targetRunnerArgs\}} will be replaced by the concatenation of the switch and value of all active parameters of the particular configuration being evaluated. The substring \code{\{targetRunner\}}, if present, will be replaced by the value of \code{targetRunner} (useful when using \code{targetRunnerLauncher}). (Default: \code{"{configurationID} {instanceID} {seed} {instance} {bound} {targetRunnerArgs}"})} +\item{\code{targetCmdline}}{Command-line arguments provided to \code{targetRunner} (or \code{targetRunnerLauncher} if defined). The substrings \verb{\\\{configurationID\\\}}, \verb{\\\{instanceID\\\}}, \verb{\\\{seed\\\}}, \verb{\\\{instance\\\}}, and \verb{\\\{bound\\\}} will be replaced by their corresponding values. The substring \verb{\\\{targetRunnerArgs\\\}} will be replaced by the concatenation of the switch and value of all active parameters of the particular configuration being evaluated. The substring \verb{\\\{targetRunner\\\}}, if present, will be replaced by the value of \code{targetRunner} (useful when using \code{targetRunnerLauncher}). (Default: \code{"{configurationID} {instanceID} {seed} {instance} {bound} {targetRunnerArgs}"})} \item{\code{targetRunnerRetries}}{Number of times to retry a call to \code{targetRunner} if the call failed. (Default: \code{0})} \item{\code{targetRunnerTimeout}}{Timeout in seconds of any \code{targetRunner} call (only applies to \code{target-runner} executables not to R functions), ignored if 0. (Default: \code{0})} \item{\code{targetRunnerData}}{Optional data passed to \code{targetRunner}. This is ignored by the default \code{targetRunner} function, but it may be used by custom \code{targetRunner} functions to pass persistent data around. (Default: \code{""})} @@ -81,7 +81,45 @@ The scenario list contains the following elements: \item{\code{instances}}{Character vector of the instances to be used in the \code{targetRunner}. (Default: \code{""})} \item{\code{trainInstancesDir}}{Directory where training instances are located; either absolute path or relative to current directory. If no \code{trainInstancesFiles} is provided, all the files in \code{trainInstancesDir} will be listed as instances. (Default: \code{""})} \item{\code{trainInstancesFile}}{File that contains a list of training instances and optionally additional parameters for them. If \code{trainInstancesDir} is provided, \code{irace} will search for the files in this folder. (Default: \code{""})} -\item{\code{blockSize}}{Number of training instances, that make up a \verb{block' in \code{trainInstancesFile}. Elimination of configurations will only be performed after evaluating a complete block and never in the middle of a block. Each block typically contains one instance from each instance class (type or family) and the block size is the number of classes. (Default: }1\verb{)\} \} \\item Tuning budget: \\describe\{ \\item\{}maxExperiments\verb{\}\{Maximum number of runs (invocations of \code{targetRunner}) that will be performed. It determines the maximum budget of experiments for the tuning. (Default: }0\verb{)\} \\item\{}minExperiments\verb{\}\{Minimum number of runs (invocations of \code{targetRunner}) that will be performed. It determines the minimum budget of experiments for the tuning. The actual budget depends on the number of parameters and \code{minSurvival}. (Default: }NA\verb{)\} \\item\{}maxTime\verb{\}\{Maximum total execution time for the executions of \code{targetRunner}. \code{targetRunner} must return two values: cost and time. This value and the one returned by \code{targetRunner} must use the same units (seconds, minutes, iterations, evaluations, ...). (Default: }0\verb{)\} \\item\{}budgetEstimation\verb{\}\{Fraction (smaller than 1) of the budget used to estimate the mean computation time of a configuration. Only used when \code{maxTime} > 0 (Default: }0.05\verb{)\} \\item\{}minMeasurableTime\verb{\}\{Minimum time unit that is still (significantly) measureable. (Default: }0.01\verb{)\} \} \\item Statistical test: \\describe\{ \\item\{}testType\verb{\}\{Statistical test used for elimination. The default value selects \code{t-test} if \code{capping} is enabled or \code{F-test}, otherwise. Valid values are: F-test (Friedman test), t-test (pairwise t-tests with no correction), t-test-bonferroni (t-test with Bonferroni's correction for multiple comparisons), t-test-holm (t-test with Holm's correction for multiple comparisons). (Default: }""\verb{)\} \\item\{}firstTest\verb{\}\{Number of instances evaluated before the first elimination test. It must be a multiple of \code{eachTest}. (Default: }5\verb{)\} \\item\{}eachTest\verb{\}\{Number of instances evaluated between elimination tests. (Default: }1\verb{)\} \\item\{}confidence\verb{\}\{Confidence level for the elimination test. (Default: }0.95\verb{)\} \} \\item Adaptive capping: \\describe\{ \\item\{}capping\verb{\}\{Enable the use of adaptive capping, a technique designed for minimizing the computation time of configurations. Capping is enabled by default if \code{elitist} is active, \code{maxTime > 0} and \code{boundMax > 0}. (Default: }NA\verb{)\} \\item\{}cappingType\verb{\}\{Measure used to obtain the execution bound from the performance of the elite configurations.\\itemize\{\\item median: Median performance of the elite configurations.\\item mean: Mean performance of the elite configurations.\\item best: Best performance of the elite configurations.\\item worst: Worst performance of the elite configurations.\} (Default: }"median"\verb{)\} \\item\{}boundType\verb{\}\{Method to calculate the mean performance of elite configurations.\\itemize\{\\item candidate: Mean execution times across the executed instances and the current one.\\item instance: Execution time of the current instance.\} (Default: }"candidate"\verb{)\} \\item\{}boundMax\verb{\}\{Maximum execution bound for \code{targetRunner}. It must be specified when capping is enabled. (Default: }0\verb{)\} \\item\{}boundDigits\verb{\}\{Precision used for calculating the execution time. It must be specified when capping is enabled. (Default: }0\verb{)\} \\item\{}boundPar\verb{\}\{Penalization constant for timed out executions (executions that reach \code{boundMax} execution time). (Default: }1\verb{)\} \\item\{}boundAsTimeout\verb{\}\{Replace the configuration cost of bounded executions with \code{boundMax}. (Default: }1\verb{)\} \} \\item Recovery: \\describe\{ \\item\{}recoveryFile\verb{\}\{Previously saved log file to recover the execution of \code{irace}, either absolute path or relative to the current directory. If empty or \code{NULL}, recovery is not performed. (Default: }""\verb{)\} \} \\item Testing: \\describe\{ \\item\{}testInstancesDir\verb{\}\{Directory where testing instances are located, either absolute or relative to current directory. (Default: }""\verb{)\} \\item\{}testInstancesFile\verb{\}\{File containing a list of test instances and optionally additional parameters for them. (Default: }""\verb{)\} \\item\{}testInstances\verb{\}\{Character vector of the instances to be used in the \code{targetRunner} when executing the testing. (Default: }""\verb{)\} \\item\{}testNbElites\verb{\}\{Number of elite configurations returned by irace that will be tested if test instances are provided. (Default: }1\verb{)\} \\item\{}testIterationElites\verb{\}\{Enable/disable testing the elite configurations found at each iteration. (Default: }0`)} +\item{\code{blockSize}}{Number of training instances, that make up a 'block' in \code{trainInstancesFile}. Elimination of configurations will only be performed after evaluating a complete block and never in the middle of a block. Each block typically contains one instance from each instance class (type or family) and the block size is the number of classes. The value of \code{blockSize} will multiply \code{firstTest}, \code{eachTest} and \code{elitistNewInstances}. (Default: \code{1})} +} +\item Tuning budget: +\describe{ +\item{\code{maxExperiments}}{Maximum number of runs (invocations of \code{targetRunner}) that will be performed. It determines the maximum budget of experiments for the tuning. (Default: \code{0})} +\item{\code{minExperiments}}{Minimum number of runs (invocations of \code{targetRunner}) that will be performed. It determines the minimum budget of experiments for the tuning. The actual budget depends on the number of parameters and \code{minSurvival}. (Default: \code{NA})} +\item{\code{maxTime}}{Maximum total execution time for the executions of \code{targetRunner}. \code{targetRunner} must return two values: cost and time. This value and the one returned by \code{targetRunner} must use the same units (seconds, minutes, iterations, evaluations, ...). (Default: \code{0})} +\item{\code{budgetEstimation}}{Fraction (smaller than 1) of the budget used to estimate the mean computation time of a configuration. Only used when \code{maxTime} > 0 (Default: \code{0.05})} +\item{\code{minMeasurableTime}}{Minimum time unit that is still (significantly) measureable. (Default: \code{0.01})} +} +\item Statistical test: +\describe{ +\item{\code{testType}}{Statistical test used for elimination. The default value selects \code{t-test} if \code{capping} is enabled or \code{F-test}, otherwise. Valid values are: F-test (Friedman test), t-test (pairwise t-tests with no correction), t-test-bonferroni (t-test with Bonferroni's correction for multiple comparisons), t-test-holm (t-test with Holm's correction for multiple comparisons). (Default: \code{""})} +\item{\code{firstTest}}{Number of instances evaluated before the first elimination test. It must be a multiple of \code{eachTest}. (Default: \code{5})} +\item{\code{eachTest}}{Number of instances evaluated between elimination tests. (Default: \code{1})} +\item{\code{confidence}}{Confidence level for the elimination test. (Default: \code{0.95})} +} +\item Adaptive capping: +\describe{ +\item{\code{capping}}{Enable the use of adaptive capping, a technique designed for minimizing the computation time of configurations. Capping is enabled by default if \code{elitist} is active, \code{maxTime > 0} and \code{boundMax > 0}. (Default: \code{NA})} +\item{\code{cappingAfterFirstTest}}{If set to 1, elimination due to capping only happens after \code{firstTest} instances are seen. (Default: \code{0})} +\item{\code{cappingType}}{Measure used to obtain the execution bound from the performance of the elite configurations.\itemize{\item median: Median performance of the elite configurations.\item mean: Mean performance of the elite configurations.\item best: Best performance of the elite configurations.\item worst: Worst performance of the elite configurations.} (Default: \code{"median"})} +\item{\code{boundType}}{Method to calculate the mean performance of elite configurations.\itemize{\item candidate: Mean execution times across the executed instances and the current one.\item instance: Execution time of the current instance.} (Default: \code{"candidate"})} +\item{\code{boundMax}}{Maximum execution bound for \code{targetRunner}. It must be specified when capping is enabled. (Default: \code{0})} +\item{\code{boundDigits}}{Precision used for calculating the execution time. It must be specified when capping is enabled. (Default: \code{0})} +\item{\code{boundPar}}{Penalization constant for timed out executions (executions that reach \code{boundMax} execution time). (Default: \code{1})} +\item{\code{boundAsTimeout}}{Replace the configuration cost of bounded executions with \code{boundMax}. (Default: \code{1})} +} +\item Recovery: +\describe{ +\item{\code{recoveryFile}}{Previously saved log file to recover the execution of \code{irace}, either absolute path or relative to the current directory. If empty or \code{NULL}, recovery is not performed. (Default: \code{""})} +} +\item Testing: +\describe{ +\item{\code{testInstancesDir}}{Directory where testing instances are located, either absolute or relative to current directory. (Default: \code{""})} +\item{\code{testInstancesFile}}{File containing a list of test instances and optionally additional parameters for them. (Default: \code{""})} +\item{\code{testInstances}}{Character vector of the instances to be used in the \code{targetRunner} when executing the testing. (Default: \code{""})} +\item{\code{testNbElites}}{Number of elite configurations returned by irace that will be tested if test instances are provided. (Default: \code{1})} +\item{\code{testIterationElites}}{Enable/disable testing the elite configurations found at each iteration. (Default: \code{0})} } } } diff --git a/man/getConfigurationById.Rd b/man/getConfigurationById.Rd index 318617e0..0845ee63 100644 --- a/man/getConfigurationById.Rd +++ b/man/getConfigurationById.Rd @@ -9,21 +9,22 @@ getConfigurationById(iraceResults, ids, drop.metadata = FALSE) \arguments{ \item{iraceResults}{\code{list()}|\code{character(1)}\cr Object created by \pkg{irace} and typically saved in the log file \code{irace.Rdata}. If a character string is given, then it is interpreted as the path to the log file from which the \code{iraceResults} object will be loaded.} -\item{ids}{(\code{integer()})\cr The id or a vector of ids of the candidates configurations to obtain.} +\item{ids}{\code{integer()}\cr The id or a vector of ids of the candidates configurations to obtain.} \item{drop.metadata}{\code{logical(1)}\cr Remove metadata, such as the configuration ID and the ID of the parent, from the returned configurations. See \code{\link[=removeConfigurationsMetaData]{removeConfigurationsMetaData()}}.} } \value{ -A data frame containing the elite configurations required. +A data frame containing the elite configurations required, in the +order and with the repetitions given by \code{ids}. } \description{ Returns the configurations selected by ID. } \examples{ log_file <- system.file("exdata/irace-acotsp.Rdata", package="irace", mustWork=TRUE) -getConfigurationById(log_file, ids = c(1,2), drop.metadata = TRUE) +getConfigurationById(log_file, ids = c(2,1), drop.metadata = TRUE) } \author{ diff --git a/man/getConfigurationByIteration.Rd b/man/getConfigurationByIteration.Rd index 6f7afe38..5259318c 100644 --- a/man/getConfigurationByIteration.Rd +++ b/man/getConfigurationByIteration.Rd @@ -9,7 +9,7 @@ getConfigurationByIteration(iraceResults, iterations, drop.metadata = FALSE) \arguments{ \item{iraceResults}{\code{list()}|\code{character(1)}\cr Object created by \pkg{irace} and typically saved in the log file \code{irace.Rdata}. If a character string is given, then it is interpreted as the path to the log file from which the \code{iraceResults} object will be loaded.} -\item{iterations}{(\code{integer()})\cr The iteration number or a vector of iteration numbers from where +\item{iterations}{\code{integer()}\cr The iteration number or a vector of iteration numbers from where the configurations should be obtained. Negative values start counting from the last iteration.} \item{drop.metadata}{\code{logical(1)}\cr Remove metadata, such as the diff --git a/man/getFinalElites.Rd b/man/getFinalElites.Rd index ad01f298..9a65bb95 100644 --- a/man/getFinalElites.Rd +++ b/man/getFinalElites.Rd @@ -9,7 +9,7 @@ getFinalElites(iraceResults, n = 0L, drop.metadata = FALSE) \arguments{ \item{iraceResults}{\code{list()}|\code{character(1)}\cr Object created by \pkg{irace} and typically saved in the log file \code{irace.Rdata}. If a character string is given, then it is interpreted as the path to the log file from which the \code{iraceResults} object will be loaded.} -\item{n}{Number of elite configurations to return, if \code{n} is larger than the +\item{n}{\code{integer(1)}\cr Number of elite configurations to return, if \code{n} is larger than the number of configurations, then only the existing ones are returned. The default (\code{n=0}) returns all of them.} \item{drop.metadata}{\code{logical(1)}\cr Remove metadata, such as the diff --git a/man/get_instanceID_seed_pairs.Rd b/man/get_instanceID_seed_pairs.Rd index 276ed05e..7fb7474c 100644 --- a/man/get_instanceID_seed_pairs.Rd +++ b/man/get_instanceID_seed_pairs.Rd @@ -10,9 +10,9 @@ get_instanceID_seed_pairs(iraceResults, index, instances = FALSE) \arguments{ \item{iraceResults}{\code{list()}|\code{character(1)}\cr Object created by \pkg{irace} and typically saved in the log file \code{irace.Rdata}. If a character string is given, then it is interpreted as the path to the log file from which the \code{iraceResults} object will be loaded.} -\item{index}{(\code{integer()})\cr Indexes of the (instanceID,seed) pairs to be returned. The default returns everything.} +\item{index}{\code{integer()}\cr Indexes of the (instanceID,seed) pairs to be returned. The default returns everything.} -\item{instances}{(\code{logical(1)})\cr Whether to add the actual instances as an additional column (only if the instances are of atomic type).} +\item{instances}{\code{logical(1)}\cr Whether to add the actual instances as an additional column (only if the instances are of atomic type).} } \value{ \code{data.table()}\cr With default arguments, a \code{data.table} containing two columns diff --git a/man/irace.Rd b/man/irace.Rd index 1bd337c3..31a46563 100644 --- a/man/irace.Rd +++ b/man/irace.Rd @@ -91,7 +91,7 @@ the FAQ section in the \href{https://cran.r-project.org/package=irace/vignettes/ } \examples{ \dontrun{ -# In general, there are three steps: +# In general, there are three steps: scenario <- readScenario(filename = "scenario.txt") irace(scenario = scenario) } @@ -121,7 +121,7 @@ f_rastrigin <- function (x) { ## We generate 20 instances (in this case, weights): weights <- rnorm(20, mean = 0.9, sd = 0.02) - + ## On this set of instances, we are interested in optimizing two ## parameters of the SANN algorithm: tmax and temp. We setup the ## parameter space as follows: @@ -140,7 +140,7 @@ target_runner <- function(experiment, scenario) { instance <- experiment$instance configuration <- experiment$configuration - + D <- 3 par <- runif(D, min=-1, max=1) fn <- function(x) { diff --git a/man/irace_cmdline.Rd b/man/irace_cmdline.Rd index a7a365fa..0a0f4ce1 100644 --- a/man/irace_cmdline.Rd +++ b/man/irace_cmdline.Rd @@ -10,11 +10,11 @@ irace_cmdline(argv = commandArgs(trailingOnly = TRUE)) irace.cmdline(argv = commandArgs(trailingOnly = TRUE)) } \arguments{ -\item{argv}{(\code{character()}) \cr The arguments +\item{argv}{\code{character()}\cr The arguments provided on the R command line as a character vector, e.g., \code{c("--scenario", "scenario.txt", "-p", "parameters.txt")}. Using the default value (not providing the parameter) is the -easiest way to call \code{irace_cmdline}.} +easiest way to call \code{\link[=irace_cmdline]{irace_cmdline()}}.} } \value{ (\code{invisible(data.frame)}) @@ -89,9 +89,9 @@ command line used to invoke R. The function reads the parameters given on the command line used to invoke R, finds the name of the scenario file, initializes the scenario from the file (with the function -\code{\link{readScenario}}) and possibly from parameters passed in +\code{\link[=readScenario]{readScenario()}}) and possibly from parameters passed in the command line. It finally starts \pkg{irace} by calling -\code{\link{irace_main}}. +\code{\link[=irace_main]{irace_main()}}. List of command-line options: @@ -147,13 +147,15 @@ List of command-line options: --first-test Number of instances evaluated before the first elimination test. It must be a multiple of `eachTest`. Default: 5. - --block-size Number of training instances, that make up a `block' + --block-size Number of training instances, that make up a 'block' in `trainInstancesFile`. Elimination of configurations will only be performed after evaluating a complete block and never in the middle of a block. Each block typically contains one instance from each instance class (type or family) and the block size is the - number of classes. Default: 1. + number of classes. The value of `blockSize` will + multiply `firstTest`, `eachTest` and + `elitistNewInstances`. Default: 1. --each-test Number of instances evaluated between elimination tests. Default: 1. --target-runner Executable called for each configuration that executes @@ -224,8 +226,8 @@ List of command-line options: -q,--quiet Reduce the output generated by irace to a minimum. Default: 0. --debug-level Debug level of the output of `irace`. Set this to 0 to - silence all debug messages. Higher values provide more - verbose debug messages. Default: 0. + silence all debug messages. Higher values (1, 2 or 3) + provide more verbose debug messages. Default: 0. --seed Seed of the random number generator (by default, generate a random seed). --soft-restart Enable/disable the soft restart strategy that avoids @@ -244,6 +246,9 @@ List of command-line options: designed for minimizing the computation time of configurations. Capping is enabled by default if `elitist` is active, `maxTime > 0` and `boundMax > 0`. + --capping-after-first-test If set to 1, elimination due to capping only + happens after `firstTest` instances are seen. Default: + 0. --capping-type Measure used to obtain the execution bound from the performance of the elite configurations: median, mean, worst, best. Default: median. diff --git a/man/multi_irace.Rd b/man/multi_irace.Rd index 66d8cea9..a90be9f2 100644 --- a/man/multi_irace.Rd +++ b/man/multi_irace.Rd @@ -8,27 +8,27 @@ multi_irace( scenarios, parameters, n = 1L, - parallel = 1, - split_output = parallel > 1, + parallel = 1L, + split_output = parallel > 1L, global_seed = NULL ) } \arguments{ -\item{scenarios}{(\code{list()}) \cr A list of scenarios. +\item{scenarios}{\code{list()}\cr A list of scenarios. If only a single scenario is supplied, it is used for all parameters.} -\item{parameters}{(\code{list()}) \cr A list of parameter space definitions. +\item{parameters}{\code{list()}\cr A list of parameter space definitions. If only a single definition is supplied, it is used for all scenarios.} -\item{n}{(\code{integer(1)}) \cr The number of repetitions.} +\item{n}{\code{integer(1)}\cr The number of repetitions.} -\item{parallel}{(\code{integer(1)}) \cr The number of workers to use. +\item{parallel}{\code{integer(1)}\cr The number of workers to use. A value of \code{1} means sequential execution. Note that \code{parallel > 1} is not supported on Windows.} -\item{split_output}{(\code{logical(1)}) \cr If \code{TRUE}, the output of \code{\link[=irace]{irace()}} is written to \verb{\{execDir\}/run_\{i\}/irace.out} +\item{split_output}{\code{logical(1)}\cr If \code{TRUE}, the output of \code{\link[=irace]{irace()}} is written to \verb{\{execDir\}/run_\{i\}/irace.out} instead of the standard output.} -\item{global_seed}{(\code{integer(1)}) \cr The global seed used to seed the individual runs.} +\item{global_seed}{\code{integer(1)}\cr The global seed used to seed the individual runs.} } \value{ A list of the outputs of \code{\link[=irace]{irace()}}. diff --git a/man/parameters.Rd b/man/parameters.Rd index 01db25d8..1f2a407f 100644 --- a/man/parameters.Rd +++ b/man/parameters.Rd @@ -30,26 +30,26 @@ param_int(name, lower, upper, label = "", condition = TRUE, transf = "") \arguments{ \item{...}{one or more parameters created by \code{param_int()}, \code{param_real()}, \code{param_ord()}, or \code{param_cat()}.} -\item{forbidden}{(\code{expression()|character(1)})\cr String or list of expressions that define forbidden parameter configurations.} +\item{forbidden}{\code{expression()|character(1)}\cr String or list of expressions that define forbidden parameter configurations.} \item{debugLevel}{\code{integer(1)}\cr Larger values produce more verbose output.} -\item{name}{Parameter name (must be alphanumeric).} +\item{name}{\code{character(1)}\cr Parameter name (must be alphanumeric).} -\item{values}{(\code{character()}) \cr Domain as a vector of strings.} +\item{values}{\code{character()}\cr Domain as a vector of strings.} -\item{label}{Label associated to the parameter. Often used to encode a command-line switch that activates the parameter.} +\item{label}{\code{character(1)}\cr Label associated to the parameter. Often used to encode a command-line switch that activates the parameter.} -\item{condition}{(\code{expression(1)|character(1)}) \cr Expression that defines when the parameter is active according to the value of other parameters.} +\item{condition}{\code{expression(1)|character(1)}\cr Expression that defines when the parameter is active according to the value of other parameters.} \item{lower, upper}{Lower and upper limits of the valid domain.} -\item{transf}{(\code{character(1)}) \cr If \code{"log"}, then the parameter is sampled in a logarithmic scale.} +\item{transf}{\code{character(1)}\cr If \code{"log"}, then the parameter is sampled in a logarithmic scale.} \item{digits}{\code{integer(1)}\cr The number of decimal places to be considered for real-valued parameters.} } \value{ -(\code{ParameterSpace}) +\code{ParameterSpace} } \description{ \itemize{ diff --git a/man/printParameters.Rd b/man/printParameters.Rd index 0fb610b5..6a821a0b 100644 --- a/man/printParameters.Rd +++ b/man/printParameters.Rd @@ -7,7 +7,7 @@ printParameters(parameters) } \arguments{ -\item{parameters}{(\code{list()}) \cr Data structure containing the parameter +\item{parameters}{\code{ParameterSpace}\cr Data structure containing the parameter space definition. The data structure has to similar to the one returned by the function \code{\link{readParameters}}.} } @@ -32,7 +32,7 @@ parameters_table <- ' elitistants "--elitistants " i (1, ants) | algorithm == "eas" nnls "--nnls " i (5, 50) | localsearch \%in\% c(1,2,3) dlb "--dlb " c (0, 1) | localsearch \%in\% c(1,2,3) - + [forbidden] (alpha == 0.0) & (beta == 0.0) ' diff --git a/man/psRace.Rd b/man/psRace.Rd index d5166bcc..2daa6137 100644 --- a/man/psRace.Rd +++ b/man/psRace.Rd @@ -2,13 +2,14 @@ % Please edit documentation in R/psRace.R \name{psRace} \alias{psRace} -\title{Postselection race} +\title{Post-selection race} \usage{ psRace( iraceResults, max_experiments, conf_ids = NULL, - iteration_elites = FALSE + iteration_elites = FALSE, + psrace_logFile = NULL ) } \arguments{ @@ -19,32 +20,39 @@ post-selection race. If it is equal to or smaller than 1, then it is a fraction of the total budget given by \code{iraceResults$scenario$maxExperiments} or \code{iraceResults$scenario$maxTime / iraceResults$state$boundEstimate}.} -\item{conf_ids}{IDs of the configurations in iraceResults$allConfigurations to be used for ablation. -If NULL, the \code{iteration_best} argument will be to decide.} +\item{conf_ids}{IDs of the configurations in \code{iraceResults$allConfigurations} to be used for the post-selection. +If \code{NULL}, then the configurations are automatically selected.} -\item{iteration_elites}{If TRUE, only select the best configuration of each iteration. -If FALSE, select from all elite configurations of all iterations. \code{max_experiments}} +\item{iteration_elites}{If \code{FALSE}, give priority to selecting the configurations that were elite in the last iteration. +If \code{TRUE}, select from all elite configurations of all iterations. This parameter only has an effect when \code{conf_ids} is not \code{NULL}.} + +\item{psrace_logFile}{\code{character(1)}\cr Log file to save the post-selection race log. If \code{NULL}, the log is saved in \code{iraceResults$scenario$logFile}.} } \value{ -If iraceLogFile is NULL, it returns a list with the following elements: +The elite configurations after the post-selection. In addition, if \code{iraceResults$scenario$logFile} is defined, +it saves an updated copy of \code{iraceResults} in that file, where \code{iraceResults$psrace_log} is a list with the following elements: \describe{ -\item{configurations}{Configurations used in the race.} -\item{instances}{A matrix with the instances used in the experiments. First column has the -instances ids from iraceResults$scenario$instances, second column the seed assigned to the instance.} -\item{maxExperiments}{Maximum number of experiments set for the race.} -\item{experiments}{A matrix with the results of the experiments (columns are configurations, rows are instances).} +\item{configurations}{Configurations used in the post-selection race.} +\item{instances}{Data frame with the instances used in the post-selection race. First column has the +instances IDs from \code{iraceResults$scenario$instances}, second column the seed assigned to the instance.} +\item{max_experiments}{Configuration budget assigned to the post-selection race.} +\item{experiments}{Matrix of results generated by the post-selection race, in the same format as the matrix \code{iraceResults$experiments}. Column names are the configuration IDs and row names are the instance IDs.} \item{elites}{Best configurations found in the experiments.} } -If \code{iraceLogFile} is provided this list object will be saved in \code{iraceResults$psrace_log}. } \description{ -\code{psRace} performs a postselection race of a set of configurations. +\code{psRace} performs a post-selection race of a set of configurations. } \examples{ \donttest{ - logfile <- system.file(package="irace", "exdata", "sann.rda") - # Execute the postselection after the execution of irace. Use 10\% of the total budget. - psRace(logfile, max_experiments=0.1) + irace_log <- read_logfile(system.file(package="irace", "exdata", "sann.rda")) + # Use a temporary file to not change the original "sann.rda". + psrace_logFile <- withr::local_tempfile(fileext = ".Rdata") + # Execute the post-selection after the execution of irace. Use 10\% of the total budget. + psRace(irace_log, max_experiments=0.1, psrace_logFile = psrace_logFile) + # Print psrace_log + irace_log <- read_logfile(psrace_logFile) + str(irace_log$psrace_log) } } \author{ diff --git a/man/readConfigurationsFile.Rd b/man/readConfigurationsFile.Rd index a9733eba..39267ffc 100644 --- a/man/readConfigurationsFile.Rd +++ b/man/readConfigurationsFile.Rd @@ -10,7 +10,7 @@ readConfigurationsFile(filename, parameters, debugLevel = 0L, text) \item{filename}{\code{character(1)}\cr Filename from which the configurations should be read. The contents should be readable by \code{read.table( , header=TRUE)}.} -\item{parameters}{(\code{list()}) \cr Data structure containing the parameter +\item{parameters}{\code{ParameterSpace}\cr Data structure containing the parameter space definition. The data structure has to similar to the one returned by the function \code{\link{readParameters}}.} diff --git a/man/readParameters.Rd b/man/readParameters.Rd index 171dc602..4c1b21ca 100644 --- a/man/readParameters.Rd +++ b/man/readParameters.Rd @@ -87,7 +87,7 @@ produced by \code{irace}, even with the same random seed. elitistants "--elitistants " i (1, ants) | algorithm == "eas" nnls "--nnls " i (5, 50) | localsearch \%in\% c(1,2,3) dlb "--dlb " c (0, 1) | localsearch \%in\% c(1,2,3) - + [forbidden] (alpha == 0.0) & (beta == 0.0) [global] diff --git a/man/read_ablogfile.Rd b/man/read_ablogfile.Rd index 08de85cc..0e643a6d 100644 --- a/man/read_ablogfile.Rd +++ b/man/read_ablogfile.Rd @@ -7,10 +7,10 @@ read_ablogfile(filename) } \arguments{ -\item{filename}{Filename that contains the log file saved by \code{\link[=ablation]{ablation()}}. Example: \code{log-ablation.Rdata}.} +\item{filename}{\code{character(1)}\cr Filename that contains the log file saved by \code{\link[=ablation]{ablation()}}. Example: \code{log-ablation.Rdata}.} } \value{ -(\code{list()}) +\code{list()} } \description{ Read the log file (\code{log-ablation.Rdata}) produced by \code{\link[=ablation]{ablation()}}. diff --git a/man/read_pcs_file.Rd b/man/read_pcs_file.Rd index b7472e4d..f581ff21 100644 --- a/man/read_pcs_file.Rd +++ b/man/read_pcs_file.Rd @@ -48,13 +48,13 @@ will stop with an error. rasrank [1, 100][1]i elitistants [1, 750][1]i nnls [5, 50][5]i - dlb {0, 1}[1] + dlb {0, 1}[1] Conditionals: q0 | algorithm in {acs} rasrank | algorithm in {ras} elitistants | algorithm in {eas} nnls | localsearch in {1,2,3} - dlb | localsearch in {1,2,3} + dlb | localsearch in {1,2,3} {alpha=0, beta=0}' parameters_table <- read_pcs_file(text=pcs_table) cat(parameters_table) diff --git a/man/save_irace_logfile.Rd b/man/save_irace_logfile.Rd new file mode 100644 index 00000000..ed2bb168 --- /dev/null +++ b/man/save_irace_logfile.Rd @@ -0,0 +1,20 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/utils.R +\name{save_irace_logfile} +\alias{save_irace_logfile} +\title{Save the log generated by \pkg{{irace}} to a file (by default \code{irace.Rdata}).} +\usage{ +save_irace_logfile(iraceResults, logfile) +} +\arguments{ +\item{iraceResults}{\code{list()}\cr Object created by \pkg{irace} and typically saved in the log file \code{irace.Rdata}.} + +\item{logfile}{\code{character(1)}\cr Filename to save \code{iraceResults}. Usually, this is given by \code{scenario$logFile}. If \code{NULL} or \code{""}, no data is saved.} +} +\description{ +This function may be useful if you are manually editing the log data generated by a run of \pkg{irace}. +} +\seealso{ +\code{\link[=read_logfile]{read_logfile()}} +} +\concept{analysis} diff --git a/man/target_evaluator_default.Rd b/man/target_evaluator_default.Rd index 5b73524c..2f83ccba 100644 --- a/man/target_evaluator_default.Rd +++ b/man/target_evaluator_default.Rd @@ -23,8 +23,6 @@ this evaluation, ignore the seed for deterministic algorithms;} \item{\code{bound}}{(only when \code{capping} is enabled) Time bound for the execution;} \item{\code{configuration}}{1-row data frame with a column per parameter name;} -\item{\code{switches}}{Vector of parameter switches (labels) in the order -of parameters used in \code{configuration}.} }} \item{num_configurations}{Number of configurations alive in the race.} diff --git a/man/target_runner_default.Rd b/man/target_runner_default.Rd index 59f56413..6cb3e923 100644 --- a/man/target_runner_default.Rd +++ b/man/target_runner_default.Rd @@ -17,8 +17,6 @@ this evaluation, ignore the seed for deterministic algorithms;} \item{\code{bound}}{(only when \code{capping} is enabled) Time bound for the execution;} \item{\code{configuration}}{1-row data frame with a column per parameter name;} -\item{\code{switches}}{Vector of parameter switches (labels) in the order -of parameters used in \code{configuration}.} }} \item{scenario}{\code{list()}\cr Data structure containing \pkg{irace} diff --git a/man/testing_fromlog.Rd b/man/testing_fromlog.Rd index 9cd8af6a..fc0c207a 100644 --- a/man/testing_fromlog.Rd +++ b/man/testing_fromlog.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/main.R \name{testing_fromlog} \alias{testing_fromlog} -\title{Test configurations given in \code{.Rdata} file} +\title{Test configurations given in the logfile (typically \code{irace.Rdata}) produced by \pkg{irace}.} \usage{ testing_fromlog( logFile, @@ -14,12 +14,12 @@ testing_fromlog( ) } \arguments{ -\item{logFile}{Path to the \code{.Rdata} file produced by \pkg{irace}.} +\item{logFile}{\code{character(1)}\cr Path to the logfile (typically \code{irace.Rdata}) produced by \pkg{irace}.} \item{testNbElites}{Number of (final) elite configurations to test. Overrides the value found in \code{logFile}.} -\item{testIterationElites}{(\code{logical(1)}) If \code{FALSE}, only the final +\item{testIterationElites}{\code{logical(1)}\cr If \code{FALSE}, only the final \code{testNbElites} configurations are tested; otherwise, also test the best configurations of each iteration. Overrides the value found in \code{logFile}.} @@ -30,7 +30,7 @@ configurations of each iteration. Overrides the value found in \code{logFile}.} \item{testInstances}{Character vector of the instances to be used in the \code{targetRunner} when executing the testing.} } \value{ -Boolean. \code{TRUE} if the testing ended successfully otherwise, \code{FALSE}. +\code{logical(1)}\cr \code{TRUE} if the testing ended successfully otherwise, \code{FALSE}. } \description{ \code{testing_fromlog} executes the testing of the target algorithm configurations diff --git a/scripts/generate-options.R b/scripts/generate-options.R index f3b3e573..c78c5b98 100644 --- a/scripts/generate-options.R +++ b/scripts/generate-options.R @@ -4,6 +4,8 @@ irace_params <- fromJSON(txt = "irace_options.json", simplifyDataFrame = TRUE) is.na.nowarn <- function(x) suppressWarnings(is.na(x)) +is.null.or.na <- function(x) (is.null(x) | is.na.nowarn(x)) + is.null.or.na.or.empty <- function(x) (is.null(x) | is.na.nowarn(x) | (x == "")) format.number.or.string <- function(x) @@ -17,7 +19,7 @@ format.number.or.string <- function(x) else return(y) } -to.Rd.text <- function(x) gsub("\\\\code\\{((?:[^{}\\\\]|\\\\{|\\\\})+)\\}", "`\\1`", x, perl=TRUE) +to.Rd.text <- function(x) gsub("\\\\(?:code|parameter)\\{((?:[^{}\\\\]|\\\\{|\\\\})+)\\}", "`\\1`", x, perl=TRUE) to.plain.text <- function(x) gsub("\\}", "}", gsub("\\{", "{", to.Rd.text(x), fixed=TRUE), fixed=TRUE) to_latex <- function(x) gsub("{", "\\{", gsub("}", "\\}", x, fixed=TRUE), fixed=TRUE) @@ -56,10 +58,10 @@ for (section in ordered_sections) { parameters <- parameters[!is.null.or.na.or.empty(parameters[, "man"]) | !is.null.or.na.or.empty(parameters[, "description"]), , drop = FALSE] if (nrow(parameters) < 1) next - sec.text <- apply(parameters, 1, function(x) { + sec.text <- apply(parameters, 1L, function(x) { paste0("#' \\item{`", x["name"], "`}{", ifelse(is.null.or.na.or.empty(x["man"]), - x["description"], x["man"]), + to.Rd.text(x["description"]), x["man"]), " (Default: `", format.number.or.string(x["default"]), "`)}")}) @@ -79,7 +81,7 @@ writeLines(text, con = conf.filename) gendefparameter <- function(x) { - sprintf(" \\defparameter%s{%s}{%s}{%s}%%\n%s\n", + sprintf(" \\defparameter%s{%s}{%s}{%s}%%\n %s\n", ifelse(x["short"] == "", "", # we have to cut the - paste0("[", substring(x["short"], 2), "]")), @@ -90,7 +92,7 @@ gendefparameter <- function(x) # we have to cut the -- substring(x["long"], 3), ifelse(is.na.nowarn(x["default"]), "", to_latex(x["default"])), - ifelse(x["vignettes"] != "", x["vignettes"], x["description"])) + ifelse(is.null.or.na.or.empty(x["vignettes"]), x["description"], x["vignettes"])) } vin.text <-"\n" @@ -101,14 +103,14 @@ for (section in ordered_sections) { "]{", section, "}"), "\\begin{description}") parameters <- irace_params[irace_params[, "section"] == section, ] - - parameters <- parameters[!is.na(parameters[,"vignettes"]), ] - parameters[is.null(parameters[,"vignettes"]) | (parameters[,"vignettes"] == ""), "vignettes"] <- - parameters[(parameters[,"vignettes"] == ""), "description"] - parameters <- parameters[parameters[,"vignettes"] != "",] - if (nrow(parameters) < 1) next - sec.text <- apply(parameters, 1, gendefparameter) + missing_vignettes <- is.null.or.na.or.empty(parameters[,"vignettes"]) + # parameters <- parameters[!is.na(parameters[,"vignettes"]), ] + parameters[missing_vignettes, "vignettes"] <- parameters[missing_vignettes, "description"] + parameters <- parameters[!is.null.or.na.or.empty(parameters[,"vignettes"]), , drop = FALSE] + + if (nrow(parameters) < 1L) next + sec.text <- apply(parameters, 1L, gendefparameter) vin.text <- c(vin.text, sec.text, "\\end{description}") } @@ -127,8 +129,10 @@ out.text <- for (i in seq_len(nrow(irace_params))) { p <- unlist(irace_params[i, ]) if (startsWith(p["name"], ".")) next + if (is.null.or.na(p["description"])) + stop("Missing description for option:", p["name"], "!") if (p["description"] == "") next - if (p["name"] %in% scenario_file_exclude) next + if (p["name"] %in% scenario_file_exclude) next out.text <- c(out.text, strwrap(to.plain.text(p["description"]), width = 79, prefix = "## "), sprintf("# %s = %s\n", p["name"], @@ -136,8 +140,7 @@ for (i in seq_len(nrow(irace_params))) { } out.text <- c(out.text, '## END of scenario file -############################################################################ -') +############################################################################') writeLines(out.text, con = "../inst/templates/scenario.txt.tmpl") @@ -159,9 +162,8 @@ r_text <- sprintf('## This file was generated by scripts/generate-options.R ## FIXME: If these values are special perhaps they should be saved in $state ? .irace.params.recover <- c("instances", "seed", "testInstances", # We need this because this data may mutate - "targetRunnerData", "elitist", "deterministic") -', -paste0(deparse(irace_params), collapse = "\n"), -paste0(deparse(irace.params.names), collapse = "\n")) + "targetRunnerData", "elitist", "deterministic")', +paste0(trimws(deparse(irace_params), which="right"), collapse = "\n"), +paste0(trimws(deparse(irace.params.names), which="right"), collapse = "\n")) writeLines(r_text, con = "../R/irace-options.R") diff --git a/scripts/irace_options.json b/scripts/irace_options.json index 354f8885..ca1e86a1 100644 --- a/scripts/irace_options.json +++ b/scripts/irace_options.json @@ -117,7 +117,7 @@ "long": "--log-file", "default": ".\/irace.Rdata", "description": "File to save tuning results as an R dataset, either absolute path or relative to execDir.", - "vignettes": " File to save tuning results as an \\aR dataset. The provided path must be either an absolute path or relative to \\parameter{execDir}. See \\autoref{sec:output r} for details on the format of the \\aR dataset.", + "vignettes": "File to save tuning results as an \\aR dataset. The provided path must be either an absolute path or relative to \\parameter{execDir}. See \\autoref{sec:output r} for details on the format of the \\aR dataset.", "section": "General options" }, { @@ -127,7 +127,7 @@ "long": "--recovery-file", "default": "", "description": "Previously saved log file to recover the execution of \\code{irace}, either absolute path or relative to the current directory. If empty or \\code{NULL}, recovery is not performed.", - "vignettes": "Previously saved \\irace log file that should be used to recover the execution of \\irace; either absolute path or relative to the current directory. If empty or \\code{NULL}, recovery is not performed. \nFor more details about recovery, see \\autoref{sec:recovery}.", + "vignettes": "Previously saved \\irace log file that should be used to recover the execution of \\irace; either absolute path or relative to the current directory. If empty or \\code{NULL}, recovery is not performed.\nFor more details about recovery, see \\autoref{sec:recovery}.", "section": "Recovery" }, { @@ -177,7 +177,6 @@ "long": "--test-instances-dir", "default": "", "description": "Directory where testing instances are located, either absolute or relative to current directory.", - "vignettes": "Directory where testing instances are located, either absolute or relative to the current directory.", "section": "Testing" }, { @@ -187,7 +186,6 @@ "long": "--test-instances-file", "default": "", "description": "File containing a list of test instances and optionally additional parameters for them.", - "vignettes": "File containing a list of test instances and, optionally, additional parameters for them.", "section": "Testing" }, { @@ -207,8 +205,8 @@ "long": "--test-num-elites", "default": 1, "description": "Number of elite configurations returned by irace that will be tested if test instances are provided.", - "vignettes": " Number of elite configurations returned by irace that will be tested if test instances are provided. For more information about the testing, see \\autoref{sec:testing}.", - "section": "Testing" + "vignettes": "Number of elite configurations returned by irace that will be tested if test instances are provided. For more information about the testing, see \\autoref{sec:testing}.", + "section": "Testing" }, { "name": "testIterationElites", @@ -217,7 +215,6 @@ "long": "--test-iteration-elites", "default": 0, "description": "Enable\/disable testing the elite configurations found at each iteration.", - "vignettes": "Enable\/disable testing the elite configurations found at each iteration.", "section": "Testing" }, { @@ -225,10 +222,10 @@ "type": "s", "short": "", "long": "--test-type", - "default": "", - "domain": "F-test,t-test,t-test-holm,t-test-bonferroni", + "default": "", + "domain": "F-test,t-test,t-test-holm,t-test-bonferroni", "description": "Statistical test used for elimination. The default value selects \\code{t-test} if \\code{capping} is enabled or \\code{F-test}, otherwise. Valid values are: F-test (Friedman test), t-test (pairwise t-tests with no correction), t-test-bonferroni (t-test with Bonferroni's correction for multiple comparisons), t-test-holm (t-test with Holm's correction for multiple comparisons).", - "vignettes": "Specifies the statistical test used for elimination:\n\\begin{itemize}\n\\item[] \\code{F-test} (Friedman test)\n\\item[] \\code{t-test} (pairwise t-tests with no correction)\n\\item[] \\code{t-test-bonferroni} (t-test with Bonferroni's correction for multiple comparisons)\n\\item[] \\code{t-test-holm} (t-test with Holm's correction for multiple comparisons).\n\\end{itemize}\n We recommend to not use corrections for multiple comparisons because the test typically becomes too strict and the search stagnates. \nSee \\autoref{sec:stat test} for details about choosing the statistical test most appropriate for your scenario.\n\\begin{xwarningbox}\n The default setting of \\parameter{testType} is \\code{F-test} unless the \\parameter{capping} option is enabled in which case, the default setting is \\code{t-test}.\n\\end{xwarningbox}", + "vignettes": "Specifies the statistical test used for elimination:\n\\begin{itemize}\n\\item[] \\code{F-test} (Friedman test)\n\\item[] \\code{t-test} (pairwise t-tests with no correction)\n\\item[] \\code{t-test-bonferroni} (t-test with Bonferroni's correction for multiple comparisons)\n\\item[] \\code{t-test-holm} (t-test with Holm's correction for multiple comparisons).\n\\end{itemize}\n We recommend to not use corrections for multiple comparisons because the test typically becomes too strict and the search stagnates.\nSee \\autoref{sec:stat test} for details about choosing the statistical test most appropriate for your scenario.\n\\begin{xwarningbox}\n The default setting of \\parameter{testType} is \\code{F-test} unless the \\parameter{capping} option is enabled in which case, the default setting is \\code{t-test}.\n\\end{xwarningbox}", "section": "Statistical test" }, { @@ -238,7 +235,7 @@ "long": "--first-test", "default": 5, "description": "Number of instances evaluated before the first elimination test. It must be a multiple of \\code{eachTest}.", - "vignettes": " Specifies how many instances are evaluated before the first elimination test.\n\\begin{xwarningbox}\nThe value of \\parameter{firstTest} must be a multiple of \\parameter{eachTest}.\n\\end{xwarningbox}", + "vignettes": "Specifies how many instances are evaluated before the first elimination test.\n\\begin{xwarningbox}\nThe value of \\parameter{firstTest} must be a multiple of \\parameter{eachTest}.\n\\end{xwarningbox}", "section": "Statistical test" }, { @@ -247,7 +244,7 @@ "short": "", "long": "--block-size", "default": 1, - "description": "Number of training instances, that make up a `block' in \\code{trainInstancesFile}. Elimination of configurations will only be performed after evaluating a complete block and never in the middle of a block. Each block typically contains one instance from each instance class (type or family) and the block size is the number of classes.", + "description": "Number of training instances, that make up a 'block' in \\parameter{trainInstancesFile}. Elimination of configurations will only be performed after evaluating a complete block and never in the middle of a block. Each block typically contains one instance from each instance class (type or family) and the block size is the number of classes. The value of \\parameter{blockSize} will multiply \\parameter{firstTest}, \\parameter{eachTest} and \\parameter{elitistNewInstances}.", "section": "Training instances" }, { @@ -276,9 +273,8 @@ "short": "", "long": "--target-runner-launcher", "default": "", - "vignettes": "Executable that will be used to launch the target runner, when \\parameter{targetRunner} cannot be executed directly (e.g., a Python script in Windows).", - "section": "Target algorithm execution", - "description": "Executable that will be used to launch the target runner, when \\code{targetRunner} cannot be executed directly (e.g., a Python script in Windows)." + "description": "Executable that will be used to launch the target runner, when \\parameter{targetRunner} cannot be executed directly (e.g., a Python script in Windows).", + "section": "Target algorithm execution" }, { "name": "targetCmdline", @@ -296,9 +292,8 @@ "short": "", "long": "--target-runner-retries", "default": 0, - "vignettes": "Number of times to retry a call to \\parameter{targetRunner} if the call failed.", - "section": "Target algorithm execution", - "description": "Number of times to retry a call to \\code{targetRunner} if the call failed." + "description": "Number of times to retry a call to \\parameter{targetRunner} if the call failed.", + "section": "Target algorithm execution" }, { "name": "targetRunnerTimeout", @@ -307,8 +302,7 @@ "long": "--target-runner-timeout", "default": 0, "section": "Target algorithm execution", - "vignettes": "Timeout in seconds of any \\parameter{targetRunner} call (only applies to \\code{target-runner} executables not to R functions), ignored if 0.", - "description": "Timeout in seconds of any \\code{targetRunner} call (only applies to \\code{target-runner} executables not to R functions), ignored if 0." + "description": "Timeout in seconds of any \\parameter{targetRunner} call (only applies to \\code{target-runner} executables not to R functions), ignored if 0." }, { "name": "targetRunnerData", @@ -316,9 +310,8 @@ "short": "", "long": "", "default": "", - "vignettes": "Optional data passed to \\parameter{targetRunner}. This is ignored by the default \\parameter{targetRunner} function, but it may be used by custom \\parameter{targetRunner} functions to pass persistent data around.", - "section": "Target algorithm execution", - "description": "Optional data passed to \\code{targetRunner}. This is ignored by the default \\code{targetRunner} function, but it may be used by custom \\code{targetRunner} functions to pass persistent data around." + "description": "Optional data passed to \\parameter{targetRunner}. This is ignored by the default \\parameter{targetRunner} function, but it may be used by custom \\parameter{targetRunner} functions to pass persistent data around.", + "section": "Target algorithm execution" }, { "name": "targetRunnerParallel", @@ -326,7 +319,7 @@ "short": "", "long": "", "default": "", - "vignettes": " Optional \\aR function to provide custom parallelization of \\parameter{targetRunner}. See \\autoref{sec:parallel} for more information.", + "vignettes": "Optional \\aR function to provide custom parallelization of \\parameter{targetRunner}. See \\autoref{sec:parallel} for more information.", "section": "Target algorithm execution", "description": "Optional R function to provide custom parallelization of \\code{targetRunner}." }, @@ -347,7 +340,7 @@ "long": "--deterministic", "default": 0, "description": "If the target algorithm is deterministic, configurations will be evaluated only once per instance.", - "vignettes": " Enable\/disable deterministic target algorithm mode. If the target algorithm is deterministic, configurations will be evaluated only once per instance. See \\autoref{sec:training} for more information.\n\\begin{xwarningbox}\n If the number of instances provided is less than the value specified for the option \\parameter{firstTest}, no statistical test will be performed.\n\\end{xwarningbox}", + "vignettes": "Enable\/disable deterministic target algorithm mode. If the target algorithm is deterministic, configurations will be evaluated only once per instance. See \\autoref{sec:training} for more information.\n\\begin{xwarningbox}\n If the number of instances provided is less than the value specified for the option \\parameter{firstTest}, no statistical test will be performed.\n\\end{xwarningbox}", "section": "Target algorithm execution" }, { @@ -397,7 +390,6 @@ "long": "--min-measurable-time", "default": 0.01, "description": "Minimum time unit that is still (significantly) measureable.", - "vignettes": "Minimum time unit that is still (significantly) measureable.", "section": "Tuning budget" }, { @@ -417,7 +409,7 @@ "long": "--load-balancing", "default": 1, "description": "Enable\/disable load-balancing when executing experiments in parallel. Load-balancing makes better use of computing resources, but increases communication overhead. If this overhead is large, disabling load-balancing may be faster.", - "vignettes": " Enable\/disable load-balancing when executing experiments in parallel. Load-balancing makes better use of computing resources, but increases communication overhead. If this overhead is large, disabling load-balancing may be faster. See \\autoref{sec:parallel}.", + "vignettes": "Enable\/disable load-balancing when executing experiments in parallel. Load-balancing makes better use of computing resources, but increases communication overhead. If this overhead is large, disabling load-balancing may be faster. See \\autoref{sec:parallel}.", "section": "Target algorithm execution" }, { @@ -427,7 +419,7 @@ "long": "--mpi", "default": 0, "description": "Enable\/disable MPI. Use \\code{Rmpi} to execute \\code{targetRunner} in parallel (parameter \\code{parallel} is the number of slaves).", - "vignettes": " Enable\/disable use of \\pkg{Rmpi} to execute the \\parameter{targetRunner} in parallel using MPI protocol. When \\parameter{mpi} is enabled, the option \\parameter{parallel} is the number of slave nodes. See \\autoref{sec:parallel}.", + "vignettes": "Enable\/disable use of \\pkg{Rmpi} to execute the \\parameter{targetRunner} in parallel using MPI protocol. When \\parameter{mpi} is enabled, the option \\parameter{parallel} is the number of slave nodes. See \\autoref{sec:parallel}.", "section": "Target algorithm execution" }, { @@ -437,7 +429,7 @@ "long": "--batchmode", "default": 0, "domain": "sge,pbs,torque,slurm,htcondor", - "vignettes": " Specify how irace waits for jobs to finish when \\parameter{targetRunner} submits jobs to a batch cluster: \\code{sge}, \\code{pbs}, \\code{torque}, \\code{slurm} or \\code{htcondor} (\\parameter{targetRunner} must submit jobs to the cluster using. for example, \\code{qsub}). See \\autoref{sec:parallel}.", + "vignettes": "Specify how irace waits for jobs to finish when \\parameter{targetRunner} submits jobs to a batch cluster: \\code{sge}, \\code{pbs}, \\code{torque}, \\code{slurm} or \\code{htcondor} (\\parameter{targetRunner} must submit jobs to the cluster using. for example, \\code{qsub}). See \\autoref{sec:parallel}.", "section": "Target algorithm execution", "description": "Specify how irace waits for jobs to finish when \\code{targetRunner} submits jobs to a batch cluster: sge, pbs, torque, slurm or htcondor. \\code{targetRunner} must submit jobs to the cluster using, for example, \\code{qsub}." }, @@ -456,9 +448,9 @@ "short": "", "long": "--debug-level", "default": 0, - "vignettes": "Level of information to display in the text output of \\irace. A value of 0 silences all debug messages. Higher values provide more verbose debug messages. Details about the text output of \\irace are given in \\autoref{sec:output text}.", + "vignettes": "Level of information to display in the text output of \\irace. A value of 0 silences all debug messages. Higher values (1, 2 or 3) provide more verbose debug messages. Details about the text output of \\irace are given in \\autoref{sec:output text}.", "section": "General options", - "description": "Debug level of the output of \\code{irace}. Set this to 0 to silence all debug messages. Higher values provide more verbose debug messages." + "description": "Debug level of the output of \\code{irace}. Set this to 0 to silence all debug messages. Higher values (1, 2 or 3) provide more verbose debug messages." }, { "name": "seed", @@ -477,7 +469,7 @@ "long": "--soft-restart", "default": 1, "description": "Enable\/disable the soft restart strategy that avoids premature convergence of the probabilistic model.", - "vignettes": " Enable\/disable the soft-restart strategy that avoids premature convergence of the probabilistic model. When a sampled configuration is \\emph{similar} to its parent configuration, the probabilistic model of these configurations is soft restarted. The soft-restart mechanism is explained in the \\irace paper~\\citep{LopDubPerStuBir2016irace}. The similarity of categorical and ordinal parameters is given by the hamming distance, and the option \\parameter{softRestartThreshold} defines the similarity of numerical parameters.", + "vignettes": "Enable\/disable the soft-restart strategy that avoids premature convergence of the probabilistic model. When a sampled configuration is \\emph{similar} to its parent configuration, the probabilistic model of these configurations is soft restarted. The soft-restart mechanism is explained in the \\irace paper~\\citep{LopDubPerStuBir2016irace}. The similarity of categorical and ordinal parameters is given by the hamming distance, and the option \\parameter{softRestartThreshold} defines the similarity of numerical parameters.", "section": "Internal \\irace options" }, { @@ -486,9 +478,8 @@ "short": "", "long": "--soft-restart-threshold", "default": 0.0001, - "vignettes": "Soft restart threshold value for numerical parameters.", - "section": "Internal \\irace options", - "description": "Soft restart threshold value for numerical parameters." + "description": "Soft restart threshold value for numerical parameters.", + "section": "Internal \\irace options" }, { "name": "elitist", @@ -497,7 +488,7 @@ "long": "--elitist", "default": 1, "description": "Enable\/disable elitist irace.", - "vignettes": "Enable\/disable elitist \\irace.\n\n In the \\textbf{elitist} version of \\code{irace}~\\citep{LopDubPerStuBir2016irace}, elite configurations are not discarded from the race until non-elite configurations have been executed on the same instances as the elite configurations.\n\n Each race begins by evaluating all configurations on a number of new instances. This number is defined by the option \\parameter{elitistNewInstances}. After the new instances have been evaluated, configurations are evaluated on instances seen in the previous race. Elite configurations already have results for most of these previous instances and, therefore, do not need to be re-evaluated. Finally, after configurations have been evaluated on all these instances, the race continues by evaluating additional new instances. \n\n The statistical tests can be performed at any moment during the race according to the setting of the options \\parameter{firstTest} and \\parameter{eachTest}. The elitist rule forbids discarding elite configurations, even if the show poor performance, until the last of the previous instances is seen in the race. \n\n The \\textbf{non-elitist} version of \\irace can discard elite configurations at any point of the race, instances are not re-used from one race to the next, and new instances are sampled for each race.", + "vignettes": "Enable\/disable elitist \\irace.\n\n In the \\textbf{elitist} version of \\code{irace}~\\citep{LopDubPerStuBir2016irace}, elite configurations are not discarded from the race until non-elite configurations have been executed on the same instances as the elite configurations.\n\n Each race begins by evaluating all configurations on a number of new instances. This number is defined by the option \\parameter{elitistNewInstances}. After the new instances have been evaluated, configurations are evaluated on instances seen in the previous race. Elite configurations already have results for most of these previous instances and, therefore, do not need to be re-evaluated. Finally, after configurations have been evaluated on all these instances, the race continues by evaluating additional new instances.\n\n The statistical tests can be performed at any moment during the race according to the setting of the options \\parameter{firstTest} and \\parameter{eachTest}. The elitist rule forbids discarding elite configurations, even if the show poor performance, until the last of the previous instances is seen in the race.\n\n The \\textbf{non-elitist} version of \\irace can discard elite configurations at any point of the race, instances are not re-used from one race to the next, and new instances are sampled for each race.", "section": "Elitist \\irace" }, { @@ -507,7 +498,7 @@ "long": "--elitist-new-instances", "default": 1, "description": "Number of instances added to the execution list before previous instances in elitist irace.", - "vignettes": " Number of new instances added to each race before evaluating instances from previous races (only for elitist \\irace).\n \\begin{xwarningbox}\n \n If \\parameter{deterministic} is \\code{TRUE} then the number of \\parameter{elitistNewInstances} will be reduced or set to \\code{0} once all instances have been evaluated.\n\\end{xwarningbox}", + "vignettes": "Number of new instances added to each race before evaluating instances from previous races (only for elitist \\irace).\n \\begin{xwarningbox}\n\n If \\parameter{deterministic} is \\code{TRUE} then the number of \\parameter{elitistNewInstances} will be reduced or set to \\code{0} once all instances have been evaluated.\n\\end{xwarningbox}", "section": "Elitist \\irace" }, { @@ -517,7 +508,7 @@ "long": "--elitist-limit", "default": 2, "description": "In elitist irace, maximum number per race of elimination tests that do not eliminate a configuration. Use 0 for no limit.", - "vignettes": " Maximum number of statistical tests performed without successful elimination after all instances from the previous race have been evaluated. If the limit is reached, the current race is stopped. Only valid for elitist \\irace. Use \\code{0} to disable the limit.", + "vignettes": "Maximum number of statistical tests performed without successful elimination after all instances from the previous race have been evaluated. If the limit is reached, the current race is stopped. Only valid for elitist \\irace. Use \\code{0} to disable the limit.", "section": "Elitist \\irace" }, { @@ -540,6 +531,15 @@ "vignettes": "Enable the use of adaptive capping. Capping is enabled by default if \\parameter{elitist} is active, $\\parameter{maxTime} > 0$ and $\\parameter{boundMax} > 0$. When using this option, \\irace provides an execution bound to each target algorithm execution (See \\autoref{sec:runner}). For more details about this option See \\autoref{sec:capping}.", "section": "Adaptive capping" }, + { + "name": "cappingAfterFirstTest", + "type": "b", + "short": "", + "long": "--capping-after-first-test", + "default": 0, + "description": "If set to 1, elimination due to capping only happens after \\parameter{firstTest} instances are seen.", + "section": "Adaptive capping" + }, { "name": "cappingType", "type": "s", @@ -548,7 +548,7 @@ "default": "median", "domain": "median,mean,worst,best", "description": "Measure used to obtain the execution bound from the performance of the elite configurations: median, mean, worst, best.", - "vignettes": " Specifies the measure used to define the execution bound:\n\\begin{itemize}\n\\item[] \\code{median} (the median of the performance of the elite configurations)\n\\item[] \\code{mean} (the mean of the performance of the elite configurations)\n\\item[] \\code{best} (the best performance of the elite configurations)\n\\item[] \\code{worst} (the worst performance of the elite configurations).\n\\end{itemize}", + "vignettes": "Specifies the measure used to define the execution bound:\n\\begin{itemize}\n\\item[] \\code{median} (the median of the performance of the elite configurations)\n\\item[] \\code{mean} (the mean of the performance of the elite configurations)\n\\item[] \\code{best} (the best performance of the elite configurations)\n\\item[] \\code{worst} (the worst performance of the elite configurations).\n\\end{itemize}", "section": "Adaptive capping", "man": "Measure used to obtain the execution bound from the performance of the elite configurations.\\itemize{\\item median: Median performance of the elite configurations.\\item mean: Mean performance of the elite configurations.\\item best: Best performance of the elite configurations.\\item worst: Worst performance of the elite configurations.}" }, @@ -570,8 +570,7 @@ "short": "", "long": "--bound-max", "default": 0, - "description": "Maximum execution bound for \\code{targetRunner}. It must be specified when capping is enabled.", - "vignettes": "Maximum execution bound for \\code{targetRunner}. It must be specified when capping is enabled.", + "description": "Maximum execution bound for \\parameter{targetRunner}. It must be specified when capping is enabled.", "section": "Adaptive capping" }, { @@ -581,8 +580,7 @@ "long": "--bound-digits", "default": 0, "description": "Precision used for calculating the execution time. It must be specified when capping is enabled.", - "vignettes": "Precision used for calculating the execution time. It must be specified when capping is enabled.", - "section": "Adaptive capping" + "section": "Adaptive capping" }, { "name": "boundPar", @@ -600,8 +598,8 @@ "short": "", "long": "--bound-as-timeout", "default": 1, - "description": "Replace the configuration cost of bounded executions with \\code{boundMax}.", - "vignettes": "Replace the configuration cost of bounded executions with \\parameter{boundMax}. See \\autoref{sec:capping}.", + "description": "Replace the configuration cost of bounded executions with \\code{boundMax}.", + "vignettes": "Replace the configuration cost of bounded executions with \\parameter{boundMax}. See \\autoref{sec:capping}.", "section": "Adaptive capping" }, { @@ -660,7 +658,7 @@ "short": "", "long": "--num-configurations", "default": 0, - "vignettes": " The number of configurations that will be raced at each iteration. By default (when equal to 0), this value changes for each iteration and depends on \\parameter{nbExperimentsPerIteration}, the iteration index and \\parameter{mu}. The precise details are given in the \\irace paper~\\citep{LopDubPerStuBir2016irace}.\nWe recommend to use the default value.", + "vignettes": "The number of configurations that will be raced at each iteration. By default (when equal to 0), this value changes for each iteration and depends on \\parameter{nbExperimentsPerIteration}, the iteration index and \\parameter{mu}. The precise details are given in the \\irace paper~\\citep{LopDubPerStuBir2016irace}.\nWe recommend to use the default value.", "section": "Internal \\irace options", "description": "Number of configurations to be sampled and evaluated at each iteration." }, @@ -670,7 +668,7 @@ "short": "", "long": "--mu", "default": 5, - "vignettes": " Parameter used to define the number of configurations to be sampled and evaluated at each iteration. The number of configurations will be calculated such that there is enough budget in each race to evaluate all configurations on at least $\\mu + \\min(5,j)$ training instances, where $j$ is the index of the current iteration. The value of $\\mu$ will be adjusted to never be lower than the value of \\parameter{firstTest}. We recommend to use the default value and, if needed, adjust \\parameter{firstTest}and \\parameter{eachTest}, instead.", + "vignettes": "Parameter used to define the number of configurations to be sampled and evaluated at each iteration. The number of configurations will be calculated such that there is enough budget in each race to evaluate all configurations on at least $\\mu + \\min(5,j)$ training instances, where $j$ is the index of the current iteration. The value of $\\mu$ will be adjusted to never be lower than the value of \\parameter{firstTest}. We recommend to use the default value and, if needed, adjust \\parameter{firstTest}and \\parameter{eachTest}, instead.", "section": "Internal \\irace options", "description": "Parameter used to define the number of configurations sampled and evaluated at each iteration." }, @@ -681,7 +679,6 @@ "long": "--confidence", "default": 0.95, "description": "Confidence level for the elimination test.", - "vignettes": "Confidence level for the elimination test.", "section": "Statistical test" } ] diff --git a/src/iracebin/irace.h b/src/iracebin/irace.h index 558d29c9..5cc353c4 100644 --- a/src/iracebin/irace.h +++ b/src/iracebin/irace.h @@ -5,45 +5,89 @@ #include #include "whereami.h" -#define max_path_length 1024 +#define MAX_PATH_LENGTH 1024 + char *get_install_path(const char * package_path) { - char fullpath[max_path_length - 1] = "\0"; - int dirname_length = 0; - int res = wai_getExecutablePath(fullpath, max_path_length, &dirname_length); - if (res <= 0) - return NULL; - if (res <= dirname_length) + char fullpath[MAX_PATH_LENGTH - 1] = "\0"; + int fullpath_length = 0; + int res = wai_getExecutablePath(fullpath, MAX_PATH_LENGTH, &fullpath_length); + if (res <= 0 || res <= fullpath_length) return NULL; - dirname_length -= strlen(package_path); - char * path = (char*) malloc(dirname_length+1); - memcpy(path, fullpath, dirname_length); - path[dirname_length] = '\0'; + // printf("fullpath: %s (%d)\n", fullpath, fullpath_length); +#ifdef WIN32 + for (size_t i = 0; i < fullpath_length; ++i) { + if (fullpath[i] == '\\') { + fullpath[i] = '/'; + } + } +#endif + // Remove package_path and everything after it from fullpath. + size_t package_path_len = strlen(package_path); + char *pos = strstr(fullpath, package_path); + char * last_found = NULL; + if (pos == NULL) { + printf("Error: irace is installed at an unexpected path: %s\n", fullpath); + exit(EXIT_FAILURE); + } + // Search the last occurrence in case fullpath is something like + // /irace/bin/R/x86_64-pc-linux-gnu-library/4.1/irace/bin/irace . + do { + last_found = pos; + pos = strstr(pos + package_path_len, package_path); + } while (pos != NULL); + *last_found = '\0'; + fullpath_length = strlen(fullpath); + char * path = (char*) malloc((fullpath_length + 1) * sizeof(char)); + memcpy(path, fullpath, fullpath_length + 1); + // printf("path: %s (%d)\n", fullpath, fullpath_length); return path; } +#undef MAX_PATH_LENGTH int exec_R(int argc, char *argv[], const char * user_code) { + const char * path = get_install_path("/irace/bin/"); + if (path == NULL) + path = ""; + +#define _libpath_str ".libPaths(c(r'{%s}', .libPaths()));%s" #ifdef WIN32 -# define EXE_EXT ".exe" +// Windows execvp() does not pass command-line arguments correctly if they +// contain space, tab, backslash, or double-quote characters. +#define libpath_str "\"" _libpath_str "\"" #else -# define EXE_EXT "" +#define libpath_str _libpath_str #endif - const char file[] = "R" EXE_EXT; - const char * path = get_install_path("/irace/bin"); - if (path == NULL) path = ""; -#define libpath_str ".libPaths(c('%s', .libPaths()));%s" + // -4 because of two times %s. char * r_code = malloc(sizeof(char) * (strlen(libpath_str) - 4 + strlen(path) + strlen(user_code) + 1)); sprintf(r_code, libpath_str, path, user_code); - //printf("%s\n", libpath); - +#undef _libpath_str +#undef libpath_str + // printf("%s\n", r_code); + char * const extra_argv[] = { "--vanilla", "--slave", "-e", r_code, "--args"}; int extra_argc = sizeof(extra_argv) / sizeof(char *); - char **cmd_args = calloc(extra_argc + argc + 1, sizeof(char *)); + char **cmd_args = malloc((extra_argc + argc + 1) * sizeof(char *)); cmd_args[0] = argv[0]; for (int i = 0; i < extra_argc; i++) cmd_args[i+1] = extra_argv[i]; for (int i = 1; i < argc; i++) cmd_args[extra_argc + i] = argv[i]; - return execvp(file, cmd_args); + cmd_args[extra_argc + argc] = NULL; + // for (int i = 0; i < extra_argc + argc; i++) + // printf("%d: %s\n", i, cmd_args[i]); + +#ifdef WIN32 +# define EXE_EXT ".exe" +#else +# define EXE_EXT "" +#endif + const char file[] = "R" EXE_EXT; +#undef EXE_EXT + if (execvp(file, cmd_args) == -1) { + perror("Error executing R"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; } diff --git a/src/iracebin/whereami.c b/src/iracebin/whereami.c index f497259d..36227c50 100644 --- a/src/iracebin/whereami.c +++ b/src/iracebin/whereami.c @@ -68,7 +68,13 @@ extern "C" { #if defined(_MSC_VER) #pragma warning(pop) #endif +#if (_MSC_VER >= 1900) #include +#else +#define bool int +#define false 0 +#define true 1 +#endif static int WAI_PREFIX(getModulePath_)(HMODULE module, char* out, int capacity, int* dirname_length) { @@ -171,6 +177,7 @@ int WAI_PREFIX(getModulePath)(char* out, int capacity, int* dirname_length) #elif defined(__linux__) || defined(__CYGWIN__) || defined(__sun) || defined(WAI_USE_PROC_SELF_EXE) || defined(__wasm__) + #include #include #include @@ -243,6 +250,11 @@ int WAI_PREFIX(getExecutablePath)(char* out, int capacity, int* dirname_length) #endif #endif +#if !defined(WAI_STRINGIZE) +#define WAI_STRINGIZE(s) +#define WAI_STRINGIZE_(s) #s +#endif + #if defined(__ANDROID__) || defined(ANDROID) #include #include @@ -264,20 +276,20 @@ int WAI_PREFIX(getModulePath)(char* out, int capacity, int* dirname_length) for (;;) { - char buffer[PATH_MAX < 1024 ? 1024 : PATH_MAX]; - uint64_t low, high; + char buffer[128 + PATH_MAX]; + uintptr_t low, high; char perms[5]; uint64_t offset; - uint32_t major, minor; - char path[PATH_MAX]; - uint32_t inode; + uint32_t major, minor, inode; + char path[PATH_MAX + 1]; if (!fgets(buffer, sizeof(buffer), maps)) break; - if (sscanf(buffer, "%" PRIx64 "-%" PRIx64 " %s %" PRIx64 " %x:%x %u %s\n", &low, &high, perms, &offset, &major, &minor, &inode, path) == 8) + if (sscanf(buffer, "%" SCNxPTR "-%" SCNxPTR " %s %" SCNx64 " %x:%x %u %" WAI_STRINGIZE(PATH_MAX) "[^\n]\n", &low, &high, perms, &offset, &major, &minor, &inode, path) == 8) { - uint64_t addr = (uintptr_t)WAI_RETURN_ADDRESS(); + void* _addr = WAI_RETURN_ADDRESS(); + uintptr_t addr = (uintptr_t)_addr; if (low <= addr && addr <= high) { char* resolved; diff --git a/src/iracebin/whereami.h b/src/iracebin/whereami.h index 670db54c..8e58c818 100644 --- a/src/iracebin/whereami.h +++ b/src/iracebin/whereami.h @@ -31,7 +31,8 @@ extern "C" { * @param out destination buffer, optional * @param capacity destination buffer capacity * @param dirname_length optional recipient for the length of the dirname part - * of the path. + * of the path. Available only when `capacity` is large enough to retrieve the + * path. * * @return the length of the executable path on success (without a terminal NUL * character), otherwise `-1` @@ -52,7 +53,8 @@ int WAI_PREFIX(getExecutablePath)(char* out, int capacity, int* dirname_length); * @param out destination buffer, optional * @param capacity destination buffer capacity * @param dirname_length optional recipient for the length of the dirname part - * of the path. + * of the path. Available only when `capacity` is large enough to retrieve the + * path. * * @return the length of the module path on success (without a terminal NUL * character), otherwise `-1` diff --git a/tests/testthat/helper-common.R b/tests/testthat/helper-common.R index bc8ab30c..b8986e88 100644 --- a/tests/testthat/helper-common.R +++ b/tests/testthat/helper-common.R @@ -1,7 +1,7 @@ # This file is loaded automatically by testthat. -generate.set.seed <- function() +generate_set_seed <- function() { - seed <- sample(2^30, 1) + seed <- sample.int(min(2147483647L, .Machine$integer.max), size = 1L, replace = TRUE) cat("Seed: ", seed, "\n") set.seed(seed) } @@ -9,6 +9,7 @@ generate.set.seed <- function() test_irace_detectCores <- function() { if (!identical(Sys.getenv("NOT_CRAN"), "true")) return(1L) + # FIXME: covr takes a very long time on github actions otherwise. if (identical(Sys.getenv("COVR_COVERAGE"), "true")) return(1L) x <- Sys.getenv("_R_CHECK_LIMIT_CORES_", "") if (nzchar(x) && x == "TRUE") return(2L) @@ -24,23 +25,31 @@ skip_on_coverage <- function() { system_os_is_windows <- function() .Platform$OS.type == "windows" +expect_valid_configurations <- function(configurations, parameters) +{ + configurations <- irace:::removeConfigurationsMetaData(as.data.frame(configurations)) + output <- capture.output(print(configurations, width=5000L, row.names = FALSE)) + output <- paste0(output, "\n", collapse="") + confs2 <- readConfigurationsFile(text=output, parameters=parameters) + expect_equal(configurations, confs2) +} + ## Functions ########################################################## f_ackley <- function (x,y, nsize = 0.01) { - - # Transformation of parameter values + # Transformation of parameter values # from [0,1] to [vmin,vmax] vmin <- -5 vmax <- 5 x <- (x*(vmax-vmin)) + vmin y <- (y*(vmax-vmin)) + vmin - + a <- -20 * exp (-0.2 * sqrt(0.5 * (x^2 + y^2))) - b <- exp(0.5 * (cos(2*pi*x) + cos(2*pi*y))) + b <- exp(0.5 * (cos(2*pi*x) + cos(2*pi*y))) f <- a - b + exp(1) + 20 # Simulating stochasticity noise <- runif(1, min = (1-nsize), max = (1+nsize)) f <- f*noise - + # Transform result from [fmin,fmax] # to [0,100] fmin <- 0 @@ -49,13 +58,13 @@ f_ackley <- function (x,y, nsize = 0.01) { } f_goldestein_price <- function (x,y, nsize = 0.01) { - # Trasfomation of parameter values + # Transformation of parameter values # from [0,1] to [vmin,vmax] vmin <- -2 vmax <- 2 x <- (x*(vmax-vmin)) + vmin y <- (y*(vmax-vmin)) + vmin - + a <- 1 + ((x + y + 1)^2) * (19 - 14*x + 3*x^2 - 14*y + 6*x*y + 3*y^2) b <- 30 + ((2*x - 3*y)^2) * (18 - 32*x + 12*x^2 + 48*y - 36*x*y + 27*y^2) f <- a*b @@ -63,7 +72,7 @@ f_goldestein_price <- function (x,y, nsize = 0.01) { noise <- runif(1, min = (1-nsize), max = (1+nsize)) f <- f*noise - # Transform result from [fmin,fmax] + # Transform result from [fmin,fmax] # to [0,100] fmin <- 0 fmax <- 1000000*(1+nsize) @@ -71,39 +80,39 @@ f_goldestein_price <- function (x,y, nsize = 0.01) { } f_matyas <- function (x,y, nsize = 0.01) { - # Trasfomation of parameter values + # Trasfomation of parameter values # from [0,1] to [vmin,vmax] vmin <- -10 vmax <- 10 x <- (x*(vmax-vmin)) + vmin y <- (y*(vmax-vmin)) + vmin - + f <- 0.26 * (x^2 + y^2) - (0.48*x*y) # Simulating stochasticity noise <- runif(1, min = (1-nsize), max = (1+nsize)) f <- f*noise - # Transform result from [fmin,fmax] + # Transform result from [fmin,fmax] # to [0,100] fmin <- 0 fmax <- 100*(1+nsize) ((f - fmin) / (fmax-fmin)) * (100-0) + 0 } -f_himmelblau <- function (x,y, nsize = 0.01) { - # Trasfomation of parameter values +f_himmelblau <- function (x,y, nsize = 0.01) { + # Trasfomation of parameter values # from [0,1] to [vmin,vmax] vmin <- -5 vmax <- 5 x <- (x*(vmax-vmin)) + vmin y <- (y*(vmax-vmin)) + vmin - + f <- (x^2 + y - 11)^2 + (x + y^2 - 7)^2 # Simulating stochasticity noise <- runif(1, min = (1-nsize), max = (1+nsize)) f <- f*noise - - # Transform result from [fmin,fmax] + + # Transform result from [fmin,fmax] # to [0,100] fmin <- 0 fmax <- 2000*(1+nsize) @@ -118,12 +127,78 @@ target_runner_capping_xy <- function(experiment, scenario) x <- configuration[["x"]] y <- configuration[["y"]] + value <- switch(instance, ackley = f_ackley(x, y), goldestein = f_goldestein_price(x, y), matyas = f_matyas(x, y), himmelblau = f_himmelblau(x, y)) - + # Simulate execution bound - list(cost = value, time=min(value + 0.1, bound), call = toString(experiment)) + list(cost = value, time=min(value + 0.1, bound)) +} + +irace_capping_xy <- function(..., targetRunner = force(target_runner_capping_xy), + parallel = test_irace_detectCores()) +{ + # Silence Error in `save(iraceResults, file = logfile, version = 3L)`: (converted from warning) 'package:irace' may not be available when loading + # See https://github.com/r-lib/testthat/issues/2044 + if (!is.null(attr(environment(targetRunner), "name", exact=TRUE))) { + environment(targetRunner) <- globalenv() + } + + args <- list(...) + parameters_table <- ' + x "" r (0, 1.00) + y "" r (0, 1.00) + reject "" c (0,1)' + + parameters <- readParameters(text = parameters_table) + logFile <- withr::local_tempfile(fileext=".Rdata") + scenario <- list(instances = c("ackley", "goldestein", "matyas", "himmelblau"), + targetRunner = targetRunner, + capping = TRUE, + boundMax = 80, + testType = "t-test", + logFile = logFile, + parallel = parallel, + parameters = parameters) + scenario <- modifyList(scenario, args) + scenario <- checkScenario (scenario) + + expect_true(irace:::checkTargetFiles(scenario = scenario)) + + confs <- irace(scenario = scenario) + best_conf <- getFinalElites(scenario$logFile, n = 1L, drop.metadata = TRUE) + expect_identical(removeConfigurationsMetaData(confs[1L, , drop = FALSE]), + best_conf) +} + +# Useful for testing recovery. +wrap_target_runner_error <- function(target_runner, limit) +{ + counter <- force(limit) + target_runner <- force(target_runner) + + fun <- function(experiment, scenario) { + counter <<- counter - 1L + if (counter <= 0L) + return(list(cost=NA)) + target_runner(experiment, scenario) + } + parent.env(environment(fun)) <- globalenv() + fun +} + +wrap_target_runner_counter <- function(target_runner) +{ + counter <- 0L + target_runner <- force(target_runner) + + fun <- function(experiment, scenario) { + counter <<- counter + 1L + target_runner(experiment, scenario) + } + parent.env(environment(fun)) <- globalenv() + fun } diff --git a/tests/testthat/test-GenericWrapper4AC.R b/tests/testthat/test-GenericWrapper4AC.R index 52df55bd..a39370ec 100644 --- a/tests/testthat/test-GenericWrapper4AC.R +++ b/tests/testthat/test-GenericWrapper4AC.R @@ -24,7 +24,8 @@ test_that("GenericWrapper4AC", { parameters = parameters, instances = scenario$instances, instances_ID = rep("cost", 2), - seeds = 0:1) + seeds = 0:1, + bounds = NULL) race_state <- irace:::RaceState$new(scenario) output <- irace:::execute_experiments(race_state, experiments, scenario) expect_equal(output[[1]]$status, "SUCCESS") @@ -38,14 +39,14 @@ test_that("GenericWrapper4AC", { parameters = parameters, instances = scenario$instances, instances_ID = rep("cost", 2), - seeds = 2) + seeds = 2, bounds = NULL) expect_error(irace:::execute_experiments(race_state, experiments, scenario), "CRASHED") experiments <- irace:::createExperimentList(configurations, parameters = parameters, instances = scenario$instances, instances_ID = rep("cost", 2), - seeds = 3) + seeds = 3, bounds = NULL) expect_error(irace:::execute_experiments(race_state, experiments, scenario), "ABORT") diff --git a/tests/testthat/test-ablation-cmdline.R b/tests/testthat/test-ablation-cmdline.R deleted file mode 100644 index 722e0034..00000000 --- a/tests/testthat/test-ablation-cmdline.R +++ /dev/null @@ -1,15 +0,0 @@ -withr::with_output_sink("test-ablation-cmdline.Rout", { - skip_on_cran() - - test_that("--help", { - expect_no_warning(ablation_cmdline("--help")) - }) - - test_that("--log-file=sann.rda", { - logfile <- system.file(package="irace", mustWork = TRUE, file.path("exdata","sann.rda")) - outfile <- tempfile(pattern = "log-ablation", fileext = ".Rdata") - plotfile <- tempfile(pattern = "ablation", fileext = ".pdf") - expect_no_warning( - ablation_cmdline(paste0("--log-file=", logfile, " -o ", outfile, " -p ", plotfile))) - }) -}) # withr::with_output_sink() diff --git a/tests/testthat/test-ablation.R b/tests/testthat/test-ablation.R new file mode 100644 index 00000000..111351f6 --- /dev/null +++ b/tests/testthat/test-ablation.R @@ -0,0 +1,237 @@ +withr::with_output_sink("test-ablation.Rout", { + skip_on_cran() + withr::local_options(warn=2) + + test_that("generateAblation", { + parameters <- parametersNew(param_cat(name = "algorithm", values = c("as", "mmas", "ras", "acs")), + param_real(name = "alpha", lower = 0.0, upper=5.0), + param_real(name = "beta", lower = 0.0, upper = 10.0), + param_int(name = "ants", lower = 2, upper = 100), + param_real(name = "q0", lower=0.0, upper=1.0, condition = expression(algorithm == "acs")), + param_int(name = "rasrank", lower=1, upper=quote(min(ants, 10)), condition = 'algorithm == "ras"'), + param_int(name = "eants", lower=0, upper=expression(rasrank)), + param_cat(name = "dlb", values = c(0,1), condition = "localsearch == 1"), + param_int(name = "nnls", lower = 5, upper = 50, condition = expression(dlb == 1)), + param_ord(name = "localsearch", values = c("0", "1")), + param_cat(name = "fixed", values = "0"), + forbidden = "(alpha == 0) & (beta == 0)") + + confs <- readConfigurationsFile(parameters = parameters, text =' +algorithm alpha beta ants q0 rasrank eants dlb nnls localsearch +ras 1 0 5 NA 5 5 1 5 1 +ras 0.5 0 5 NA 2 2 1 5 1 +ras 0.5 0 4 NA 2 2 1 5 1 +acs 0 1 2 0.5 NA NA NA NA 0 +mmas 1 1 6 NA NA NA 0 NA 1 +') + confs[[".ID."]] <- seq_len(nrow(confs)) + colClasses <- c(localsearch="character", q0="numeric", + rasrank="integer", eants="integer", dlb="character", nnls="integer") + + check_generate_ablation <- function(src, target, configurations_table, changed) { + aux <- irace:::generate_ablation(confs[src, , drop=FALSE], confs[target, , drop=FALSE], + parameters, param_names = parameters$names_variable) + expect_valid_configurations(aux$configurations, parameters) + configurations <- read.table(header=TRUE, colClasses=colClasses, text=configurations_table) + configurations[["fixed"]] <- "0" + configurations[[".ID."]] <- src + configurations[[".PARENT."]] <- src + expect_equal(aux, list(configurations=configurations, changed_params = changed)) + } + + check_generate_ablation(1L, 2L, ' +algorithm alpha beta ants q0 rasrank eants dlb nnls localsearch + ras 0.5 0 5 NA 5 5 1 5 1 + ras 1.0 0 5 NA 2 2 1 5 1 + ras 1.0 0 5 NA 5 2 1 5 1 +', changed = list("alpha", c("rasrank", "eants"), "eants")) + + check_generate_ablation(1L, 3L, ' +algorithm alpha beta ants q0 rasrank eants dlb nnls localsearch + ras 0.5 0 5 NA 5 5 1 5 1 + ras 1.0 0 4 NA 2 2 1 5 1 + ras 1.0 0 5 NA 2 2 1 5 1 + ras 1.0 0 5 NA 5 2 1 5 1 +', changed = list("alpha", c("ants", "rasrank", "eants"), c("rasrank", "eants"),"eants")) + + check_generate_ablation(1L, 4L, ' +algorithm alpha beta ants q0 rasrank eants dlb nnls localsearch + acs 1.0 0 5 0.5 NA NA 1 5 1 + ras 1.0 1.0 5 NA 5 5 1 5 1 + ras 1.0 0 5 NA 5 5 NA NA 0 +', changed = list(c("algorithm", "q0", "rasrank", "eants"), "beta", + c("localsearch", "dlb", "nnls"))) + + check_generate_ablation(1L, 5L, ' +algorithm alpha beta ants q0 rasrank eants dlb nnls localsearch + mmas 1.0 0 5 NA NA NA 1 5 1 + ras 1.0 1.0 5 NA 5 5 1 5 1 + ras 1 0 6 NA 5 5 1 5 1 + ras 1.0 0 5 NA 5 5 0 NA 1 +', changed = list(c("algorithm", "rasrank", "eants"), "beta", "ants", c("dlb", "nnls"))) + + check_generate_ablation(2L, 1L, ' +algorithm alpha beta ants q0 rasrank eants dlb nnls localsearch + ras 1.0 0 5 NA 2 2 1 5 1 + ras 0.5 0 5 NA 5 2 1 5 1 +', changed = list("alpha", c("rasrank"))) + + check_generate_ablation(3L, 1L, ' +algorithm alpha beta ants q0 rasrank eants dlb nnls localsearch + ras 1.0 0 4 NA 2 2 1 5 1 + ras 0.5 0 5 NA 2 2 1 5 1 +', changed = list("alpha", c("ants"))) + + check_generate_ablation(4L, 1L, ' +algorithm alpha beta ants q0 rasrank eants dlb nnls localsearch + acs 1 1 2 0.5 NA NA NA NA 0 + acs 0 1 5 0.5 NA NA NA NA 0 + acs 0 1 2 0.5 NA NA 1 5 1 +', changed = list("alpha", "ants", c("localsearch", "dlb", "nnls"))) + + check_generate_ablation(5L, 1L, ' +algorithm alpha beta ants q0 rasrank eants dlb nnls localsearch + ras 1 1 6 NA 5 5 0 NA 1 + mmas 1 0 6 NA NA NA 0 NA 1 + mmas 1 1 5 NA NA NA 0 NA 1 + mmas 1 1 6 NA NA NA 1 5 1 +', changed = list(c("algorithm", "rasrank", "eants"), "beta", "ants", c("dlb", "nnls"))) + + }) + + test_that("--help", { + expect_output(ablation_cmdline("--help")) + }) + + outfile <- withr::local_tempfile(pattern = "log-ablation", fileext = ".Rdata") + logfile <- withr::local_tempfile(pattern = "irace", fileext = ".Rdata") + + parameters <- parametersNew( + param_cat("cat", values = c("0", "1", "2", "3", "4")), + param_real("real", lower = 0.0, upper=1.0), + param_int("int", lower = 100, upper = 500), + param_cat("bool", values = c("0", "1"))) + + default <- data.frame(cat="4", real=1.0, int=500L, bool = "1") + + target_runner <- function(experiment, scenario) { + conf <- experiment$configuration + instance <- experiment$instance + seed <- experiment$seed + k <- if (as.logical(as.integer(conf[["bool"]]))) 1000 else 100 + list(cost = instance + 1/seed + k * (conf[["int"]] + as.integer(conf[["cat"]]) + (conf[["real"]]-0.5)^2)) + } + + check_log <- function(log) { + instances_log <- log$state$instances_log + instances_log[, instance:=.I] + experiment_log <- log$state$experiment_log[instances_log, on="instance"] + experiment_log[["instance_value"]] <- log$scenario$instances[experiment_log[["instanceID"]]] + experiment_log <- experiment_log[log$allConfigurations, on = c(configuration=".ID.")] + experiments <- log$experiments + experiments = data.table( + instance = rep(seq_len(nrow(experiments)), ncol(experiments)), + configuration = rep(seq_len(ncol(experiments)), each = nrow(experiments)), + cost3 = c(experiments) + ) + experiments <- experiments[!is.na(experiments$cost3),] + experiment_log <- experiment_log[experiments, on=.NATURAL] + experiment_log[, cost2:=instance_value + 1/seed + fifelse(as.logical(as.integer(bool)), 1000, 100) * (int + as.integer(cat) + (real - 0.5)^2)] + if ("bound" %in% colnames(experiment_log)) { + experiment_log[, cost2 := pmin.int(cost2, bound)] + experiment_log[, cost3 := pmin.int(cost3, bound)] + } + expect_equal(experiment_log[["cost"]], experiment_log[["cost2"]]) + expect_equal(experiment_log[["cost"]], experiment_log[["cost3"]]) + } + + src_file <- withr::local_tempfile(pattern="src", fileext=".txt", + lines=c("cat real int bool", "4 1.0 500 1")) + target_file <- withr::local_tempfile(pattern="target", fileext=".txt", + lines=c("cat real int bool", "0 0.0 100 0")) + + test_that("ablation maxTime", { + target_runner_time <- function(experiment, scenario) + list(cost = target_runner(experiment, scenario)$cost, + time = runif(1, min=0.1, max=1)) + + scenario <- list(targetRunner = target_runner_time, + instances = seq(1000, 10000, 1000), + seed = 42, + maxTime = 1000, + initConfigurations = default, + logFile = logfile, + parameters = parameters) + scenario <- checkScenario (scenario) + irace(scenario = scenario) + check_log(read_logfile(logfile)) + res <- ablation(logfile, ablationLogFile = outfile) + check_log(res) + expect_true(res$complete) + res <- ablation(logfile, ablationLogFile = outfile, type = "racing") + check_log(res) + expect_true(res$complete) + + res <- ablation(logfile, ablationLogFile = outfile, src = src_file, target = target_file) + check_log(res) + expect_true(res$complete) + + }) + + test_that("ablation capping", { + target_runner_capping <- function(experiment, scenario) { + cost <- min(experiment$bound, target_runner(experiment, scenario)$cost) + list(cost = cost, time = cost) + } + + boundMax <- 1000 + (1000 * 505) + scenario <- list(targetRunner = target_runner_capping, + instances = seq(1000, 10000, 1000), + seed = 42, + maxTime = 100 * boundMax, boundMax = boundMax, + initConfigurations = default, + logFile = logfile, + parameters = parameters) + scenario <- checkScenario(scenario) + expect_warning(irace(scenario = scenario), "is too large") + check_log(read_logfile(logfile)) + + res <- ablation(logfile, ablationLogFile = outfile) + check_log(res) + expect_true(res$complete) + + res <- ablation(logfile, ablationLogFile = outfile, type = "racing") + check_log(res) + expect_true(res$complete) + }) + + test_that("ablation maxExperiments", { + scenario <- list(targetRunner = target_runner, + instances = seq(1000, 10000, 1000), + maxExperiments = 1000, seed = 42, + initConfigurations = default, + logFile = logfile, + parameters = parameters) + scenario <- checkScenario (scenario) + irace(scenario = scenario) + check_log(read_logfile(logfile)) + + res <- ablation(logfile, ablationLogFile = outfile) + check_log(res) + expect_true(res$complete) + + res <- ablation(logfile, ablationLogFile = outfile, type = "racing") + check_log(res) + expect_true(res$complete) + + res <- ablation(logfile, ablationLogFile = outfile, src = src_file, target = target_file) + check_log(res) + expect_true(res$complete) + + plotfile <- withr::local_tempfile(pattern = "ablation", fileext = ".pdf") + res <- ablation_cmdline(paste0("--log-file=", logfile, " -o ", outfile, " -p ", plotfile)) + check_log(res) + expect_true(res$complete) + }) + +}) # withr::with_output_sink() diff --git a/tests/testthat/test-blocksize.R b/tests/testthat/test-blocksize.R index d41f3691..792346cc 100644 --- a/tests/testthat/test-blocksize.R +++ b/tests/testthat/test-blocksize.R @@ -1,27 +1,34 @@ withr::with_output_sink("test-blocksize.Rout", { -cap_irace <- function(...) +cap_irace <- function(..., targetRunner = force(target_runner_capping_xy)) { + # Silence Error in `save(iraceResults, file = logfile, version = 3L)`: (converted from warning) 'package:irace' may not be available when loading + # See https://github.com/r-lib/testthat/issues/2044 + if (!is.null(attr(environment(targetRunner), "name", exact=TRUE))) { + environment(targetRunner) <- globalenv() + } + args <- list(...) parameters_table <- ' x "" r (0, 1.00) y "" r (0, 1.00)' - + parameters <- readParameters(text = parameters_table) + logFile <- withr::local_tempfile(fileext=".Rdata") scenario <- list(instances = c("ackley", "goldestein", "matyas", "himmelblau"), - targetRunner = target_runner_capping_xy, + targetRunner = targetRunner, capping = TRUE, blockSize = 4, boundMax = 80, + logFile = logFile, testType = "t-test", parameters = parameters) scenario <- modifyList(scenario, args) scenario <- checkScenario (scenario) - confs <- irace(scenario = scenario) - best.conf <- getFinalElites(scenario$logFile, n = 1L, drop.metadata = TRUE) + best_conf <- getFinalElites(scenario$logFile, n = 1L, drop.metadata = TRUE) expect_identical(removeConfigurationsMetaData(confs[1L, , drop = FALSE]), - best.conf) + best_conf) invisible(read_logfile(scenario$logFile)) } @@ -31,7 +38,7 @@ target_runner_time <- function(experiment, scenario) tmax <- configuration[["tmax"]] temp <- configuration[["temp"]] time <- max(1, abs(rnorm(1, mean=(tmax+temp)/10))) - list(cost = time, time = time, call = toString(experiment)) + list(cost = time, time = time) } time_irace <- function(...) @@ -47,11 +54,11 @@ time_irace <- function(...) parameters = parameters) scenario <- modifyList(scenario, args) scenario <- checkScenario (scenario) - + confs <- irace(scenario = scenario) - best.conf <- getFinalElites(scenario$logFile, n = 1L, drop.metadata = TRUE) + best_conf <- getFinalElites(scenario$logFile, n = 1L, drop.metadata = TRUE) expect_identical(removeConfigurationsMetaData(confs[1L, , drop = FALSE]), - best.conf) + best_conf) invisible(read_logfile(scenario$logFile)) } @@ -67,22 +74,22 @@ test_that("blockSize error", { }) test_that("blockSize cap_irace maxExperiments = 1000", { - generate.set.seed() + generate_set_seed() expect_warning(check_blocksize(cap_irace(maxExperiments = 1000, debugLevel = 3)), "Assuming 'mu = firstTest * blockSize' because 'mu' cannot be smaller", fixed = TRUE) }) test_that("blockSize maxTime=1000", { - generate.set.seed() + generate_set_seed() check_blocksize(time_irace(maxTime = 1000)) }) -test_that("blockSize maxTime=1000 large newInstances", { +test_that("blockSize maxTime=1000 elitistNewInstances", { skip_on_cran() - generate.set.seed() + generate_set_seed() check_blocksize(time_irace(maxTime = 1000, instances = letters[1:9], - elitistNewInstances = 6, elitistLimit = 2)) + elitistNewInstances = 2, elitistLimit = 2)) }) }) # withr::with_output_sink() diff --git a/tests/testthat/test-bug-10.R b/tests/testthat/test-bug-10.R index 21ce7bb0..f49420f6 100644 --- a/tests/testthat/test-bug-10.R +++ b/tests/testthat/test-bug-10.R @@ -1,7 +1,5 @@ -# https://github.com/MLopez-Ibanez/irace/issues/10 withr::with_output_sink("test-bug-10.Rout", { -test_that("bug 10", { - skip_on_cran() + parameters_txt <- ' algorithm "--" c (as,mmas,eas,ras,acs) localsearch "--localsearch " c (0, 1, 2, 3) @@ -15,13 +13,38 @@ elitistants "--elitistants " i (1, 750) | algorithm == "eas" nnls "--nnls " i (5, 50) | localsearch %in% c(1,2,3) dlb "--dlb " c (0, 1) | localsearch %in% c(1,2,3) ' -target.runner <- function(experiment, scenario) - list(cost = 100, call = toString(experiment)) -parameters <- irace:::readParameters(text=parameters_txt) +parameters <- readParameters(text=parameters_txt) + +test_that("bug blocksize", { + skip_on_cran() + + target_runner <- function(experiment, scenario) + list(cost = 100 + rnorm(1, 0, 0.1)) + +withr::with_options(list(warning=2), { + scenario <- list(targetRunner = target_runner, + instances=1:5, firstTest=5*5, eachTest=5, + sampleInstances=FALSE, + maxExperiments = 5000, logFile = "", + elitistNewInstances = 1, + elitist = TRUE, + parameters = parameters) + scenario <- checkScenario (scenario) + confs <- irace(scenario = scenario) + expect_false(is.null(confs)) +}) +}) + +# https://github.com/MLopez-Ibanez/irace/issues/10 +test_that("bug 10", { + skip_on_cran() + + target_runner <- function(experiment, scenario) + list(cost = 100, call = toString(experiment)) withr::with_options(list(warning=2), { - scenario <- list(targetRunner = target.runner, + scenario <- list(targetRunner = target_runner, instances=1:10, maxExperiments = 5000, logFile = "", deterministic = TRUE, @@ -34,4 +57,5 @@ withr::with_options(list(warning=2), { expect_false(is.null(confs)) }) }) -}) + +}) # withr::with_output_sink diff --git a/tests/testthat/test-bug-55.R b/tests/testthat/test-bug-55.R index ae67da2e..aca8d2a6 100644 --- a/tests/testthat/test-bug-55.R +++ b/tests/testthat/test-bug-55.R @@ -6,7 +6,7 @@ test_that("bug-55", { foo "--foo " i (0, 1) foo2 "--foo2 " c (true, false) | foo == 0 ' - params <- irace::readParameters(text=parameters_txt) + params <- readParameters(text=parameters_txt) configurations_txt <- ' foo foo2 1 0 false diff --git a/tests/testthat/test-bug-blocksize.R b/tests/testthat/test-bug-blocksize.R deleted file mode 100644 index 5e429aec..00000000 --- a/tests/testthat/test-bug-blocksize.R +++ /dev/null @@ -1,36 +0,0 @@ -# https://github.com/MLopez-Ibanez/irace/issues/10 -withr::with_output_sink("test-bug-blocksize.Rout", { -test_that("bug blocksize", { - skip_on_cran() -parameters_txt <- ' -algorithm "--" c (as,mmas,eas,ras,acs) -localsearch "--localsearch " c (0, 1, 2, 3) -alpha "--alpha " r (0.00, 5.00) -beta "--beta " r (0.00, 10.00) -rho "--rho " r (0.01, 1.00) -ants "--ants " i (5, 100) -q0 "--q0 " r (0.0, 1.0) | algorithm == "acs" -rasrank "--rasranks " i (1, 100) | algorithm == "ras" -elitistants "--elitistants " i (1, 750) | algorithm == "eas" -nnls "--nnls " i (5, 50) | localsearch %in% c(1,2,3) -dlb "--dlb " c (0, 1) | localsearch %in% c(1,2,3) -' -target.runner <- function(experiment, scenario) - list(cost = 100 + rnorm(1, 0, 0.1), call = toString(experiment)) - -parameters <- irace:::readParameters(text=parameters_txt) - -withr::with_options(list(warning=2), { - scenario <- list(targetRunner = target.runner, - instances=1:5, firstTest=5*5, eachTest=5, - sampleInstances=FALSE, - maxExperiments = 5000, logFile = "", - elitistNewInstances = 1, - elitist = TRUE, - parameters = parameters) - scenario <- checkScenario (scenario) - confs <- irace(scenario = scenario) - expect_false(is.null(confs)) -}) -}) -}) diff --git a/tests/testthat/test-capping.R b/tests/testthat/test-capping.R index 6dcc46e1..2cc782f1 100644 --- a/tests/testthat/test-capping.R +++ b/tests/testthat/test-capping.R @@ -2,58 +2,37 @@ withr::with_output_sink("test-capping.Rout", { ## target runner ########################################################### -target.runner.reject <- function(experiment, scenario) +target_runner_reject <- function(experiment, scenario) { if (experiment$configuration[["reject"]] == "1" && runif(1) <= 0.01) - list(cost = -Inf, time = experiment$bound, call = toString(experiment)) + return(list(cost = -Inf, time = experiment$bound)) target_runner_capping_xy(experiment, scenario) } -cap.irace <- function(...) -{ - args <- list(...) - parameters_table <- ' - x "" r (0, 1.00) - y "" r (0, 1.00) - reject "" c (0,1)' - - parameters <- readParameters(text = parameters_table) - scenario <- list(instances = c("ackley", "goldestein", "matyas", "himmelblau"), - targetRunner = target_runner_capping_xy, - capping = TRUE, - boundMax = 80, - testType = "t-test", - logFile = tempfile(fileext=".Rdata"), - parallel = if (system_os_is_windows()) 1L else test_irace_detectCores(), - parameters = parameters) - scenario <- modifyList(scenario, args) - scenario <- checkScenario (scenario) - - irace:::checkTargetFiles(scenario = scenario) - - confs <- irace(scenario = scenario) - best.conf <- getFinalElites(scenario$logFile, n = 1L, drop.metadata = TRUE) - expect_identical(removeConfigurationsMetaData(confs[1L, , drop = FALSE]), - best.conf) -} +test_that("irace_capping_xy maxExperiments = 1000", { + generate_set_seed() + irace_capping_xy(maxExperiments = 1000) +}) -test_that("cap.irace maxExperiments = 1000", { - generate.set.seed() - cap.irace(maxExperiments = 1000) +test_that("irace_capping_xy maxExperiments = 1000 cappingAfterFirstTest", { + generate_set_seed() + irace_capping_xy(maxExperiments = 1000, cappingAfterFirstTest=1) }) -test_that("cap.irace maxTime = 1000", { - generate.set.seed() - expect_warning(cap.irace(maxTime = 1000), - "boundMax = 80 is too large, using 5 instead") +test_that("irace_capping_xy maxTime = 1000", { + generate_set_seed() + expect_warning(irace_capping_xy(maxTime = 1000), + "boundMax=80 is too large, using 5 instead") }) -test_that("cap.irace targetRunner = target.runner.reject, maxTime = 1000", { - skip_on_cran() +test_that("irace_capping_xy targetRunner = target_runner_reject, maxTime = 1000", { + skip_on_cran() skip_on_coverage() # This test sometimes fails randomly - generate.set.seed() - cap.irace(targetRunner = target.runner.reject, maxTime = 1000, boundMax = 5, debugLevel = 3) + generate_set_seed() + irace_capping_xy(targetRunner = target_runner_reject, maxTime = 1000, boundMax = 5, debugLevel = 3, + # FIXME: target_runner_reject does not work in parallel on Windows. + parallel = if (system_os_is_windows()) 1L else test_irace_detectCores()) }) test_that("capping default value", { @@ -61,13 +40,12 @@ test_that("capping default value", { x "" r (0, 1.00) y "" r (0, 1.00) reject "" c (0,1)' - + parameters <- readParameters(text = parameters_table) def_scenario <- list(instances = c("ackley", "goldestein", "matyas", "himmelblau"), - targetRunner = target.runner.reject, + targetRunner = target_runner_reject, maxTime = 1200, - logFile = tempfile(fileext=".Rdata"), parameters = parameters) scenario <- checkScenario(def_scenario) expect_false(scenario$capping) diff --git a/tests/testthat/test-dependencies.R b/tests/testthat/test-dependencies.R index 148dd446..0e6f7cf3 100644 --- a/tests/testthat/test-dependencies.R +++ b/tests/testthat/test-dependencies.R @@ -94,13 +94,11 @@ p1 p2 p3 ') }) -test_that("checkDependencies", { - -test.checkDependencies <- function(parameterFile, ...) +test_checkDependencies <- function(parameterFile, ...) { args <- list(...) - target.runner <- function(experiment, scenario) { + target_runner <- function(experiment, scenario) { configuration <- experiment$configuration tmax <- configuration[["real"]] stopifnot(is.numeric(tmax)) @@ -115,8 +113,8 @@ test.checkDependencies <- function(parameterFile, ...) weights <- rnorm(200, mean = 0.9, sd = 0.02) parameters <- readParameters(parameterFile) - scenario <- list(targetRunner = target.runner, instances = weights, seed = 1234567, maxExperiments=200, - parameters = parameters) + scenario <- list(targetRunner = target_runner, instances = weights, + seed = 1234567, maxExperiments = 200, parameters = parameters) scenario <- modifyList(scenario, args) scenario <- checkScenario (scenario) @@ -139,8 +137,12 @@ test.checkDependencies <- function(parameterFile, ...) checkConditionalAndDependency(confs[i,], parameters) } } - test.checkDependencies(parameterFile="dependencies.txt") - test.checkDependencies(parameterFile="dependencies2.txt") + +test_that("checkDependencies", { + test_checkDependencies(parameterFile="dependencies.txt") +}) +test_that("checkDependencies2", { + test_checkDependencies(parameterFile="dependencies2.txt") }) }) # withr::with_output_sink() diff --git a/tests/testthat/test-get-functions.R b/tests/testthat/test-get-functions.R new file mode 100644 index 00000000..90c55a57 --- /dev/null +++ b/tests/testthat/test-get-functions.R @@ -0,0 +1,11 @@ +withr::with_output_sink("test-get-functions.Rout", { + + test_that("getConfigurationById()", { + log <- read_logfile(system.file(package="irace", "exdata", "irace-acotsp.Rdata", mustWork=TRUE)) + ids <- sample(log$allConfigurations[[".ID."]], size=3) + ids <- c(ids, rev(ids)) + sel_ids <- getConfigurationById(log, ids = ids)[[".ID."]] + expect_identical(sel_ids, ids) +}) + +}) diff --git a/tests/testthat/test-maxTime.R b/tests/testthat/test-maxTime.R index ebfcedd2..e486ab6d 100644 --- a/tests/testthat/test-maxTime.R +++ b/tests/testthat/test-maxTime.R @@ -1,35 +1,31 @@ withr::with_output_sink("test-maxTime.Rout", { -target.runner <- function(experiment, scenario) +target_runner <- function(experiment, scenario) { configuration <- experiment$configuration tmax <- configuration[["tmax"]] temp <- configuration[["temp"]] - stopifnot(is.numeric(tmax)) - stopifnot(is.numeric(temp)) time <- max(1, abs(rnorm(1, mean=(tmax+temp)/10))) - list(cost = time, time = time, call = toString(experiment)) + list(cost = time, time = time) } -time.irace <- function(...) +time_irace <- function(...) { args <- list(...) - weights <- rnorm(200, mean = 0.9, sd = 0.02) - test_weights <- rnorm(2, mean = 0.9, sd = 0.02) parameters <- readParameters(text = ' tmax "" i (1, 50) temp "" r (0, 10) dummy "" c ("dummy") ') - scenario <- list(targetRunner = target.runner, - instances = weights, - testInstances = test_weights, + scenario <- list(targetRunner = target_runner, + instances = 1:10, + testInstances = 11:20, seed = 1234567, parameters = parameters) scenario <- modifyList(scenario, args) scenario <- checkScenario (scenario) - irace:::checkTargetFiles(scenario = scenario) + expect_true(irace:::checkTargetFiles(scenario = scenario)) confs <- irace(scenario = scenario) final_ids <- sort(as.character(confs$.ID.[1:scenario$testNbElites])) @@ -48,14 +44,14 @@ time.irace <- function(...) test_that("maxTime=500 testNbElites=2 testIterationElites=FALSE", { - generate.set.seed() - time.irace(maxTime = 500, testNbElites=2) + generate_set_seed() + time_irace(maxTime = 500, testNbElites=2) }) test_that("maxTime=1111 testNbElites=3 testIterationElites=TRUE", { skip_on_cran() - generate.set.seed() - time.irace(maxTime = 1111, testNbElites=3, testIterationElites=TRUE) + generate_set_seed() + time_irace(maxTime = 1111, testNbElites=3, testIterationElites=TRUE) }) }) # withr::with_output_sink() diff --git a/tests/testthat/test-multi_irace.R b/tests/testthat/test-multi_irace.R index 3b82f7de..ef8dc8c7 100644 --- a/tests/testthat/test-multi_irace.R +++ b/tests/testthat/test-multi_irace.R @@ -1,5 +1,6 @@ withr::with_output_sink("test-multi_irace.Rout", { +# FIXME: Use temporary files and directories. make.target.runner <- function(parameters) { function(experiment, scenario) { cost <- max(1, abs(rnorm(1, mean=sum(unlist(experiment$configuration[parameters]))))) @@ -24,17 +25,15 @@ parameters.table.2 <- make.parameters.table(c("x2", "x3")) parameters.table.3 <- make.parameters.table(c("x3", "x1")) check.default.logFiles <- function(n) { - for (i in 1:n) { - dir <- sprintf("run_%02d", i) - expect_true(dir.exists(dir)) - expect_true(file.exists(file.path(dir, "irace.Rdata"))) - } + dir <- sprintf("run_%02d", 1:n) + expect_equal(dir.exists(dir), rep_len(TRUE, n)) + expect_equal(file.exists(file.path(dir, "irace.Rdata")), rep_len(TRUE, n)) } test_that("multiple scenarios, multiple parameters", { skip_on_cran() # Reproducible results - generate.set.seed() + generate_set_seed() scenarios <- lapply(list(target.runner.1, target.runner.2, target.runner.3), make.scenario) parameters <- list(parameters.table.1, parameters.table.2, parameters.table.3) @@ -48,7 +47,7 @@ test_that("multiple scenarios, multiple parameters", { test_that("one scenario, multiple parameters", { skip_on_cran() # Reproducible results - generate.set.seed() + generate_set_seed() scenarios <- list(make.scenario(target.runner.1)) dummy_parameters_names <- c("dummy1", "dummy2", "dummy3") @@ -63,7 +62,7 @@ test_that("one scenario, multiple parameters", { test_that("multiple scenarios, one parameters", { skip_on_cran() # Reproducible results - generate.set.seed() + generate_set_seed() scenarios <- lapply(list(100, 500, 1000), function(maxExperiments) make.scenario(target.runner.1, maxExperiments) ) parameters <- list(parameters.table.1) @@ -76,7 +75,7 @@ test_that("multiple scenarios, one parameters", { test_that("one scenario, one parameters, multiple n", { skip_on_cran() # Reproducible results - generate.set.seed() + generate_set_seed() scenarios <- list(make.scenario(target.runner.1)) parameters <- list(parameters.table.1) @@ -88,7 +87,7 @@ test_that("one scenario, one parameters, multiple n", { test_that("multiple scenarios, multiple parameters, multiple n", { skip_on_cran() # Reproducible results - generate.set.seed() + generate_set_seed() scenarios <- lapply(list(target.runner.1, target.runner.2, target.runner.3), make.scenario) parameters <- list(parameters.table.1, parameters.table.2, parameters.table.3) @@ -101,7 +100,7 @@ test_that("multiple scenarios, multiple parameters, multiple n", { test_that("logFile not in execDir", { skip_on_cran() # Reproducible results - generate.set.seed() + generate_set_seed() execDir <- path_rel2abs("./multi_irace_execDir") logFileDir <- path_rel2abs("./multi_irace_logFileDir") @@ -141,13 +140,14 @@ test_that("global seed", { test_that("sequential and parallel identical", { skip_on_cran() skip_on_os("windows") - skip_if(test_irace_detectCores() <= 1L, + ncores <- test_irace_detectCores() + skip_if(ncores <= 1L, message = "This test only makes sense if multiple cores are available") scenarios <- lapply(list(target.runner.1, target.runner.2, target.runner.3), make.scenario) parameters <- list(parameters.table.1, parameters.table.2, parameters.table.3) runs.sequential <- multi_irace(scenarios, parameters, global_seed = 42) - runs.parallel <- multi_irace(scenarios, parameters, global_seed = 42, parallel = test_irace_detectCores()) + runs.parallel <- multi_irace(scenarios, parameters, global_seed = 42, parallel = ncores) expect_equal(runs.sequential, runs.parallel) }) diff --git a/tests/testthat/test-psrace.R b/tests/testthat/test-psrace.R new file mode 100644 index 00000000..11910ac3 --- /dev/null +++ b/tests/testthat/test-psrace.R @@ -0,0 +1,34 @@ +withr::with_output_sink("test-psrace.Rout", { + +test_that("max_experiments=0.1", { + skip_on_cran() + generate_set_seed() + irace_log <- read_logfile(system.file(package="irace", "exdata", "sann.rda")) + # Use a temporary file to not change the original "sann.rda". + psrace_logFile <- withr::local_tempfile(fileext = ".Rdata") + # Execute the post-selection after the execution of irace. Use 10% of the total budget. + psRace(irace_log, max_experiments=0.1, psrace_logFile = psrace_logFile) + irace_log <- read_logfile(psrace_logFile) + budget <- nrow(irace_log$state$experiment_log[iteration == max(iteration)]) + expect_gt(budget, 50L) + # It should be equal but elitist_race sometimes fails to consume all the budget. + expect_lte(budget, irace_log$psrace_log$max_experiments) +}) + +test_that("max_experiments=101", { + skip_on_cran() + generate_set_seed() + irace_log <- read_logfile(system.file(package="irace", "exdata", "sann.rda")) + conf_ids <- unlist(tail(irace_log$allElites, n=1L)) + # Use a temporary file to not change the original "sann.rda". + psrace_logFile <- withr::local_tempfile(fileext = ".Rdata") + # Execute the post-selection after the execution of irace. + psRace(irace_log, max_experiments=101, conf_ids = conf_ids, psrace_logFile = psrace_logFile) + irace_log <- read_logfile(psrace_logFile) + budget <- nrow(irace_log$state$experiment_log[iteration == max(iteration)]) + expect_gt(budget, 10L) + # It should be equal but elitist_race sometimes fails to consume all the budget. + expect_lte(budget, irace_log$psrace_log$max_experiments) +}) + +}) # withr::with_output_sink() diff --git a/tests/testthat/test-raceconfs.R b/tests/testthat/test-raceconfs.R index 1df14e8d..cc0f7dc7 100644 --- a/tests/testthat/test-raceconfs.R +++ b/tests/testthat/test-raceconfs.R @@ -17,19 +17,23 @@ L-BFGS-B DE DE SLSQP DE DE ' -target.runner <- function(experiment, scenario) - return(list(cost = 100, call = toString(experiment))) +target_runner <- function(experiment, scenario) + list(cost = 100, call = toString(experiment)) - withr::with_options(list(warning=2), { - parameters <- readParameters(text=parameters_txt) - initconfs <- readConfigurationsFile(text=confs_txt, parameters=parameters) - scenario <- list(targetRunner = target.runner, - instances=1:10, - nbConfigurations=8, - maxExperiments = 96, logFile = "", - initConfigurations=initconfs, - parameters = parameters) - scenario <- checkScenario (scenario) - confs <- irace(scenario = scenario) - }) +withr::with_options(list(warning=2), { + parameters <- readParameters(text=parameters_txt) + initconfs <- readConfigurationsFile(text=confs_txt, parameters=parameters) + scenario <- list(targetRunner = target_runner, + instances = 1:10, + nbConfigurations = 8, + maxExperiments = 96, logFile = "", + initConfigurations = initconfs, + parameters = parameters) + scenario <- checkScenario (scenario) + confs <- irace(scenario = scenario) + confs <- data.table::as.data.table(removeConfigurationsMetaData(confs)) + initconfs <- data.table::as.data.table(initconfs) + expect_equal(nrow(data.table::fsetdiff(confs, initconfs)), 0L) }) + +}) # withr::with_output_sink() diff --git a/tests/testthat/test-readParameters.R b/tests/testthat/test-readParameters.R index bdf2d490..9b77dc09 100644 --- a/tests/testthat/test-readParameters.R +++ b/tests/testthat/test-readParameters.R @@ -1,5 +1,22 @@ withr::with_output_sink("test-readParameters.Rout", { + test_that("optimize conditions", { + p <- readParameters(text=' +ROOT "" c ("S") +ROOT_T "" c ("FIC") | ROOT %in% c("S") +ROOT_T_FIC.sum "" r (-50.0, 50.0) | ROOT_T %in% c("FIC") +ROOT_T.FFI "" c ("true") | ROOT_T == "FIC" +ROOT_T.im "" c ("FFI", "FIC") | ROOT == "S" & ROOT_T.FFI == "true" +ROOT_E.FFI "" c ("false") | ROOT_T_FIC.sum < 0 +ROOT_E.FIC "" i (0,100) | ROOT_E.FFI == "false" +') + expect_identical(p$conditions, + list(ROOT=TRUE, ROOT_T=TRUE, ROOT_T_FIC.sum=TRUE, + ROOT_T.FFI=TRUE, ROOT_T.im=TRUE, + ROOT_E.FFI=expression(ROOT_T_FIC.sum < 0), + ROOT_E.FIC=expression(ROOT_E.FFI == "false"))) + }) + test_that("error checking", { ref <- parametersNew(param_real(name = "tmp", lower = 0, upper=1, diff --git a/tests/testthat/test-readconfs.R b/tests/testthat/test-readconfs.R index db1f7df0..1f2190f0 100644 --- a/tests/testthat/test-readconfs.R +++ b/tests/testthat/test-readconfs.R @@ -1,6 +1,6 @@ withr::with_output_sink("test-readconfs.Rout", { -params <- irace::readParameters("parameters.txt") +params <- readParameters("parameters.txt") expect_error_readconfs <- function(table, exp) expect_error(irace::readConfigurationsFile(text=table, parameters = params), exp) diff --git a/tests/testthat/test-recovery.R b/tests/testthat/test-recovery.R new file mode 100644 index 00000000..4bb0ef63 --- /dev/null +++ b/tests/testthat/test-recovery.R @@ -0,0 +1,108 @@ +withr::with_output_sink("test-recovery.Rout", { + +test_that("recovery works", { + +target_runner_xy <- function(experiment, scenario) +{ + configuration <- experiment$configuration + instance <- experiment$instance + + x <- configuration[["x"]] + y <- configuration[["y"]] + value <- switch(instance, + ackley = f_ackley(x, y), + goldestein = f_goldestein_price(x, y), + matyas = f_matyas(x, y), + himmelblau = f_himmelblau(x, y)) + list(cost = value) +} + +parameters_table <- ' + x "" r (0, 1.00) + y "" r (0, 1.00) + ' + + parameters <- readParameters(text = parameters_table) + logFile <- withr::local_tempfile(pattern = "irace", fileext = ".Rdata") + + seed <- sample.int(min(2147483647L, .Machine$integer.max), size = 1, replace = TRUE) + maxExperiments <- 500L + limit <- 100L + + scenario <- list( + instances = c("ackley", "goldestein", "matyas", "himmelblau"), + parameters = parameters, + targetRunner = target_runner_xy, + logFile = logFile, + seed = seed, + quiet = TRUE, + maxExperiments = maxExperiments) + + expect_silent(confs <- irace(scenario = scenario)) + + scenario$targetRunner <- wrap_target_runner_error(target_runner_xy, limit) + # Otherwise, the tests are too fast. + with_mocked_bindings({ + expect_error(irace(scenario = scenario), "== irace == The cost returned by targetRunner is not numeric") + }, + .irace_minimum_saving_time = 0 + ) + + logFile_new <- withr::local_tempfile(pattern = "irace", fileext = ".Rdata") + scenario <- modifyList(scenario, list( + targetRunner = wrap_target_runner_counter(target_runner_xy), + recoveryFile = logFile, + seed = NA, + quiet = FALSE, + logFile = logFile_new)) + recover_confs <- irace(scenario = scenario) + expect_identical(confs, recover_confs) + expect_lt(environment(scenario$targetRunner)$counter, maxExperiments - 50) +}) + +test_that("recovery maxTime", { + target_runner <- function(experiment, scenario) { + configuration <- experiment$configuration + tmax <- configuration[["tmax"]] + temp <- configuration[["temp"]] + time <- max(1, abs(rnorm(1, mean=(tmax+temp)/10))) + list(cost = time, time = time) + } + parameters <- readParameters(text = ' + tmax "" i (1, 50) + temp "" r (0, 10) + ') + logFile <- withr::local_tempfile(pattern = "irace", fileext = ".Rdata") + seed <- 1234567 + + scenario <- list(targetRunner = target_runner, + instances = 1:10, + seed = seed, + maxTime = 500, + logFile = logFile, + quiet = TRUE, + parameters = parameters) + expect_silent(confs <- irace(scenario = scenario)) + + scenario$targetRunner <- wrap_target_runner_error(target_runner, 200L) + # Otherwise, the tests are too fast. + with_mocked_bindings({ + expect_error(irace(scenario = scenario), "== irace == The cost returned by targetRunner is not numeric") + }, + .irace_minimum_saving_time = 0 + ) + + logFile_new <- withr::local_tempfile(pattern = "irace", fileext = ".Rdata") + scenario <- modifyList(scenario, list( + targetRunner = wrap_target_runner_counter(target_runner), + recoveryFile = logFile, + seed = NA, + quiet = FALSE, + logFile = logFile_new)) + recover_confs <- irace(scenario = scenario) + expect_identical(confs, recover_confs) + expect_lt(environment(scenario$targetRunner)$counter, + nrow(read_logfile(logFile_new)$state$experiment_log) - 150L) +}) + +}) # withr::with_output_sink() diff --git a/tests/testthat/test-repair.R b/tests/testthat/test-repair.R index fd46a74b..7652c0a2 100644 --- a/tests/testthat/test-repair.R +++ b/tests/testthat/test-repair.R @@ -16,7 +16,7 @@ repair_irace <- function(targetRunner, repair) seed = 1234567, parameters = parameters) scenario <- checkScenario (scenario) - irace:::checkTargetFiles(scenario = scenario) + expect_true(irace:::checkTargetFiles(scenario = scenario)) confs <- irace(scenario = scenario) expect_gt(nrow(confs), 0L) confs @@ -64,13 +64,13 @@ repair_order <- function(configuration, parameters) } test_that("repair: sum to one", { - generate.set.seed() + generate_set_seed() confs <- repair_irace(target_sum2one, repair_sum2one) expect_equal(unique(apply(confs[, c("p1", "p2", "p3")], 1L, sum)), 1) }) test_that("repair: increasing order", { - generate.set.seed() + generate_set_seed() confs <- repair_irace(target_order, repair_order) expect_true(all(apply(confs[, c("p1", "p2", "p3")], 1L, diff) >= 0)) }) diff --git a/tests/testthat/test-sann-irace.R b/tests/testthat/test-sann-irace.R index 96d245b4..889658f8 100644 --- a/tests/testthat/test-sann-irace.R +++ b/tests/testthat/test-sann-irace.R @@ -44,7 +44,7 @@ target_runner_reject <- function(experiment, scenario) } ## Run function ######################################################## -sann.irace <- function(log.param=FALSE, ...) +sann_irace <- function(log.param=FALSE, ...) { args <- list(...) @@ -76,40 +76,40 @@ sann.irace <- function(log.param=FALSE, ...) test_that("parallel", { skip_on_cran() # Reproducible results - generate.set.seed() + generate_set_seed() weights <- rnorm(200, mean = 0.9, sd = 0.02) - sann.irace(instances = weights, parallel = test_irace_detectCores()) + sann_irace(instances = weights, parallel = test_irace_detectCores()) }) test_that("parallel reject", { # Reproducible results - generate.set.seed() + generate_set_seed() weights <- rnorm(200, mean = 0.9, sd = 0.02) - sann.irace(instances = weights, parallel = test_irace_detectCores(), targetRunner = target_runner_reject) + sann_irace(instances = weights, parallel = test_irace_detectCores(), targetRunner = target_runner_reject) }) test_that("deterministic", { skip_on_cran() # Reproducible results - generate.set.seed() + generate_set_seed() weights <- rnorm(200, mean = 0.9, sd = 0.02) - sann.irace(deterministic = TRUE, instances = weights[1:7]) + sann_irace(deterministic = TRUE, instances = weights[1:7]) }) test_that("log", { skip_on_cran() # Reproducible results - generate.set.seed() + generate_set_seed() weights <- rnorm(200, mean = 0.9, sd = 0.02) - sann.irace(log.param=TRUE, instances = weights) + sann_irace(log.param=TRUE, instances = weights) }) test_that("large newInstances", { skip_on_cran() # Reproducible results - generate.set.seed() + generate_set_seed() weights <- rnorm(200, mean = 0.9, sd = 0.02) - sann.irace(instances = weights, elitistNewInstances = 6, elitistLimit = 2) + sann_irace(instances = weights, elitistNewInstances = 6, elitistLimit = 2) }) }) # withr::with_output_sink() diff --git a/tests/testthat/test-similar.R b/tests/testthat/test-similar.R index bb604ea7..7be79206 100644 --- a/tests/testthat/test-similar.R +++ b/tests/testthat/test-similar.R @@ -1,6 +1,6 @@ test_that("similarConfigurations", { - parameters <- irace:::readParameters(text = ' + parameters <- readParameters(text = ' n1 "" r (0,1) n2 "" r (0,1) n3 "" r (0,1) diff --git a/tests/testthat/test-sobol.R b/tests/testthat/test-sobol.R index e099445d..8a36d9c7 100644 --- a/tests/testthat/test-sobol.R +++ b/tests/testthat/test-sobol.R @@ -1,8 +1,26 @@ withr::with_output_sink("test-sobol.Rout", { + test_that("bug with conditional dependent", { + parameters <- parametersNew(param_cat(name = "algorithm", values = c("as", "mmas", "ras", "acs")), + param_real(name = "alpha", lower = 0.0, upper=5.0), + param_real(name = "beta", lower = 0.0, upper = 10.0), + param_int(name = "ants", lower = 2, upper = 100), + param_real(name = "q0", lower=0.0, upper=1.0, condition = expression(algorithm == "acs")), + param_int(name = "rasrank", lower=1, upper=quote(min(ants, 10)), condition = 'algorithm == "ras"'), + param_int(name = "eants", lower=0, upper=expression(rasrank)), + param_cat(name = "dlb", values = c(0,1), condition = "localsearch == 1"), + param_int(name = "nnls", lower = 5, upper = 50, condition = expression(dlb == 1)), + param_ord(name = "localsearch", values = c("0", "1")), + param_cat(name = "fixed", values = "0")) + + confs <- irace:::sampleSobol(parameters, 1000L) + expect_equal(nrow(confs), 1000L) + expect_valid_configurations(confs, parameters) + }) + test_that("bug with dependent fixed", { params <- readParameters(text=' -ROOT "ROOT=" c ("SimpleAlgorithm") +ROOT "ROOT=" c ("SimpleAlgorithm") ROOT_SimpleAlgorithm.constructive "ROOT_SimpleAlgorithm.constructive=" c ("FasterInvertedConstructive", "SlowConstructive") | ROOT %in% c("SimpleAlgorithm") ROOT_SimpleAlgorithm.constructive_FasterInvertedConstructive.sumThis "ROOT_SimpleAlgorithm.constructive_FasterInvertedConstructive.sumThis=" r (-50.0, 50.0) | ROOT_SimpleAlgorithm.constructive %in% c("FasterInvertedConstructive") ROOT_SimpleAlgorithm.constructive_SlowConstructive.sumThis "ROOT_SimpleAlgorithm.constructive_SlowConstructive.sumThis=" i (-10, 10) | ROOT_SimpleAlgorithm.constructive %in% c("SlowConstructive") @@ -10,12 +28,13 @@ ROOT_SimpleAlgorithm.improver "ROOT_SimpleAlgorithm.improver=" c ("FlippyFlop ROOT_SimpleAlgorithm.improver_FlippyFlopImprover.enabled "ROOT_SimpleAlgorithm.improver_FlippyFlopImprover.enabled=" c ("true", "false") | ROOT_SimpleAlgorithm.improver %in% c("FlippyFlopImprover") ROOT_SimpleAlgorithm.improver_FlippyFlopImprover.sleepy "ROOT_SimpleAlgorithm.improver_FlippyFlopImprover.sleepy=" c ("8", "6", "12", "11", "7", "5", "4", "10", "1", "9", "2", "3", "13") | ROOT_SimpleAlgorithm.improver %in% c("FlippyFlopImprover") ') - confs <- irace:::sampleSobol(params, 10) - expect_equal(nrow(confs), 10) + confs <- irace:::sampleSobol(params, 10L) + expect_equal(nrow(confs), 10L) + expect_valid_configurations(confs, params) }) test_that("bug with dependent fixed #2", { params <- readParameters(text=' -ROOT "ROOT=" c ("SimpleAlgorithm", "ComplexAlgorithms") +ROOT "ROOT=" c ("SimpleAlgorithm", "ComplexAlgorithms") ROOT_SimpleAlgorithm.constructive "ROOT_SimpleAlgorithm.constructive=" c ("FasterInvertedConstructive", "SlowConstructive") | ROOT %in% c("SimpleAlgorithm") ROOT_SimpleAlgorithm.constructive_FasterInvertedConstructive.sumThis "ROOT_SimpleAlgorithm.constructive_FasterInvertedConstructive.sumThis=" r (-50.0, 50.0) | ROOT_SimpleAlgorithm.constructive %in% c("FasterInvertedConstructive") ROOT_SimpleAlgorithm.constructive_SlowConstructive.sumThis "ROOT_SimpleAlgorithm.constructive_SlowConstructive.sumThis=" i (-10, 10) | ROOT_SimpleAlgorithm.constructive %in% c("SlowConstructive") @@ -25,6 +44,7 @@ ROOT_SimpleAlgorithm.improver_FlippyFlopImprover.sleepy "ROOT_SimpleAlgorithm.i ') confs <- irace:::sampleSobol(params, 10) expect_equal(nrow(confs), 10) + expect_valid_configurations(confs, params) }) }) diff --git a/tests/testthat/test-target-runner-dummy.R b/tests/testthat/test-target-runner-dummy.R index de16143a..e07b0666 100644 --- a/tests/testthat/test-target-runner-dummy.R +++ b/tests/testthat/test-target-runner-dummy.R @@ -33,10 +33,12 @@ withr::with_output_sink("test-target-runner-dummy.Rout", { skip_if_not(nzchar(iraceexe), "Not run because 'irace' is not installed") expect_true(file.exists(iraceexe)) # FIXME: For some reason, this does not generate any output on Windows - output <- expect_silent(system2(iraceexe, "--help", stdout = TRUE, stderr = TRUE)) - skip_on_os("windows") - expect_match(paste(collapse="", output), - "irace: An implementation in R of.*called with: --help") + output <- expect_silent(runexe(iraceexe, "--help")) + ## cat("irace --help\n") + ## print(output) + expected_output <- if (system_os_is_windows()) "^$" + else "irace: An implementation in R.*called with: --help" + expect_match(paste(collapse="", output), expected_output) }) test_that("ablation exe works", { @@ -44,10 +46,12 @@ withr::with_output_sink("test-target-runner-dummy.Rout", { skip_if_not(nzchar(ablationexe), "Not run because 'ablation' is not installed") expect_true(file.exists(ablationexe)) # FIXME: For some reason, this does not generate any output on Windows - output <- expect_silent(system2(ablationexe, "--help", stdout = TRUE, stderr = TRUE)) - skip_on_os("windows") - expect_match(paste(collapse="", output), - "ablation: An implementation in R of Ablation Analysis.*called with: --help") + output <- expect_silent(runexe(ablationexe, "--help")) + ## cat("ablation --help\n") + ## print(output) + expected_output <- if (system_os_is_windows()) "^$" + else "ablation: An implementation in R of Ablation Analysis.*called with: --help" + expect_match(paste(collapse="", output), expected_output) }) run_cmdline <- function(parameters, args) { @@ -65,7 +69,7 @@ withr::with_output_sink("test-target-runner-dummy.Rout", { target_runner_dummy <- get_executable("target-runner-dummy") skip_if_not(nzchar(target_runner_dummy), "Not run because 'target-runner-dummy' is not installed") expect_true(file.exists(target_runner_dummy)) - + test_that("--check", { expect_warning( run_cmdline(paste0('p_int "--p_int " i (1, 10)\n', @@ -91,7 +95,7 @@ withr::with_output_sink("test-target-runner-dummy.Rout", { 'capping "--opt-time " c (1)\n'), '--max-time 1000 --bound-max 100 --capping 1'), "is too large"), - "No scenario file given") + "No scenario file given") }) test_that("--capping", { expect_warning( @@ -120,7 +124,7 @@ withr::with_output_sink("test-target-runner-dummy.Rout", { 'capping "--opt-time " c (1)\n'), '--max-time 500 --bound-max 1 --capping 1 '), "With the current settings and estimated time per run"), - "No scenario file given") + "No scenario file given") }) test_that("Error cost time", { expect_error( @@ -151,5 +155,5 @@ withr::with_output_sink("test-target-runner-dummy.Rout", { '--min-experiments 50'), "No scenario file given")) }) - + }) # withr::with_output_sink() diff --git a/tests/testthat/test-targeteval.R b/tests/testthat/test-targeteval.R index f3009c95..0312a74e 100644 --- a/tests/testthat/test-targeteval.R +++ b/tests/testthat/test-targeteval.R @@ -1,27 +1,106 @@ withr::with_output_sink("test-targeteval.Rout", { -test_that("target.evaluator", { + target_evaluator <- function(experiment, num_configurations, all_conf_id, + scenario, target_runner_call) { + withr::local_seed(experiment$seed) + list(cost = runif(1), call = deparse1(experiment)) + } -target.runner <- function(experiment, scenario) - list(call = toString(experiment)) - -target.evaluator <- function(experiment, num_configurations, all_conf_id, - scenario, target_runner_call) - list(cost = runif(1), call = toString(experiment)) - -parameters <- readParameters(text = ' + parameters <- readParameters(text = ' algorithm "--" c (as,mmas,eas,ras,acs) ') - generate.set.seed() - scenario <- checkScenario(list( - targetRunner = target.runner, targetEvaluator = target.evaluator, - maxExperiments = 200, instances = runif(100), - parameters = parameters)) - irace:::checkTargetFiles(scenario = scenario) - confs <- irace(scenario = scenario) - expect_gt(nrow(confs), 0L) +test_that("target_evaluator", { + + target_runner <- function(experiment, scenario) { + list(call = deparse1(experiment)) + } + seed <- sample.int(min(2147483647L, .Machine$integer.max), size = 1L, replace = TRUE) + instances <- 1:10 + limit <- 58L + maxExperiments <- 200 + logFile <- withr::local_tempfile(pattern = "irace", fileext = ".Rdata") + + scenario <- checkScenario(list( + targetRunner = target_runner, targetEvaluator = target_evaluator, + maxExperiments = maxExperiments, instances = instances, + logFile = logFile, + seed = seed, + quiet = TRUE, + parameters = parameters)) + + expect_true(irace:::checkTargetFiles(scenario = scenario)) + + scenario <- checkScenario(list( + targetRunner = target_runner, targetEvaluator = target_evaluator, + maxExperiments = maxExperiments, instances = instances, + logFile = logFile, + seed = seed, + quiet = TRUE, + parameters = parameters)) + + expect_silent(confs <- irace(scenario = scenario)) + expect_gt(nrow(confs), 0L) + scenario$targetRunner <- wrap_target_runner_error(target_runner, limit) + # Otherwise, the tests are too fast. + with_mocked_bindings({ + expect_error(irace(scenario = scenario), "== irace == The cost returned by targetRunner is not numeric") + }, + .irace_minimum_saving_time = 0 + ) + + logFile_new <- withr::local_tempfile(pattern = "irace", fileext = ".Rdata") + scenario <- modifyList(scenario, list( + targetRunner = wrap_target_runner_counter(target_runner), + recoveryFile = logFile, + seed = NA, + quiet = FALSE, + logFile = logFile_new)) + recover_confs <- irace(scenario = scenario) + expect_identical(confs, recover_confs) + expect_equal(environment(scenario$targetRunner)$counter, maxExperiments - limit + 1L) }) +test_that("target_evaluator maxTime", { + + target_runner <- function(experiment, scenario) { + withr::local_seed(experiment$seed) + list(time = min(experiment$bound, as.integer(1 + 10*runif(1)))) + } + + seed <- sample.int(min(2147483647L, .Machine$integer.max), size = 1L, replace = TRUE) + instances <- 1:10 + limit <- 100L + logFile <- withr::local_tempfile(pattern = "irace", fileext = ".Rdata") + scenario <- checkScenario(list( + targetRunner = target_runner, targetEvaluator = target_evaluator, + maxTime = 2000, boundMax = 10, instances = instances, + logFile = logFile, seed = seed, + quiet = TRUE, + parameters = parameters)) + expect_true(scenario$capping) + expect_silent(confs <- irace(scenario = scenario)) + expect_gt(nrow(confs), 0L) + + scenario$targetRunner <- wrap_target_runner_error(target_runner, limit) + # Otherwise, the tests are too fast. + with_mocked_bindings({ + expect_error(irace(scenario = scenario), "== irace == The cost returned by targetRunner is not numeric") + }, + .irace_minimum_saving_time = 0 + ) + + logFile_new <- withr::local_tempfile(pattern = "irace", fileext = ".Rdata") + scenario <- modifyList(scenario, list( + targetRunner = wrap_target_runner_counter(target_runner), + recoveryFile = logFile, + seed = NA, + quiet = FALSE, + logFile = logFile_new)) + recover_confs <- irace(scenario = scenario) + expect_identical(confs, recover_confs) + expect_lt(environment(scenario$targetRunner)$counter, + sum(!is.na(read_logfile(logFile_new)$experiments)) - 10L) +}) }) # withr::with_output_sink() diff --git a/vignettes/examples.Rdata b/vignettes/examples.Rdata index 59e9a813..96805d49 100644 Binary files a/vignettes/examples.Rdata and b/vignettes/examples.Rdata differ diff --git a/vignettes/irace-acotsp-stdout.txt b/vignettes/irace-acotsp-stdout.txt index 5e57c6b7..e2d0d294 100644 --- a/vignettes/irace-acotsp-stdout.txt +++ b/vignettes/irace-acotsp-stdout.txt @@ -1,9 +1,9 @@ #------------------------------------------------------------------------------ # irace: An implementation in R of (Elitist) Iterated Racing -# Version: 3.9.0.9000.08fb612d -# Copyright (C) 2010-2020 +# Version: 4.2.0.ee928b9 +# Copyright (C) 2010-2025 # Manuel Lopez-Ibanez -# Jeremie Dubois-Lacoste +# Jeremie Dubois-Lacoste # Leslie Perez Caceres # # This is free software, and you are welcome to redistribute it under certain @@ -16,10 +16,10 @@ #------------------------------------------------------------------------------ # installed at: /home/manu/R/x86_64-pc-linux-gnu-library/4.1/irace # called with: --parallel 2 -# 2024-07-26 16:40:13 BST: Reading parameter file '/home/manu/work/irace/git/devel-examples/vignette-example/parameters.txt'. -# 2024-07-26 16:40:13 BST: 1 expression(s) specifying forbidden configurations read. -# 2024-07-26 16:40:13 BST: Read 1 configuration(s) from file '/home/manu/work/irace/git/devel-examples/vignette-example/default.txt' -# 2024-07-26 16:40:13 BST: Initialization +# 2025-01-28 13:47:31 GMT: Reading parameter file '/home/manu/work/irace/git/devel-examples/vignette-example/parameters.txt'. +# 2025-01-28 13:47:31 GMT: 1 expression(s) specifying forbidden configurations read. +# 2025-01-28 13:47:31 GMT: Read 1 configuration(s) from file '/home/manu/work/irace/git/devel-examples/vignette-example/default.txt' +# 2025-01-28 13:47:31 GMT: Initialization # Elitist race # Elitist new instances: 1 # Elitist limit: 2 @@ -32,8 +32,8 @@ # mu: 5 # deterministic: FALSE -# 2024-07-26 16:40:13 BST: Iteration 1 of 5 -# experimentsUsedSoFar: 0 +# 2025-01-28 13:47:31 GMT: Iteration 1 of 5 +# experimentsUsed: 0 # remainingBudget: 1000 # currentBudget: 200 # nbConfigurations: 33 @@ -48,23 +48,24 @@ +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ | | Instance| Alive| Best| Mean best| Exp so far| W time| rho|KenW| Qvar| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ -|x| 1| 33| 10| 23189189.00| 33|00:01:28| NA| NA| NA| -|x| 2| 33| 5| 23331092.00| 66|00:01:26|+0.97|0.99|0.0041| -|x| 3| 33| 10| 23317166.33| 99|00:01:26|+0.96|0.98|0.0085| -|x| 4| 33| 10| 23225711.00| 132|00:01:26|+0.96|0.97|0.0077| -|-| 5| 2| 10| 23215310.80| 165|00:01:26|+0.20|0.36|0.4000| +|x| 1| 33| 22| 33693816.00| 33|00:01:41| NA| NA| NA| +|x| 2| 33| 31| 33233161.00| 66|00:01:41|+0.96|0.98|0.0060| +|x| 3| 33| 31| 33285969.67| 99|00:01:35|+0.97|0.98|0.0052| +|x| 4| 33| 31| 33264133.00| 132|00:01:37|+0.97|0.98|0.0046| +|-| 5| 3| 31| 33251469.40| 165|00:01:36|-0.05|0.16|0.4985| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ -Best-so-far configuration: 10 mean value: 23215310.80 +Best-so-far configuration: 31 mean value: 33251469.40 Description of the best-so-far configuration: - .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. -10 10 acs 3 1.6745 4.0179 0.6924 16 29 0.9348 1 NA NA 5 NA + .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. +31 31 acs 3 4.2717 0.2871 0.9362 6 17 0.918 1 NA NA 5 NA -# 2024-07-26 16:47:29 BST: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): - algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time -10 acs 3 1.6745 4.0179 0.6924 16 29 0.9348 1 NA NA 5 -5 acs 2 3.2370 0.8929 0.1356 9 49 0.1223 0 NA NA 5 -# 2024-07-26 16:47:29 BST: Iteration 2 of 5 -# experimentsUsedSoFar: 165 +# 2025-01-28 13:55:43 GMT: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): + algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time +31 acs 3 4.2717 0.2871 0.9362 6 17 0.918 1 NA NA 5 +9 mmas 3 3.4904 4.9746 0.5959 13 33 NA 0 NA NA 5 +22 as 3 2.0842 2.1621 0.7506 25 26 NA 1 NA NA 5 +# 2025-01-28 13:55:43 GMT: Iteration 2 of 5 +# experimentsUsed: 165 # remainingBudget: 835 # currentBudget: 208 # nbConfigurations: 31 @@ -79,27 +80,31 @@ Description of the best-so-far configuration: +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ | | Instance| Alive| Best| Mean best| Exp so far| W time| rho|KenW| Qvar| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ -|x| 6| 31| 44| 23316661.00| 31|00:01:21| NA| NA| NA| -|x| 3| 31| 45| 23305842.50| 60|00:01:16|+0.81|0.91|0.0160| -|x| 4| 31| 45| 23175565.33| 89|00:01:16|+0.72|0.81|0.0235| -|x| 5| 31| 45| 23182437.25| 118|00:01:16|+0.74|0.80|0.0240| -|-| 2| 6| 45| 23223595.60| 147|00:01:17|+0.50|0.60|0.3482| -|-| 1| 2| 45| 23198916.50| 151|00:00:10|-0.07|0.11|0.5333| +|x| 6| 31| 31| 32987639.00| 31|00:01:31| NA| NA| NA| +|x| 1| 31| 61| 33317599.00| 59|00:01:19|+0.95|0.97|0.0029| +|x| 4| 31| 61| 33270043.67| 87|00:01:19|+0.94|0.96|0.0026| +|x| 3| 31| 61| 33282925.75| 115|00:01:20|+0.94|0.96|0.0025| +|-| 2| 7| 61| 33187955.60| 143|00:01:19|+0.20|0.36|0.7082| +|=| 5| 7| 31| 33207497.67| 147|00:00:12|+0.09|0.24|0.7586| +|=| 7| 7| 31| 33197039.29| 154|00:00:21|+0.16|0.28|0.7095| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ -Best-so-far configuration: 45 mean value: 23198916.50 +Best-so-far configuration: 31 mean value: 33197039.29 Description of the best-so-far configuration: - .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. -45 45 acs 3 2.3609 1.3517 0.648 8 9 0.4225 1 NA NA 5 10 + .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. +31 31 acs 3 4.2717 0.2871 0.9362 6 17 0.918 1 NA NA 5 NA -# 2024-07-26 16:54:08 BST: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): +# 2025-01-28 14:03:09 GMT: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time -45 acs 3 2.3609 1.3517 0.6480 8 9 0.4225 1 NA NA 5 -44 eas 3 1.8176 3.8120 0.1294 40 12 NA 1 NA 433 5 -# 2024-07-26 16:54:08 BST: Iteration 3 of 5 -# experimentsUsedSoFar: 316 -# remainingBudget: 684 -# currentBudget: 228 -# nbConfigurations: 30 +31 acs 3 4.2717 0.2871 0.9362 6 17 0.9180 1 NA NA 5 +61 ras 3 3.6680 2.4269 0.8717 96 14 NA 1 89 NA 5 +37 acs 3 3.8927 1.6181 0.6188 6 35 0.8587 1 NA NA 5 +41 ras 3 3.3674 5.9837 0.7399 12 40 NA 1 95 NA 5 +9 mmas 3 3.4904 4.9746 0.5959 13 33 NA 0 NA NA 5 +# 2025-01-28 14:03:09 GMT: Iteration 3 of 5 +# experimentsUsed: 319 +# remainingBudget: 681 +# currentBudget: 227 +# nbConfigurations: 32 # Markers: x No test is performed. c Configurations are discarded only due to capping. @@ -111,32 +116,28 @@ Description of the best-so-far configuration: +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ | | Instance| Alive| Best| Mean best| Exp so far| W time| rho|KenW| Qvar| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ -|x| 7| 30| 73| 23103320.00| 30|00:01:17| NA| NA| NA| -|x| 6| 30| 73| 23244548.50| 58|00:01:12|+0.88|0.94|0.0157| -|x| 5| 30| 44| 23212749.67| 86|00:01:12|+0.89|0.93|0.0206| -|x| 4| 30| 45| 23148953.75| 114|00:01:12|+0.89|0.92|0.0155| -|-| 1| 8| 45| 23134267.20| 142|00:01:12|-0.13|0.10|0.9890| -|=| 2| 8| 45| 23176594.17| 148|00:00:15|-0.06|0.12|0.9120| -|=| 3| 8| 45| 23188036.86| 154|00:00:15|-0.03|0.12|0.8647| -|=| 8| 8| 45| 23196884.00| 162|00:00:20|+0.01|0.13|0.8402| +|x| 8| 32| 65| 32811562.00| 32|00:01:27| NA| NA| NA| +|x| 2| 32| 75| 32779249.00| 59|00:01:16|+0.92|0.96|0.0031| +|x| 6| 32| 75| 32799039.00| 86|00:01:16|+0.92|0.95|0.0036| +|x| 7| 32| 75| 32842164.00| 113|00:01:16|+0.89|0.92|0.0044| +|-| 5| 8| 75| 32875613.60| 140|00:01:16|+0.64|0.71|0.2166| +|-| 1| 7| 75| 32974384.00| 143|00:00:10|+0.66|0.71|0.2031| +|-| 4| 6| 75| 32972595.86| 145|00:00:05|+0.53|0.60|0.2424| +|-| 3| 1| 75| 32995307.50| 146|00:00:05| NA| NA| NA| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ -Best-so-far configuration: 45 mean value: 23196884.00 +Best-so-far configuration: 75 mean value: 32995307.50 Description of the best-so-far configuration: - .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. -45 45 acs 3 2.3609 1.3517 0.648 8 9 0.4225 1 NA NA 5 10 + .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. +75 75 ras 3 4.5827 1.6918 0.818 13 10 NA 1 15 NA 5 37 -# 2024-07-26 17:01:06 BST: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): - algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time -45 acs 3 2.3609 1.3517 0.6480 8 9 0.4225 1 NA NA 5 -65 acs 3 1.5782 1.0607 0.6276 15 17 0.7555 1 NA NA 5 -76 acs 3 4.6355 1.3756 0.7615 5 30 0.1230 1 NA NA 5 -85 acs 3 2.4145 1.0828 0.1661 5 16 0.6588 1 NA NA 5 -44 eas 3 1.8176 3.8120 0.1294 40 12 NA 1 NA 433 5 -# 2024-07-26 17:01:06 BST: Iteration 4 of 5 -# experimentsUsedSoFar: 478 -# remainingBudget: 522 -# currentBudget: 261 -# nbConfigurations: 33 +# 2025-01-28 14:10:04 GMT: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): + algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time +75 ras 3 4.5827 1.6918 0.818 13 10 NA 1 15 NA 5 +# 2025-01-28 14:10:04 GMT: Iteration 4 of 5 +# experimentsUsed: 465 +# remainingBudget: 535 +# currentBudget: 267 +# nbConfigurations: 30 # Markers: x No test is performed. c Configurations are discarded only due to capping. @@ -148,34 +149,34 @@ Description of the best-so-far configuration: +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ | | Instance| Alive| Best| Mean best| Exp so far| W time| rho|KenW| Qvar| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ -|x| 9| 33| 85| 22966599.00| 33|00:01:27| NA| NA| NA| -|x| 5| 33| 103| 23071137.50| 61|00:01:12|+0.53|0.77|0.5240| -|x| 4| 33| 103| 23013928.33| 89|00:01:12|+0.56|0.71|0.4244| -|x| 7| 33| 103| 23042138.25| 117|00:01:12|+0.62|0.71|0.3418| -|-| 2| 17| 103| 23125805.00| 145|00:01:13|-0.10|0.12|1.0326| -|=| 1| 17| 111| 23118782.83| 157|00:00:30|-0.08|0.10|1.0105| -|=| 8| 17| 108| 23142870.43| 169|00:00:32|-0.05|0.10|0.9931| -|=| 3| 17| 108| 23148337.25| 181|00:00:31|-0.01|0.12|0.9448| -|=| 6| 17| 45| 23179679.44| 193|00:00:31|-0.03|0.09|0.9534| -|=| 10| 17| 108| 23193047.90| 210|00:00:46|-0.00|0.10|0.9401| +|x| 9| 30| 113| 32734055.00| 30|00:01:21| NA| NA| NA| +|x| 6| 30| 91| 32806654.50| 59|00:01:21|+0.83|0.91|0.0624| +|x| 2| 30| 91| 32738738.33| 88|00:01:21|+0.75|0.83|0.1103| +|x| 5| 30| 117| 32780373.75| 117|00:01:21|+0.72|0.79|0.1597| +|-| 1| 12| 117| 32893747.40| 146|00:01:21|-0.01|0.19|0.9196| +|=| 4| 12| 113| 32874137.17| 157|00:00:32|+0.05|0.20|0.8612| +|=| 3| 12| 113| 32910758.71| 168|00:00:32|+0.12|0.25|0.8001| +|-| 8| 6| 113| 32892247.62| 179|00:00:32|+0.08|0.19|0.7116| +|=| 7| 6| 113| 32894494.00| 184|00:00:16|+0.09|0.19|0.6930| +|=| 10| 6| 113| 32912596.60| 190|00:00:16|+0.01|0.11|0.7806| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ -Best-so-far configuration: 108 mean value: 23193047.90 +Best-so-far configuration: 113 mean value: 32912596.60 Description of the best-so-far configuration: - .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. -108 108 acs 3 1.7392 1.9072 0.7255 41 11 0.4645 1 NA NA 5 45 + .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. +113 113 ras 3 4.1042 1.531 0.8104 16 12 NA 1 24 NA 5 75 -# 2024-07-26 17:10:17 BST: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): - algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time -108 acs 3 1.7392 1.9072 0.7255 41 11 0.4645 1 NA NA 5 -45 acs 3 2.3609 1.3517 0.6480 8 9 0.4225 1 NA NA 5 -112 acs 3 3.2349 2.2329 0.0832 5 17 0.5623 1 NA NA 5 -103 acs 3 1.2519 3.4743 0.4586 42 15 0.7227 1 NA NA 5 -94 acs 3 1.1587 2.1115 0.3135 17 21 0.9533 1 NA NA 5 -# 2024-07-26 17:10:17 BST: Iteration 5 of 5 -# experimentsUsedSoFar: 688 -# remainingBudget: 312 -# currentBudget: 312 -# nbConfigurations: 32 +# 2025-01-28 14:19:01 GMT: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): + algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time +113 ras 3 4.1042 1.5310 0.8104 16 12 NA 1 24 NA 5 +117 ras 3 4.1817 0.5324 0.8893 13 18 NA 1 23 NA 5 +92 ras 3 4.9632 4.1134 0.8790 7 9 NA 1 26 NA 5 +102 ras 3 4.3582 2.8458 0.7508 9 19 NA 1 86 NA 5 +93 ras 3 4.1500 2.0082 0.8067 10 14 NA 1 32 NA 5 +# 2025-01-28 14:19:01 GMT: Iteration 5 of 5 +# experimentsUsed: 655 +# remainingBudget: 345 +# currentBudget: 345 +# nbConfigurations: 35 # Markers: x No test is performed. c Configurations are discarded only due to capping. @@ -187,36 +188,36 @@ Description of the best-so-far configuration: +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ | | Instance| Alive| Best| Mean best| Exp so far| W time| rho|KenW| Qvar| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ -|x| 11| 32| 136| 23136304.00| 32|00:01:21| NA| NA| NA| -|x| 9| 32| 103| 23069252.50| 59|00:01:11|+0.51|0.75|0.0197| -|x| 1| 32| 108| 23095136.00| 86|00:01:14|+0.56|0.71|0.0376| -|x| 8| 32| 108| 23135933.25| 113|00:01:11|+0.61|0.71|0.0371| -|-| 5| 14| 124| 23122405.00| 140|00:01:11|-0.05|0.16|0.9814| -|=| 3| 14| 108| 23152299.83| 149|00:00:25|-0.02|0.15|0.9692| -|=| 6| 14| 119| 23182651.57| 158|00:00:25|-0.03|0.11|0.9825| -|=| 10| 14| 108| 23207197.50| 167|00:00:25|-0.02|0.11|0.9587| -|=| 2| 14| 108| 23232719.89| 176|00:00:25|-0.02|0.09|0.9612| -|=| 7| 14| 108| 23219207.80| 185|00:00:25|+0.03|0.13|0.9175| -|=| 4| 14| 108| 23188771.00| 194|00:00:25|+0.03|0.12|0.9157| -|=| 12| 14| 108| 23192893.83| 208|00:00:35|+0.02|0.10|0.9239| +|x| 11| 35| 124| 32571228.00| 35|00:01:37| NA| NA| NA| +|x| 2| 35| 147| 32589096.50| 65|00:01:21|+0.28|0.64|0.1066| +|x| 1| 35| 102| 32841930.67| 95|00:01:21|+0.42|0.61|0.0852| +|x| 3| 35| 102| 32917142.50| 125|00:01:21|+0.48|0.61|0.0727| +|-| 4| 19| 102| 32911207.20| 155|00:01:23|-0.06|0.15|0.9874| +|=| 7| 19| 113| 32914603.83| 169|00:00:38|-0.07|0.10|0.9949| +|=| 9| 19| 113| 32888811.14| 183|00:00:38|-0.02|0.12|0.9456| +|=| 8| 19| 113| 32873043.50| 197|00:00:37|+0.02|0.14|0.9046| +|=| 5| 19| 113| 32870705.56| 211|00:00:38|+0.05|0.15|0.8924| +|=| 10| 19| 113| 32891187.00| 225|00:00:37|+0.03|0.13|0.9091| +|=| 6| 19| 118| 32883264.36| 239|00:00:37|+0.03|0.12|0.9045| +|=| 12| 19| 118| 32868008.67| 258|00:00:54|+0.05|0.13|0.8860| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ -Best-so-far configuration: 108 mean value: 23192893.83 +Best-so-far configuration: 118 mean value: 32868008.67 Description of the best-so-far configuration: - .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. -108 108 acs 3 1.7392 1.9072 0.7255 41 11 0.4645 1 NA NA 5 45 + .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. +118 118 ras 3 4.0526 2.5427 0.7522 17 20 NA 1 9 NA 5 113 -# 2024-07-26 17:19:37 BST: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): - algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time -108 acs 3 1.7392 1.9072 0.7255 41 11 0.4645 1 NA NA 5 -119 acs 3 1.8415 1.5342 0.2492 41 10 0.7389 1 NA NA 5 -141 acs 3 1.8949 0.1375 0.7172 41 8 0.2731 1 NA NA 5 -138 acs 3 1.2856 0.5537 0.7490 38 9 0.8335 1 NA NA 5 -45 acs 3 2.3609 1.3517 0.6480 8 9 0.4225 1 NA NA 5 -# 2024-07-26 17:19:37 BST: Iteration 6 of 6 -# experimentsUsedSoFar: 896 -# remainingBudget: 104 -# currentBudget: 104 -# nbConfigurations: 12 +# 2025-01-28 14:30:48 GMT: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): + algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time +118 ras 3 4.0526 2.5427 0.7522 17 20 NA 1 9 NA 5 +124 ras 3 3.1663 0.6971 0.8396 24 26 NA 1 19 NA 5 +113 ras 3 4.1042 1.5310 0.8104 16 12 NA 1 24 NA 5 +117 ras 3 4.1817 0.5324 0.8893 13 18 NA 1 23 NA 5 +133 ras 3 3.1718 0.9297 0.8386 9 24 NA 1 23 NA 5 +# 2025-01-28 14:30:48 GMT: Iteration 6 of 6 +# experimentsUsed: 913 +# remainingBudget: 87 +# currentBudget: 87 +# nbConfigurations: 11 # Markers: x No test is performed. c Configurations are discarded only due to capping. @@ -228,38 +229,38 @@ Description of the best-so-far configuration: +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ | | Instance| Alive| Best| Mean best| Exp so far| W time| rho|KenW| Qvar| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ -|x| 13| 12| 146| 22849509.00| 12|00:00:30| NA| NA| NA| -|x| 11| 12| 108| 23000546.50| 19|00:00:20|+0.87|0.93|0.0256| -|x| 4| 12| 108| 22961832.00| 26|00:00:20|+0.72|0.81|0.0893| -|x| 5| 12| 108| 23017239.25| 33|00:00:20|+0.69|0.77|0.0934| -|-| 7| 6| 108| 23033311.20| 40|00:00:20|-0.04|0.17|0.7911| -|=| 8| 6| 138| 23065830.17| 41|00:00:05|-0.01|0.16|0.8081| -|=| 1| 6| 108| 23076105.14| 42|00:00:05|-0.08|0.07|0.8881| -|=| 12| 6| 108| 23096372.62| 43|00:00:05|-0.06|0.07|0.8634| -|=| 2| 6| 108| 23134208.89| 44|00:00:05|-0.09|0.03|0.8857| -|=| 10| 6| 108| 23149440.40| 45|00:00:05|-0.08|0.03|0.8970| -|=| 3| 6| 108| 23152819.00| 46|00:00:05|-0.05|0.05|0.8745| -|=| 9| 6| 108| 23142713.33| 47|00:00:05|-0.03|0.06|0.8642| -|=| 6| 6| 108| 23166909.00| 48|00:00:05|-0.05|0.03|0.8790| -|=| 14| 6| 108| 23165023.00| 54|00:00:15|-0.03|0.04|0.8762| +|x| 13| 11| 113| 32697505.00| 11|00:00:32| NA| NA| NA| +|x| 4| 11| 113| 32789761.00| 17|00:00:16|+0.55|0.77|0.4924| +|x| 2| 11| 113| 32743042.67| 23|00:00:16|+0.62|0.75|0.4061| +|x| 5| 11| 113| 32770282.50| 29|00:00:16|+0.62|0.72|0.4564| +|-| 1| 6| 113| 32872017.00| 35|00:00:16|+0.24|0.39|0.6408| +|!| 9| 6| 113| 32849023.33| 36|00:00:05|+0.31|0.42|0.6091| +|=| 3| 6| 113| 32889232.57| 37|00:00:05|+0.18|0.29|0.6964| +|=| 12| 6| 113| 32872605.38| 38|00:00:05|+0.04|0.16|0.7812| +|=| 7| 6| 113| 32877034.22| 39|00:00:05|-0.05|0.07|0.8518| +|=| 8| 6| 113| 32865597.80| 40|00:00:05|-0.06|0.05|0.8491| +|=| 11| 6| 113| 32844551.82| 41|00:00:05|-0.04|0.06|0.8388| +|=| 6| 6| 113| 32844854.83| 42|00:00:05|-0.06|0.03|0.8736| +|=| 10| 6| 118| 32867735.77| 43|00:00:05|-0.05|0.03|0.8810| +|=| 14| 6| 118| 32851875.71| 49|00:00:16|-0.05|0.03|0.8818| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ -Best-so-far configuration: 108 mean value: 23165023.00 +Best-so-far configuration: 118 mean value: 32851875.71 Description of the best-so-far configuration: - .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. -108 108 acs 3 1.7392 1.9072 0.7255 41 11 0.4645 1 NA NA 5 45 + .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. +118 118 ras 3 4.0526 2.5427 0.7522 17 20 NA 1 9 NA 5 113 -# 2024-07-26 17:22:25 BST: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): - algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time -108 acs 3 1.7392 1.9072 0.7255 41 11 0.4645 1 NA NA 5 -119 acs 3 1.8415 1.5342 0.2492 41 10 0.7389 1 NA NA 5 -141 acs 3 1.8949 0.1375 0.7172 41 8 0.2731 1 NA NA 5 -138 acs 3 1.2856 0.5537 0.7490 38 9 0.8335 1 NA NA 5 -146 acs 3 2.7520 1.2268 0.4055 56 6 0.6684 1 NA NA 5 -# 2024-07-26 17:22:26 BST: Iteration 7 of 7 -# experimentsUsedSoFar: 950 -# remainingBudget: 50 -# currentBudget: 50 -# nbConfigurations: 8 +# 2025-01-28 14:33:27 GMT: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): + algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time +118 ras 3 4.0526 2.5427 0.7522 17 20 NA 1 9 NA 5 +113 ras 3 4.1042 1.5310 0.8104 16 12 NA 1 24 NA 5 +149 ras 3 3.7080 0.4873 0.9452 8 11 NA 1 50 NA 5 +117 ras 3 4.1817 0.5324 0.8893 13 18 NA 1 23 NA 5 +124 ras 3 3.1663 0.6971 0.8396 24 26 NA 1 19 NA 5 +# 2025-01-28 14:33:27 GMT: Iteration 7 of 7 +# experimentsUsed: 962 +# remainingBudget: 38 +# currentBudget: 38 +# nbConfigurations: 7 # Markers: x No test is performed. c Configurations are discarded only due to capping. @@ -271,52 +272,51 @@ Description of the best-so-far configuration: +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ | | Instance| Alive| Best| Mean best| Exp so far| W time| rho|KenW| Qvar| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ -|x| 15| 8| 155| 23394351.00| 8|00:00:20| NA| NA| NA| -|x| 8| 8| 138| 23314313.50| 11|00:00:10|+0.36|0.68|0.1608| -|x| 6| 8| 119| 23321571.67| 14|00:00:10|+0.27|0.51|0.2394| -|x| 3| 8| 119| 23294773.50| 17|00:00:10|+0.27|0.45|0.2627| -|=| 12| 8| 138| 23290689.20| 20|00:00:10|+0.24|0.39|0.2368| -|-| 7| 7| 119| 23255303.17| 23|00:00:10|-0.08|0.10|0.9184| -|=| 13| 7| 119| 23201610.29| 25|00:00:05|-0.06|0.09|0.8946| -|=| 11| 7| 108| 23213088.88| 27|00:00:05|-0.05|0.08|0.8845| -|=| 5| 7| 119| 23194511.11| 29|00:00:05|-0.00|0.11|0.8413| -|=| 10| 7| 119| 23208386.80| 31|00:00:05|+0.03|0.13|0.8318| -|=| 2| 7| 141| 23227483.18| 33|00:00:05|+0.03|0.12|0.8199| -|=| 1| 7| 141| 23220665.75| 35|00:00:05|-0.02|0.06|0.8710| -|=| 9| 7| 108| 23211615.46| 37|00:00:05|-0.02|0.05|0.8729| -|=| 4| 7| 108| 23188243.14| 39|00:00:05|-0.02|0.06|0.8655| -|=| 14| 7| 119| 23178977.27| 41|00:00:05|-0.02|0.05|0.8580| -|=| 16| 7| 108| 23176226.25| 48|00:00:20|-0.00|0.06|0.8465| +|x| 15| 7| 149| 32886640.00| 7|00:00:21| NA| NA| NA| +|x| 10| 7| 149| 32935718.00| 9|00:00:05|+0.21|0.61|0.8607| +|x| 14| 7| 118| 32827495.67| 11|00:00:05|+0.11|0.40|0.7721| +|x| 13| 7| 149| 32825055.00| 13|00:00:05|+0.26|0.44|0.7264| +|=| 11| 7| 113| 32781621.00| 15|00:00:05|+0.20|0.36|0.6939| +|=| 2| 7| 149| 32759707.67| 17|00:00:05|+0.06|0.21|0.8239| +|=| 12| 7| 149| 32751285.86| 19|00:00:05|+0.10|0.23|0.7552| +|=| 7| 7| 149| 32771017.12| 21|00:00:05|+0.02|0.15|0.8085| +|=| 5| 7| 149| 32787355.11| 23|00:00:05|+0.06|0.17|0.7630| +|=| 3| 7| 149| 32819324.50| 25|00:00:05|+0.04|0.14|0.8019| +|=| 4| 7| 149| 32831015.73| 27|00:00:05|+0.05|0.13|0.7921| +|=| 1| 7| 149| 32874709.83| 29|00:00:05|+0.05|0.13|0.7786| +|=| 9| 7| 113| 32854146.77| 31|00:00:05|+0.03|0.11|0.8026| +|=| 6| 7| 149| 32859769.21| 33|00:00:05|+0.05|0.12|0.7833| +|=| 8| 7| 149| 32855221.33| 35|00:00:05|+0.06|0.13|0.7587| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ -Best-so-far configuration: 108 mean value: 23176226.25 +Best-so-far configuration: 149 mean value: 32855221.33 Description of the best-so-far configuration: - .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. -108 108 acs 3 1.7392 1.9072 0.7255 41 11 0.4645 1 NA NA 5 45 + .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. +149 149 ras 3 3.708 0.4873 0.9452 8 11 NA 1 50 NA 5 117 -# 2024-07-26 17:24:44 BST: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): - algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time -108 acs 3 1.7392 1.9072 0.7255 41 11 0.4645 1 NA NA 5 -119 acs 3 1.8415 1.5342 0.2492 41 10 0.7389 1 NA NA 5 -146 acs 3 2.7520 1.2268 0.4055 56 6 0.6684 1 NA NA 5 -138 acs 3 1.2856 0.5537 0.7490 38 9 0.8335 1 NA NA 5 -141 acs 3 1.8949 0.1375 0.7172 41 8 0.2731 1 NA NA 5 -# 2024-07-26 17:24:44 BST: Stopped because there is not enough budget left to race more than the minimum (5). +# 2025-01-28 14:35:06 GMT: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): + algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time +149 ras 3 3.7080 0.4873 0.9452 8 11 NA 1 50 NA 5 +113 ras 3 4.1042 1.5310 0.8104 16 12 NA 1 24 NA 5 +118 ras 3 4.0526 2.5427 0.7522 17 20 NA 1 9 NA 5 +117 ras 3 4.1817 0.5324 0.8893 13 18 NA 1 23 NA 5 +124 ras 3 3.1663 0.6971 0.8396 24 26 NA 1 19 NA 5 +# 2025-01-28 14:35:06 GMT: Stopped because there is not enough budget left to race more than the minimum (5). # You may either increase the budget or set 'minNbSurvival' to a lower value. # Iteration: 8 # nbIterations: 8 -# experimentsUsedSoFar: 998 +# experimentsUsed: 997 # timeUsed: 0 -# remainingBudget: 2 -# currentBudget: 2 +# remainingBudget: 3 +# currentBudget: 3 # number of elites: 5 # nbConfigurations: 5 -# Total CPU user time: 5065.315, CPU sys time: 47.329, Wall-clock time: 2670.861 -# 2024-07-26 17:24:44 BST: Starting post-selection: -# 2024-07-26 17:24:44 BST: Configurations selected: 108, 119. -# 2024-07-26 17:24:44 BST: Pending instances: 0, 0. -# Seed: 687542627 -# Configurations: 2 -# Available experiments: 2 +# Total CPU user time: 5397.903, CPU sys time: 78.97, Wall-clock time: 2855.561 +# 2025-01-28 14:35:06 GMT: Starting post-selection: +# Configurations selected: 149, 113, 118. +# Pending instances: 0, 0, 0. +# 2025-01-28 14:35:07 GMT: seed: 687542627 +# Configurations: 3 +# Available experiments: 3 # minSurvival: 1 # Markers: x No test is performed. @@ -329,72 +329,115 @@ Description of the best-so-far configuration: +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ | | Instance| Alive| Best| Mean best| Exp so far| W time| rho|KenW| Qvar| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ -|.| 14| 2| 119| 23114658.00| 0|00:00:00| NA| NA| NA| -|.| 3| 2| 119| 23164518.50| 0|00:00:00|-1.00|0.00|1.0000| -|.| 8| 2| 119| 23189668.67| 0|00:00:00|-0.33|0.11|0.6667| -|.| 16| 2| 119| 23161428.50| 0|00:00:00|-0.33|0.00|0.6667| -|.| 11| 2| 119| 23163622.00| 0|00:00:00|-0.20|0.04|0.6000| -|.| 10| 2| 119| 23191896.33| 0|00:00:00|-0.07|0.11|0.5333| -|.| 1| 2| 119| 23182644.57| 0|00:00:00|+0.05|0.18|0.4762| -|.| 12| 2| 119| 23195234.75| 0|00:00:00|+0.14|0.25|0.4286| -|.| 6| 2| 119| 23208937.00| 0|00:00:00|+0.00|0.11|0.5000| -|.| 2| 2| 119| 23233372.10| 0|00:00:00|+0.07|0.16|0.4667| -|.| 7| 2| 119| 23218461.82| 0|00:00:00|-0.02|0.07|0.5091| -|.| 4| 2| 119| 23189132.92| 0|00:00:00|-0.06|0.03|0.5303| -|.| 9| 2| 119| 23177599.31| 0|00:00:00|-0.03|0.05|0.5128| -|.| 5| 2| 119| 23176837.36| 0|00:00:00|-0.05|0.02|0.5275| -|.| 13| 2| 119| 23157011.73| 0|00:00:00|-0.03|0.04|0.5143| -|.| 15| 2| 119| 23172585.44| 0|00:00:00|-0.05|0.02|0.5250| -|=| 17| 2| 108| 23186535.53| 2|00:00:05|-0.03|0.03|0.5147| +|.| 8| 3| 118| 32752987.00| 0|00:00:00| NA| NA| NA| +|.| 7| 3| 118| 32831948.00| 0|00:00:00|-0.50|0.25|1.1636| +|.| 12| 3| 118| 32788030.67| 0|00:00:00|+0.17|0.44|0.7047| +|.| 15| 3| 118| 32818929.25| 0|00:00:00|-0.08|0.19|0.7650| +|.| 14| 3| 118| 32784282.40| 0|00:00:00|-0.20|0.04|0.8230| +|.| 2| 3| 118| 32753987.33| 0|00:00:00|-0.10|0.08|0.7476| +|.| 6| 3| 118| 32749294.86| 0|00:00:00|+0.00|0.14|0.6861| +|.| 3| 3| 118| 32795767.12| 0|00:00:00|+0.07|0.19|0.6370| +|.| 9| 3| 118| 32798232.11| 0|00:00:00|-0.03|0.09|0.6834| +|.| 10| 3| 118| 32810925.60| 0|00:00:00|+0.02|0.12|0.6534| +|.| 13| 3| 118| 32815792.45| 0|00:00:00|-0.04|0.06|0.6765| +|.| 1| 3| 118| 32856770.08| 0|00:00:00|-0.07|0.02|0.7054| +|.| 11| 3| 118| 32843955.00| 0|00:00:00|-0.08|0.01|0.7140| +|.| 5| 3| 118| 32849549.21| 0|00:00:00|-0.07|0.01|0.7135| +|.| 4| 3| 118| 32855859.00| 0|00:00:00|-0.07|0.00|0.7072| +|=| 16| 3| 113| 32847622.06| 3|00:00:10|-0.06|0.00|0.7069| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ -Best-so-far configuration: 108 mean value: 23186535.53 +Best-so-far configuration: 113 mean value: 32847622.06 Description of the best-so-far configuration: - .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. -108 108 acs 3 1.7392 1.9072 0.7255 41 11 0.4645 1 NA NA 5 45 + .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. +113 113 ras 3 4.1042 1.531 0.8104 16 12 NA 1 24 NA 5 75 -# 2024-07-26 17:24:49 BST: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): - algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time -108 acs 3 1.7392 1.9072 0.7255 41 11 0.4645 1 NA NA 5 -119 acs 3 1.8415 1.5342 0.2492 41 10 0.7389 1 NA NA 5 -# Total CPU user time: 5075.465, CPU sys time: 47.431, Wall-clock time: 2676.009 +# 2025-01-28 14:35:17 GMT: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): + algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time +113 ras 3 4.1042 1.5310 0.8104 16 12 NA 1 24 NA 5 +118 ras 3 4.0526 2.5427 0.7522 17 20 NA 1 9 NA 5 +149 ras 3 3.7080 0.4873 0.9452 8 11 NA 1 50 NA 5 +# Total CPU user time: 5414.035, CPU sys time: 79.259, Wall-clock time: 2866.547 # Best configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): - algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time -108 acs 3 1.7392 1.9072 0.7255 41 11 0.4645 1 NA NA 5 -119 acs 3 1.8415 1.5342 0.2492 41 10 0.7389 1 NA NA 5 + algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time +113 ras 3 4.1042 1.5310 0.8104 16 12 NA 1 24 NA 5 +118 ras 3 4.0526 2.5427 0.7522 17 20 NA 1 9 NA 5 +149 ras 3 3.7080 0.4873 0.9452 8 11 NA 1 50 NA 5 # Best configurations as commandlines (first number is the configuration ID; listed from best to worst according to the sum of ranks): -108 --acs --localsearch 3 --alpha 1.7392 --beta 1.9072 --rho 0.7255 --ants 41 --nnls 11 --q0 0.4645 --dlb 1 --time 5 -119 --acs --localsearch 3 --alpha 1.8415 --beta 1.5342 --rho 0.2492 --ants 41 --nnls 10 --q0 0.7389 --dlb 1 --time 5 +113 --ras --localsearch 3 --alpha 4.1042 --beta 1.531 --rho 0.8104 --ants 16 --nnls 12 --dlb 1 --rasranks 24 --time 5 +118 --ras --localsearch 3 --alpha 4.0526 --beta 2.5427 --rho 0.7522 --ants 17 --nnls 20 --dlb 1 --rasranks 9 --time 5 +149 --ras --localsearch 3 --alpha 3.708 --beta 0.4873 --rho 0.9452 --ants 8 --nnls 11 --dlb 1 --rasranks 50 --time 5 -# Testing of elite configurations: 5 -# Testing iteration configurations: TRUE -# 2024-07-26 17:24:49 BST: Testing configurations (in no particular order): 10 5 45 44 65 76 85 108 112 103 94 119 141 138 146 +# Testing of elite configurations: 5 +# Testing iteration configurations: TRUE +# 2025-01-28 14:35:18 GMT: Testing configurations (in no particular order): 31 9 22 61 37 41 75 113 117 92 102 93 118 124 133 149 algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time -10 acs 3 1.6745 4.0179 0.6924 16 29 0.9348 1 NA NA 5 -5 acs 2 3.2370 0.8929 0.1356 9 49 0.1223 0 NA NA 5 -45 acs 3 2.3609 1.3517 0.6480 8 9 0.4225 1 NA NA 5 -44 eas 3 1.8176 3.8120 0.1294 40 12 NA 1 NA 433 5 -65 acs 3 1.5782 1.0607 0.6276 15 17 0.7555 1 NA NA 5 -76 acs 3 4.6355 1.3756 0.7615 5 30 0.1230 1 NA NA 5 -85 acs 3 2.4145 1.0828 0.1661 5 16 0.6588 1 NA NA 5 -108 acs 3 1.7392 1.9072 0.7255 41 11 0.4645 1 NA NA 5 -112 acs 3 3.2349 2.2329 0.0832 5 17 0.5623 1 NA NA 5 -103 acs 3 1.2519 3.4743 0.4586 42 15 0.7227 1 NA NA 5 -94 acs 3 1.1587 2.1115 0.3135 17 21 0.9533 1 NA NA 5 -119 acs 3 1.8415 1.5342 0.2492 41 10 0.7389 1 NA NA 5 -141 acs 3 1.8949 0.1375 0.7172 41 8 0.2731 1 NA NA 5 -138 acs 3 1.2856 0.5537 0.7490 38 9 0.8335 1 NA NA 5 -146 acs 3 2.7520 1.2268 0.4055 56 6 0.6684 1 NA NA 5 -# 2024-07-26 17:31:15 BST: Testing results (column number is configuration ID in no particular order): - seeds 10 5 45 44 65 76 85 108 112 103 94 119 141 138 146 -1t 1569194624 23428468 23611720 23335072 23418902 23366444 23354462 23384148 23409850 23401463 23366982 23373177 23396273 23331449 23466607 23368936 -2t 2141208548 23392230 23303742 23214064 23299315 23153951 23206569 23200275 23214663 23145147 23149476 23195844 23167440 23230580 23137859 23213402 -3t 1648754028 23086793 23268934 23025144 23108338 23032681 23081589 23058615 23098408 23085992 23098849 23063345 23028198 23079569 23084634 23098011 -4t 1116160272 23104893 23313512 23072521 23159865 23098524 23108077 23101026 23033254 23090507 23055489 23082011 23027180 23066772 23098211 23047228 -5t 1682044763 23247804 23393004 23232877 23260905 23247440 23286403 23203563 23225636 23256739 23204694 23241322 23172494 23195335 23239104 23265720 -6t 1200580020 23468768 23611004 23501132 23432133 23414990 23498047 23443069 23411674 23418225 23404301 23481524 23456813 23421776 23410287 23418507 -7t 652316604 23516548 23463318 23421431 23467354 23360479 23405867 23325936 23344578 23406818 23351218 23401264 23330841 23338088 23367653 23356813 -8t 371492673 23373850 23345624 23279921 23296563 23265183 23306797 23264743 23260374 23229259 23233504 23249694 23219584 23279206 23288807 23267451 -9t 1144046723 23379543 23499305 23357416 23536070 23336107 23342437 23343394 23311237 23281159 23328219 23318124 23309017 23285611 23328544 23328301 -10t 754704287 23044975 23100683 23084329 23139348 23096753 23039784 23113793 23051231 23054564 23110133 23071944 23041882 23055953 23110158 23051498 -# 2024-07-26 17:31:15 BST: Finished testing +31 acs 3 4.2717 0.2871 0.9362 6 17 0.9180 1 NA NA 5 +9 mmas 3 3.4904 4.9746 0.5959 13 33 NA 0 NA NA 5 +22 as 3 2.0842 2.1621 0.7506 25 26 NA 1 NA NA 5 +61 ras 3 3.6680 2.4269 0.8717 96 14 NA 1 89 NA 5 +37 acs 3 3.8927 1.6181 0.6188 6 35 0.8587 1 NA NA 5 +41 ras 3 3.3674 5.9837 0.7399 12 40 NA 1 95 NA 5 +75 ras 3 4.5827 1.6918 0.8180 13 10 NA 1 15 NA 5 +113 ras 3 4.1042 1.5310 0.8104 16 12 NA 1 24 NA 5 +117 ras 3 4.1817 0.5324 0.8893 13 18 NA 1 23 NA 5 +92 ras 3 4.9632 4.1134 0.8790 7 9 NA 1 26 NA 5 +102 ras 3 4.3582 2.8458 0.7508 9 19 NA 1 86 NA 5 +93 ras 3 4.1500 2.0082 0.8067 10 14 NA 1 32 NA 5 +118 ras 3 4.0526 2.5427 0.7522 17 20 NA 1 9 NA 5 +124 ras 3 3.1663 0.6971 0.8396 24 26 NA 1 19 NA 5 +133 ras 3 3.1718 0.9297 0.8386 9 24 NA 1 23 NA 5 +149 ras 3 3.7080 0.4873 0.9452 8 11 NA 1 50 NA 5 +# 2025-01-28 15:14:38 GMT: Testing results (column number is configuration ID in no particular order): + seeds 31 9 22 61 37 41 75 113 117 92 102 93 118 124 133 149 +1t 2046302398 33098140 33239149 33129540 33109131 33201943 32976999 32852645 32833572 32784419 32875049 32777346 32820444 32780508 32923537 32871318 32805333 +2t 827626108 32954263 32987443 33039177 33056915 32887007 32780271 32567740 32607386 32661338 32605363 32590997 32514163 32595732 32744276 32605256 32534045 +3t 978077451 33313400 33446626 33376846 33415690 33387989 33266730 33183722 33097013 33077258 33090479 33197150 33032552 33062375 33147446 33042611 33083261 +4t 1348269770 33021447 32897210 33165710 33045317 32904477 33068992 32790813 32824418 32867503 32755126 32926804 32786122 32782807 32923497 32806207 32739172 +5t 243391689 32944849 33053706 33204377 33003552 32978293 33010655 32819909 32919999 32726458 32790603 32821804 32783965 32760191 32934499 32771715 32921237 +6t 1588668262 32690232 32914113 32953951 32875620 32928834 32963678 32717001 32651566 32595506 32617190 32749972 32626934 32573375 32602356 32606513 32692338 +7t 423372130 33026281 33211159 33194496 33148754 33213814 33170418 32939764 32989898 32895022 32943900 32836359 32892448 32884365 33044123 32877345 32974480 +8t 652122407 33071578 33255488 33244533 33055455 33040474 33118029 32937849 32891239 32898150 32855158 32925102 32884699 33028371 32911267 32946823 33040194 +9t 317806051 33033130 33463764 33244304 33208355 33143082 33190011 32874390 32849625 32913245 32933978 32994331 32925936 32887713 33015638 32747034 32812335 +10t 747706567 32823698 32971836 32989937 32984342 32843959 32913020 32706959 32756032 32756123 32765789 32614143 32719902 32728343 32857316 32700242 32868859 +11t 50933476 33002325 33221790 33224313 33062546 32955083 33021896 32967532 32857920 32912572 32883301 32937237 32842348 33001929 32989392 32982455 32855097 +12t 1739405996 32929209 32941102 32856706 33027699 32825183 32820039 32587056 32598397 32561786 32506831 32581913 32643712 32519887 32636143 32595651 32504224 +13t 1957203093 32788760 32959449 33065012 32989738 32823546 33016071 32661707 32638211 32607545 32684776 32619844 32841206 32575148 32759115 32739988 32687250 +14t 2032826826 32831117 33062008 32923055 32885600 32905397 32973393 32562020 32627456 32606172 32803159 32630108 32648439 32832893 32796466 32593158 32662798 +15t 1206236755 32893521 33041201 33083686 33068523 33049312 32984492 32699498 32633428 32733892 32745855 32724253 32731723 32677060 32928235 32739696 32665586 +16t 763851782 33159005 33378520 33316334 33228824 33111096 33182851 32900293 33104319 32979011 32964066 32928362 33018843 32927669 33144143 32968303 32984175 +17t 349953428 33536691 33491757 33602689 33536763 33359766 33523909 33238122 33127750 33073259 33175404 33148729 33110551 33272981 33264666 33129590 33190112 +18t 897599183 32616720 32867845 32930559 32859355 32788206 32793763 32566177 32556947 32550547 32516828 32705214 32629259 32429610 32573661 32525739 32593153 +19t 1400970198 33203949 33268680 33373564 33183534 33177561 33272717 33039458 32908001 32889287 32964453 33028715 32959378 32917060 32981568 33004692 33082049 +20t 1782793253 32920981 33086354 33144045 32971140 33023459 33035694 32692608 32735331 32742981 32743665 32755353 32721542 32768699 32951370 32864674 32829495 +21t 1660845956 33060600 32970450 33071257 32906030 33161795 33015612 32682340 32780886 32764194 32784629 32666204 32614331 32811076 32868182 32716060 32684395 +22t 290410408 32969734 33240958 33158577 33077389 33089633 33099623 32791908 32898456 32846216 32885236 32964063 32795078 32902746 32869519 32718033 32851606 +23t 1495337927 33058071 33043020 33172331 33129424 32900129 33131786 32825696 32716477 32857494 32876402 32902149 32739830 32773697 32750528 32780460 32807391 +24t 1364123825 32936519 33175700 33216248 33127283 33119860 33094549 32862215 32855988 32975309 32893142 32914054 32846935 32862968 33047339 32841995 33002341 +25t 915618954 33221437 33340356 33324125 33267849 33350663 33061580 32938672 32895342 32896788 32923194 33025517 32940073 32916386 33044500 33003287 32811036 +26t 1066725944 33309234 33413626 33378515 33377976 33100601 33284699 33064354 32973237 32962348 33040082 32977859 32956226 33102636 33196297 33077908 33060265 +27t 1012167492 33116237 33212972 33144225 33187194 33123043 33089640 32775498 32747432 32728200 32870284 32843344 32744397 32722944 33026740 32873759 32812007 +28t 405072419 32944552 33183445 33260768 33190870 33030890 33070083 32809618 32824751 32825054 32804746 32709328 32709832 32788312 32880987 32854797 32763291 +29t 343646063 33287500 33435506 33446599 33374038 33291798 33367811 33077554 33075585 33179998 33127231 33178973 33074167 33081719 33222527 32937251 33268605 +30t 2062824562 32892958 33144194 33079207 33019337 33016877 32994210 32758852 32730821 32705369 32776245 32768151 32684416 32675816 32801298 32757259 32647672 +31t 1315404608 33362159 33420071 33375785 33466081 33364889 33308878 33035136 33050152 32977172 33184455 33100022 33086417 33007821 33060061 33201336 33111088 +32t 718135261 33080581 33148836 33178597 33209213 33075259 33090111 32808179 33073806 32824862 32862792 32910569 32809320 32832661 32916787 33013878 32868270 +33t 1756600186 32912578 33140256 33172058 33122171 33082595 32980922 32945569 32778835 32836996 32805414 32804116 32776818 32789844 32877954 32783432 32825277 +34t 1084471193 33058926 33089505 33039589 33036017 32921606 32947257 32721148 32709831 32685964 32693570 32687319 32690564 32707820 32743104 32705370 32621660 +35t 1163321042 32483594 32752995 32803194 32828186 32771137 32690789 32506559 32396349 32366269 32479742 32416846 32376150 32458495 32567191 32361580 32418296 +36t 1318942249 32670682 32971766 32936095 32913150 32717950 32698178 32527927 32575185 32547154 32572291 32474010 32532734 32627421 32750209 32653173 32612411 +37t 424612740 32701106 33002602 32819010 32800764 32830202 32743558 32373724 32380917 32391978 32440573 32431067 32539230 32508876 32564875 32388837 32421787 +38t 507331628 33045967 33268562 33103741 33098088 32963127 32987509 32829137 32679588 32726309 32747132 32804395 32698469 32753361 32922700 32763021 32692158 +39t 1640400685 33006811 33199652 33170436 33106016 33188579 33118545 32864846 32896914 32870078 32916441 32911873 32885323 32959799 33071726 32922589 32905922 +40t 1364449617 32623835 32874869 32761751 32828312 32583441 32723534 32457904 32452281 32404861 32344336 32366792 32410717 32465983 32524448 32533048 32335468 +41t 752897771 33218038 33453910 33489170 33450105 33482242 33400984 33084008 33033849 33008163 33011304 32990212 32965457 33100836 33246586 33109029 33052769 +42t 1025182031 33308181 33416507 33348551 33353589 33078035 33194525 32937731 33096404 32873261 32856868 32900612 32859590 32917712 33107581 32950684 32954981 +43t 696942227 32794732 33077276 32835350 33004223 32905747 32813582 32543692 32458725 32599183 32662034 32659345 32519506 32665713 32802829 32620304 32619233 +44t 1699723808 32859196 33014447 33005264 33098533 32851130 32863953 32647020 32620048 32544882 32629933 32675907 32701896 32691564 32808682 32804464 32680084 +45t 410855221 32973357 33262500 33202569 33189160 33055825 33044003 32646518 32844211 32823867 32790803 32658075 32812383 32899920 32935307 32971815 32763179 +46t 1502438571 32936173 33260372 33144182 33085343 33088220 33037123 32752761 32735861 32749653 32834556 32824232 32788860 32760354 32856958 32786384 32726020 +47t 1580656301 32814152 33179790 32882078 32970050 32863158 32909574 32521834 32706354 32582668 32607947 32648027 32492769 32721354 32846821 32790095 32548698 +48t 1099243655 32864590 33060995 33010305 32994239 32896624 32941142 32602521 32629683 32612903 32561942 32651405 32725952 32587778 32799409 32722572 32540867 +49t 1611639012 33389112 33558167 33487694 33554983 33454508 33472490 33183976 33256869 33202243 33186646 33124595 33205308 33311120 33235674 33293977 33179711 +50t 1429022466 32683888 32708498 32842868 32898058 32851506 32778780 32610345 32425351 32506621 32515430 32592616 32513821 32588071 32672271 32437732 32531407 +# 2025-01-28 15:14:38 GMT: Finished testing diff --git a/vignettes/irace-package.Rnw b/vignettes/irace-package.Rnw index c2c15e3f..4ede3f67 100644 --- a/vignettes/irace-package.Rnw +++ b/vignettes/irace-package.Rnw @@ -130,6 +130,7 @@ \emph{default:}~\texttt{#4} \\ } \newcommand{\parameter}[1]{\hyperlink{opt:#1}{\code{#1}}} +\newcommand{\iracefun}[1]{\href{https://mlopez-ibanez.github.io/irace/reference/#1.html}{\code{#1()}}} %\usepackage{showlabels} %\showlabels{hypertarget} @@ -598,9 +599,9 @@ target algorithm; the default value is the current directory. \item[\parameter{maxExperiments}] (\code{--max-experiments}): the maximum number of executions of the target algorithm that \irace will perform. \item[\parameter{maxTime}] (\code{--max-time}): maximum total execution time in seconds for the executions of \code{targetRunner}. In this case, \code{targetRunner} must return two values: cost and time. Note that you must provide either \parameter{maxTime} or \parameter{maxExperiments}. -\item[\parameter{trainInstancesDir}] (\code{--train-instances-dir}): set if to \path{./Instances} if you put the training instances in that folder as instructed above. +\item[\parameter{trainInstancesDir}] (\code{--train-instances-dir}): set to \path{./Instances} if you put the training instances in that folder as instructed above. \end{description} -For setting the tuning budget see \autoref{sec:budget}. For more information on \irace options and their default values, see \autoref{sec:irace options}. +For setting the tuning budget, see \autoref{sec:budget}. For more information on \irace options and their default values, see \autoref{sec:irace options}. \item Modify the \code{target-runner} script to run your algorithm. This script must execute your algorithm with the parameters and instance specified by @@ -608,7 +609,7 @@ For setting the tuning budget see \autoref{sec:budget}. For more information on time (\code{cost [time]}). When the \parameter{maxTime} option is used, returning \code{time} is mandatory. The \code{target-runner} template is written in \proglang{GNU Bash} scripting language, which can be executed easily in GNU/Linux and OS X - systems. However, you may use any other programming language. We provide examples written in \proglang{Python}, \proglang{MATLAB} and other languages in \IRACEHOME{/examples/}. % + systems. However, you may use any other programming language. We provide examples written in \proglang{Python}, \proglang{MATLAB} and other languages in \IRACEHOME{/examples/}. An example using Julia is available at \url{https://github.com/sbomsdorf/An-example-of-irace-using-Julia-code}. % Follow these instructions to adjust the given \code{target-runner} template to your algorithm: \begin{enumerate} @@ -710,7 +711,7 @@ cd ./tuning/ # with --scenario. $IRACE_HOME/bin/irace \end{lstlisting} -For this example we assume that the needed scenario files have been set +For this example, we assume that the needed scenario files have been set properly in the \code{scenario.txt} file using the options described in \autoref{sec:irace options}. Most \irace options can be specified in the command line or directly in the \code{scenario.txt} file. @@ -758,44 +759,47 @@ located in the \code{examples} folder to the created directory: mkdir ./tuning cd ./tuning # $IRACE_HOME is the installation directory of irace. -cp $IRACE_HOME/examples/acotsp/* ./tuning/ +cp $IRACE_HOME/examples/acotsp/* ./ +ls ./ +# Make sure that target-runner is executable +chmod u+x target-runner \end{lstlisting} %@ -\item Download the training instances from \url{https://iridia.ulb.ac.be/supp/IridiaSupp2016-003/index.html} to the \path{./tuning/} directory. -\item Create the instance directory (\eg~\path{./tuning/Instances}) and decompress the instance files on it. +\item Download the training instances from \url{https://iridia.ulb.ac.be/supp/IridiaSupp2016-003/scenarios/acotsp/instances.tar.gz} to the \path{./tuning/} directory and decompress it, which creates create a folder \path{instances}: %<>= \begin{lstlisting}[style=BashInputStyle] -mkdir ./tuning/Instances/ -cd ./tuning/ -tar -xvf tsp-instances-training.tar.bz2 Instances/ +tar -xvf instances.tar.gz +ls instances/ \end{lstlisting} %@ -\item Download the \ACOTSP software from \url{http://www.aco-metaheuristic.org/aco-code/} to the \path{./tuning/} -directory and compile it. +If the above gives an error or does not show any files, then the files were not extract correctly. Maybe the \path{instances.tar.gz} file did not download correctly or maybe it is not in the correct place. It should be within the folder \path{tuning}.correctly. + +\item Download the \ACOTSP software from \url{https://github.com/MLopez-Ibanez/ACOTSPQAP/archive/refs/heads/master.zip} to the \path{./tuning/} +directory and compile the \path{acotsp} executable using \code{make}. %<>= \begin{lstlisting}[style=BashInputStyle] -cd ./tuning/ -tar -xvf ACOTSP-1.03.tgz -cd ./tuning/ACOTSP-1.03 -make +unzip master.zip +make -C ACOTSPQAP-master acotsp +./ACOTSPQAP-master/acotsp --help \end{lstlisting} %@ +If the above gives an error, then the \path{acotsp} executable failed to compile for some reason. Maybe you are missing the C compiler or some files did not extract correctly. + \item Create a directory for executing the experiments and execute \irace: %<>= \begin{lstlisting}[style=BashInputStyle] -mkdir ./tuning/acotsp-arena/ -cd ./tuning/ +mkdir ./acotsp-arena/ # $IRACE_HOME is the installation directory of irace. $IRACE_HOME/bin/irace \end{lstlisting} %@ -\item Or you can also execute \irace from the \aR console using: +Or you can also execute \irace from the \aR console using: <>= library("irace") @@ -805,6 +809,17 @@ irace_cmdline() \end{enumerate} +The most usual sources of error when running the above commands are: +\begin{itemize} +\item The \irace package is not correctly installed. Please make sure that installing \irace did not give any errors. +\item The location of the files is not correct. Please make sure that you have: + \begin{itemize} + \item The folder \path{tuning} and that it contains the files \path{scenario.txt}, \path{parameters-acotsp.txt}, \path{target-runner}, and the folders \path{instances}, \path{ACOTSPQAP-master} and \path{acotsp-arena}. + \item The folder \path{instances} should contain the TSP instance files (\path{*.tsp}). + \item The folder \path{ACOTSPQAP-master} should contain the executable \path{acotsp}. You should be able to invoke \code{./ACOTSPQAP-master/acotsp --help} without an error. + \item The folder \path{acotsp-arena} should be empty. + \end{itemize} +\end{itemize} %% %% %% @@ -1204,8 +1219,6 @@ list. The structure of the \code{experiment} list is as follows: \code{instance} & string giving the instance to be used for this evaluation;\\ \code{bound} & optional execution time bound;\\ \code{configuration} & 1-row data frame with a column per parameter name;\\ - \code{switches} & vector of parameter switches (labels) in the order of parameters used in \code{configuration}.\\ - \end{tabularx} \end{center} @@ -1215,6 +1228,9 @@ The following is an example of an experiment list for the \ACOTSP scenario: print(experiment) @ +If you need the \code{switches} of each parameter (or any other information +about the parameters), you can obtain it from \code{scenario$parameters}. + You can find an example that calls MATLAB from \aR using this approach here: \url{https://github.com/MLopez-Ibanez/irace/blob/master/inst/examples/matlab/scenario.txt}. If \parameter{targetEvaluator} is \code{NULL}, then @@ -1351,7 +1367,7 @@ specify where to find the training instances. If you only set the value of \parameter{trainInstancesDir} (e.g., to \path{./Instances}), \irace will consider all files within that directory as training instances. Otherwise, the value of \parameter{trainInstancesFile} may specify a text -file. The format of this file is one instance per line. Within each line, elements separated by white-space will be parsed as separate arguments to be supplied to \parameter{targetRunner}. This allows defining instance-specific parameter settings. Quoted strings will be parsed as a single argument. +file. The format of this file is one instance per line. Within each line, elements separated by white-space will be parsed as separate arguments to be supplied to \parameter{targetRunner}. This allows defining instance-specific parameter settings. Quoted strings will be parsed as a single argument. % The following example shows a training instance file for the \ACOTSP scenario: @@ -1614,7 +1630,7 @@ See the examples in @ % However, the user is free to set up the calls in any way, perhaps - implementing its own replacement for \code{target_runner} and/or + implementing their own replacement for \code{target_runner} and/or \code{exec_target_runner}. The user may load and call other \aR packages, such as \pkg{batchtools} (\url{https://mllg.github.io/batchtools/}). The only requirement is that the \parameter{targetRunnerParallel} function @@ -2235,7 +2251,7 @@ stderr <- function(x) sqrt(var(x)/length(x)) err <- apply(iraceResults$testing$experiments[, elites], 2L, stderr) plot(fes, values, type = "s", xlab = "Number of runs of the target algorithm", - ylab = "Mean value over testing set", ylim=c(23000000,23500000)) + ylab = "Mean value over testing set", ylim=c(min(values-err),max(values+err))) points(fes, values, pch=19) arrows(fes, values - err, fes, values + err, length=0.05, angle=90, code=3) text(fes, values, elites, pos = 1) @@ -2527,12 +2543,13 @@ instances, while they perform poorly for a different subset. When facing a heterogeneous scenario, the first question should be whether the objective of tuning is to find configurations that perform reasonably well over -all instances, even if that configuration is not the best ones in any particular instance (a generalist). If this is -not the goal, then it would be better to partition instances into more similar -subsets and execute \irace separately on each subset. This will lead to a -portfolio of algorithm configurations, one for each subset, and algorithm -selection techniques can be used to select the best configuration from the -portfolio when facing a new instance. +all instances, even if that configuration is not the best ones in any +particular instance (a generalist). If this is not the goal, then it would be +better to partition instances into more similar subsets and execute \irace +separately on each subset. This will lead to a portfolio of algorithm +configurations, one for each subset, and algorithm selection techniques can be +used to select the best configuration from the portfolio when facing a new +instance. To make sure \irace is not misled by results on few instances, it may be useful to increase the number of instances executed before doing a statistical test @@ -2553,10 +2570,11 @@ create ``blocks'' of instances in \parameter{trainInstancesFile}, where each block should contain one instance from each class. Then set the option \parameter{blockSize} to the number of classes within each block, so that \irace will always see a complete block of instances before eliminating -configurations. The value of \parameter{blockSize} will multiply the effective -values of \parameter{firstTest} and \parameter{eachTest}. Randomly sampling -instances (\parameter{sampleInstances}\code{=1}) will randomly sample the -blocks but not break the blocks. +configurations. The value of \parameter{blockSize} will multiply the values of +\parameter{firstTest}, \parameter{eachTest} and +\parameter{elitistNewInstances}. Randomly sampling instances +(\parameter{sampleInstances}\code{=1}) will randomly sample the blocks but not +break the blocks. While executing \irace, the homogeneity of the scenario can be observed by examining the values of Spearman's rank correlation coefficient and Kendall's @@ -2734,17 +2752,25 @@ the target configuration. This can be used to find new better ``intermediate'' configurations or to analyse the impact of the parameters in the performance. -To perform ablation, you can use the \code{ablation()} \aR function or the -\path{ablation} command-line executable (see more details below). You may -specify the IDs of the source and target configurations. By default, the -source is taken as the first configuration evaluated by \irace and the target -as the best overall configuration found. Use the function \code{plotAblation} to visualize the ablation results -(Fig.~\ref{fig:testAb}). - +To perform ablation, you can use the \code{ablation()} \aR function +% <>= -ablog <- ablation("irace.Rdata", src = 1, target = 60) +ablog <- ablation("irace.Rdata", src = 1) plotAblation(ablog) @ +% +\noindent or the \path{ablation} command-line executable (see more details below). +% +%<>= +\begin{lstlisting}[style=BashInputStyle] +ablation -l irace.Rdata --src 1 --plot plot-ablation.pdf --plot-type rank,boxplot +\end{lstlisting} +%@ + +You may specify the IDs of the source and target configurations. By default, the +source is taken as the first configuration evaluated by \irace and the target +as the best overall configuration found. Use the function \code{plotAblation} to visualize the ablation results +(Fig.~\ref{fig:testAb}). <>= logfile <- system.file(package="irace", "exdata", "log-ablation.Rdata", mustWork=TRUE) @@ -2766,40 +2792,45 @@ We also provide a command-line executable (\path{ablation.exe} in Windows) that ablation_cmdline("--help") @ -\subsection{Postselection race}\label{sec:postselection} +\subsection{Post-selection race}\label{sec:postselection} -After the configuration process is finished it is possible perform a -postselection race by specifying the \irace option \parameter{postselection} -with value larger than $0$. This option will perform a post-selection race of -the set of best configurations of each iteration. The budget assigned for this -race is obtained using the \parameter{postselection} option which defines a -percentage of the \irace configuration budget. This budget is not considered in -the total configuration budget that is, these evaluations are extra -computation. +At the end of the configuration process, \irace will automatically run a +post-selection race \citep{YuaStuMonLauBir13} to try to consume any remaining +budget. This post-selection race will not generate new configurations. It will +only do additional runs on configurations already evaluated in the current run +of \irace. First, it will try to evaluate the best configurations found in new +instances not seen yet. If this is not possible due to the budget available, +then it will evaluate configurations that have been evaluated on fewer +instances than the best configuration found. Currently, this automatic +post-selection is not supported when the budget is measured in runtime +(\parameter{maxTime}. The automatic post-selection can be disabled using the +option \parameter{postselection}. -The execution of the postselection race add an element (\code{psrace.log}) to -the \code{iraceResults} list saved in the \irace log file. The postselection log -consists of a list with the following elements: -\begin{description} - \item \code{configurations}: Configurations used in the postselection race. - \item \code{instances}: Instances used in the in the postselection race. - \item \code{maxExperiments}: Configuration budget assigned for the postselection race. - \item \code{experiments}: Matrix of experiments in the same format as the - \code{iraceResults$experiments} matrix. The column names indicate the candidate - configuration identifier and the row names contain the name of the instances. - \item \code{elites}: Elite configurations obtained in the postselection race. -\end{description} +It is also possible to manually run a post-selection race after completing a run of \irace using the R function \iracefun{psRace}. -Optionally, it is possible to perform a postselection race with all -elite configurations of the iterations or selecting a set of configurations -from \code{iraceResults$allConfigurations}. <>= # Execute all elite configurations in the iterations -psRace("irace.Rdata", max_experiments = 0.5, elites=TRUE) +psRace("irace.Rdata", max_experiments = 0.5, iteration_elites=TRUE) # Execute a set of configurations IDs providing budget psRace("irace.Rdata", conf_ids = c(34, 87, 102, 172, 293), max_experiments = 500) @ +The execution of the post-selection race adds an element (\code{psrace_log}) to +the \code{iraceResults} object saved in the \irace log file +(\parameter{logFile}). The post-selection log consists of a list with the +following elements: +\begin{description} +\item\code{configurations}: Configurations used in the post-selection race. +\item\code{instances}: Data frame with the instances and seeds used in the post-selection race. +\item \code{max_experiments}: Configuration budget assigned to the post-selection race. +\item \code{experiments}: Matrix of results generated by the post-selection + race, in the same format as the matrix + \code{iraceResults$experiments}. Column names are the configuration IDs and + row names are the instance IDs. +\item \code{elites}: Best configuration(s) found in the post-selection race. +\end{description} + + \subsection[Parameter importance analysis using PyImp]{Parameter importance analysis using \PyImp} The \PyImp\footnote{\url{https://github.com/automl/ParameterImportance}} tool developed by the AutoML group\footnote{\url{https://www.automl.org/}} @@ -3077,6 +3108,11 @@ C:\path\to\python.exe C:\path\to\target-runner.py %instance% %seed% \ This error may arise if you activate the \parameter{parallel} option of \irace and your \parameter{targetRunner} or \parameter{targetEvaluator} tries to setup a parallel cluster or execute code in parallel in a way that interacts badly with the parallel mechanism in \aR. In this case, you need to either investigate yourself if there is a way for the two parallel mechanisms to co-exist or, if that is not possible, disable parallelism in \irace or in your code. Note that packages or software used by your \parameter{targetRunner} may have a parallel mechanism enabled by default and unknown to you. This is definitely NOT a bug in \irace. +\subsection[irace does not print the call to the target-runner with --debug-level 2 when using the --parallel option]{\irace does not print the call to the \parameter{targetRunner} with \parameter{debugLevel}\code{=2} when using the \parameter{parallel}} + +This is a limitation of Windows or Rstudio. Running without \parameter{parallel} should work. Unfortunately, we cannot fix this limitation in \irace. If you need to understand how \irace calls \parameter{targetRunner} when running in parallel, you can implement a logging mechanism able to handle parallelism directly inside the \parameter{targetRunner}. + + %% %% %% diff --git a/vignettes/irace-package.bib b/vignettes/irace-package.bib index 82e1397a..a1b329fb 100644 --- a/vignettes/irace-package.bib +++ b/vignettes/irace-package.bib @@ -12,8 +12,12 @@ @string{add-cham @string{add-heidelberg = { Heidelberg }} +@string{add-ny = { New York, NY }} + @string{aaaip-pub = {{AAAI} Press}} +@string{acm-pub = {ACM Press}} + @string{ieeep = {IEEE Press}} @string{ieeep-ad = {Piscataway, NJ}} @@ -32,6 +36,10 @@ @string{cec @string{cec2006 = proc_of_the # {2006} # cec # { 2006)}} +@string{gecco = proc_of_the # {Genetic and Evolutionary Computation Conference, GECCO }} + +@string{gecco2013 = gecco # {2013}} + @string{icml = { International Conference on Machine Learning, {ICML} }} @string{icml2014 = proc_of_the # {31st} # icml # {2014}} @@ -48,6 +56,8 @@ @string{orp @string{telo = {ACM Transactions on Evolutionary Learning and Optimization}} +@string{alba_e = { Alba, Enrique }} + @string{balaprakash = { Prasanna Balaprakash }} @string{bartz-beielstein = { Thomas Bartz-Beielstein }} @@ -58,6 +68,8 @@ @string{biedenkapp @string{birattari = { Mauro Birattari }} +@string{blum = { Christian Blum }} + @string{branke = { J{\"u}rgen Branke }} @string{chiarandini = { Marco Chiarandini }} @@ -76,6 +88,8 @@ @string{hoos @string{hutter = { Frank Hutter }} +@string{lau_hc = { Hoong Chuin Lau }} + @string{leyton-brown = { Kevin Leyton-Brown }} @string{lindauer_m = { Marius Thomas Lindauer }} @@ -84,6 +98,8 @@ @string{lopez-ibanez @string{mcgeoch_cc = { Catherine C. McGeoch }} +@string{montesdeoca = { Marco A. {Montes de Oca} }} + @string{paquete = { Lu{\'i}s Paquete }} @string{pardalos = { Panos M. Pardalos }} @@ -360,6 +376,13 @@ @incollection{SchHoo2012quanti doi = {10.1007/978-3-642-34413-8_14} } +@incollection{YuaStuMonLauBir13, + author = yuan_z # and # montesdeoca # and # stuetzle # and # lau_hc # and # birattari, + title = {An Analysis of Post-selection in Automatic Configuration}, + pages = {1557--1564}, + crossref = {GECCO2013} +} + @book{AAAI2017, booktitle = aaai, editor = {Satinder P. Singh and Shaul Markovitch}, @@ -393,6 +416,17 @@ @proceedings{CEC2006 address = ieeep-ad } +@book{GECCO2013, + title = {Genetic and Evolutionary Computation Conference, GECCO 2013, + Proceedings, Amsterdam, The Netherlands, July 6-10, 2013}, + booktitle = gecco2013, + editor = blum # and # alba_e, + year = 2013, + publisher = acm-pub, + address = add-ny, + isbn = {978-1-4503-1963-8} +} + @proceedings{ICML2014, editor = {Xing, Eric P. and Jebara, Tony}, title = {Proceedings of the 31st International Conference on Machine @@ -441,4 +475,3 @@ @book{LION2017 publisher = springer, address = add-cham } - diff --git a/vignettes/section/irace-options.Rnw b/vignettes/section/irace-options.Rnw index c9cf777e..46ab3e35 100644 --- a/vignettes/section/irace-options.Rnw +++ b/vignettes/section/irace-options.Rnw @@ -4,118 +4,121 @@ \subsection[General options]{General options} \begin{description} \defparameter[h]{-{}-help}{help}{}% -Show the list of command-line options of \irace. + Show the list of command-line options of \irace. \defparameter[v]{-{}-version}{version}{}% -Show the version of \irace. + Show the version of \irace. \defparameter[c]{-{}-check}{check}{}% -Check that the scenario and parameter definitions are correct and test the execution of the target algorithm. See \autoref{sec:execution}. + Check that the scenario and parameter definitions are correct and test the execution of the target algorithm. See \autoref{sec:execution}. \defparameter[i]{-{}-init}{init}{}% -Initialize the working directory with the template config files. This copies the files in \code{\$IRACE_HOME/templates} to the working directory without overwriting the files with the same names as those of the template files. + Initialize the working directory with the template config files. This copies the files in \code{\$IRACE_HOME/templates} to the working directory without overwriting the files with the same names as those of the template files. \defparameter[s]{scenarioFile}{scenario}{./scenario.txt}% -File that contains the scenario setup and other irace options. All options listed in this section can be included in this file. See \IRACEHOME{/templates/} for an example. Relative file-system paths specified in the scenario file are relative to the scenario file itself. + File that contains the scenario setup and other irace options. All options listed in this section can be included in this file. See \IRACEHOME{/templates/} for an example. Relative file-system paths specified in the scenario file are relative to the scenario file itself. \defparameter{execDir}{exec-dir}{./}% -Directory where the target runner will be run. The default execution directory is the current directory. + Directory where the target runner will be run. The default execution directory is the current directory. \begin{xwarningbox} The execution directory must exist before executing \irace, it will not be created automatically. \end{xwarningbox} \defparameter[l]{logFile}{log-file}{./irace.Rdata}% - File to save tuning results as an \aR dataset. The provided path must be either an absolute path or relative to \parameter{execDir}. See \autoref{sec:output r} for details on the format of the \aR dataset. + File to save tuning results as an \aR dataset. The provided path must be either an absolute path or relative to \parameter{execDir}. See \autoref{sec:output r} for details on the format of the \aR dataset. + + \defparameter[q]{quiet}{quiet}{0}% + Reduce the output generated by irace to a minimum. \defparameter{debugLevel}{debug-level}{0}% -Level of information to display in the text output of \irace. A value of 0 silences all debug messages. Higher values provide more verbose debug messages. Details about the text output of \irace are given in \autoref{sec:output text}. + Level of information to display in the text output of \irace. A value of 0 silences all debug messages. Higher values (1, 2 or 3) provide more verbose debug messages. Details about the text output of \irace are given in \autoref{sec:output text}. \defparameter{seed}{seed}{}% -Seed to initiallize the random number generator. The seed must be a positive integer. If the seed is \code{""} or \code{NULL}, a random seed will be generated. + Seed to initiallize the random number generator. The seed must be a positive integer. If the seed is \code{""} or \code{NULL}, a random seed will be generated. \defparameter{repairConfiguration}{}{}% -User-defined \aR function that takes a configuration generated by \irace and repairs it. See \autoref{sec:repairconf} for details. + User-defined \aR function that takes a configuration generated by \irace and repairs it. See \autoref{sec:repairconf} for details. \defparameter{postselection}{postselection}{1}% -Perform a postselection race after the execution of \irace to consume all remaining budget. Value 0 disables the postselection race. See \autoref{sec:postselection}. + Perform a postselection race after the execution of \irace to consume all remaining budget. Value 0 disables the postselection race. See \autoref{sec:postselection}. \defparameter{aclib}{aclib}{0}% -Enable/disable AClib mode. This option enables compatibility with \texttt{GenericWrapper4AC} (\url{https://github.com/automl/GenericWrapper4AC/}) as \parameter{targetRunner} script. + Enable/disable AClib mode. This option enables compatibility with \texttt{GenericWrapper4AC} (\url{https://github.com/automl/GenericWrapper4AC/}) as \parameter{targetRunner} script. \end{description} \subsection[Elitist irace]{Elitist \irace} \begin{description} \defparameter[e]{elitist}{elitist}{1}% -Enable/disable elitist \irace. + Enable/disable elitist \irace. In the \textbf{elitist} version of \code{irace}~\citep{LopDubPerStuBir2016irace}, elite configurations are not discarded from the race until non-elite configurations have been executed on the same instances as the elite configurations. - Each race begins by evaluating all configurations on a number of new instances. This number is defined by the option \parameter{elitistNewInstances}. After the new instances have been evaluated, configurations are evaluated on instances seen in the previous race. Elite configurations already have results for most of these previous instances and, therefore, do not need to be re-evaluated. Finally, after configurations have been evaluated on all these instances, the race continues by evaluating additional new instances. + Each race begins by evaluating all configurations on a number of new instances. This number is defined by the option \parameter{elitistNewInstances}. After the new instances have been evaluated, configurations are evaluated on instances seen in the previous race. Elite configurations already have results for most of these previous instances and, therefore, do not need to be re-evaluated. Finally, after configurations have been evaluated on all these instances, the race continues by evaluating additional new instances. - The statistical tests can be performed at any moment during the race according to the setting of the options \parameter{firstTest} and \parameter{eachTest}. The elitist rule forbids discarding elite configurations, even if the show poor performance, until the last of the previous instances is seen in the race. + The statistical tests can be performed at any moment during the race according to the setting of the options \parameter{firstTest} and \parameter{eachTest}. The elitist rule forbids discarding elite configurations, even if the show poor performance, until the last of the previous instances is seen in the race. The \textbf{non-elitist} version of \irace can discard elite configurations at any point of the race, instances are not re-used from one race to the next, and new instances are sampled for each race. \defparameter{elitistNewInstances}{elitist-new-instances}{1}% Number of new instances added to each race before evaluating instances from previous races (only for elitist \irace). \begin{xwarningbox} - + If \parameter{deterministic} is \code{TRUE} then the number of \parameter{elitistNewInstances} will be reduced or set to \code{0} once all instances have been evaluated. \end{xwarningbox} \defparameter{elitistLimit}{elitist-limit}{2}% - Maximum number of statistical tests performed without successful elimination after all instances from the previous race have been evaluated. If the limit is reached, the current race is stopped. Only valid for elitist \irace. Use \code{0} to disable the limit. + Maximum number of statistical tests performed without successful elimination after all instances from the previous race have been evaluated. If the limit is reached, the current race is stopped. Only valid for elitist \irace. Use \code{0} to disable the limit. \end{description} \subsection[Internal irace options]{Internal \irace options} \begin{description} \defparameter{sampleInstances}{sample-instances}{1}% -Enable/disable the sampling of the training instances. If the option \parameter{sampleInstances} is disabled, the instances are used in the order provided in the \parameter{trainInstancesFile} or in the order they are read from the \parameter{trainInstancesDir} when\parameter{trainInstancesFile} is not provided. For more information about training instances see \autoref{sec:training}. + Enable/disable the sampling of the training instances. If the option \parameter{sampleInstances} is disabled, the instances are used in the order provided in the \parameter{trainInstancesFile} or in the order they are read from the \parameter{trainInstancesDir} when\parameter{trainInstancesFile} is not provided. For more information about training instances see \autoref{sec:training}. \defparameter{softRestart}{soft-restart}{1}% - Enable/disable the soft-restart strategy that avoids premature convergence of the probabilistic model. When a sampled configuration is \emph{similar} to its parent configuration, the probabilistic model of these configurations is soft restarted. The soft-restart mechanism is explained in the \irace paper~\citep{LopDubPerStuBir2016irace}. The similarity of categorical and ordinal parameters is given by the hamming distance, and the option \parameter{softRestartThreshold} defines the similarity of numerical parameters. + Enable/disable the soft-restart strategy that avoids premature convergence of the probabilistic model. When a sampled configuration is \emph{similar} to its parent configuration, the probabilistic model of these configurations is soft restarted. The soft-restart mechanism is explained in the \irace paper~\citep{LopDubPerStuBir2016irace}. The similarity of categorical and ordinal parameters is given by the hamming distance, and the option \parameter{softRestartThreshold} defines the similarity of numerical parameters. \defparameter{softRestartThreshold}{soft-restart-threshold}{1e-04}% -Soft restart threshold value for numerical parameters. + Soft restart threshold value for numerical parameters. \defparameter{nbIterations}{iterations}{0}% -Maximum number of iterations to be executed. Each iteration involves the generation of new configurations and the use of racing to select the best configurations. By default (with 0), \irace calculates a \emph{minimum} number of iterations as $\Niter = \lfloor 2 + \log_{2}\Nparam \rfloor$, where $\Nparam$ is the number of non-fixed parameters to be tuned. Setting this parameter may make \irace stop sooner than it should without using all the available budget. We recommend to use the default value. + Maximum number of iterations to be executed. Each iteration involves the generation of new configurations and the use of racing to select the best configurations. By default (with 0), \irace calculates a \emph{minimum} number of iterations as $\Niter = \lfloor 2 + \log_{2}\Nparam \rfloor$, where $\Nparam$ is the number of non-fixed parameters to be tuned. Setting this parameter may make \irace stop sooner than it should without using all the available budget. We recommend to use the default value. \defparameter{nbExperimentsPerIteration}{experiments-per-iteration}{0}% -Number of runs of the target algorithm per iteration. By default (when equal to 0), this value changes for each iteration and depends on the iteration index and the remaining budget. Further details are provided in the \irace paper~\citep{LopDubPerStuBir2016irace}. + Number of runs of the target algorithm per iteration. By default (when equal to 0), this value changes for each iteration and depends on the iteration index and the remaining budget. Further details are provided in the \irace paper~\citep{LopDubPerStuBir2016irace}. We recommend to use the default value. \defparameter{minNbSurvival}{min-survival}{0}% -Minimum number of configurations needed to continue the execution of each race (iteration). If the number of configurations alive in the race is not larger than this value, the current iteration will stop and a new iteration will start, even if there is budget left to continue the current race. By default (when equal to 0), the value is calculated automatically as $\lfloor 2 + \log_{2}\Nparam \rfloor$, where $\Nparam$ is the number of non-fixed parameters to be tuned. + Minimum number of configurations needed to continue the execution of each race (iteration). If the number of configurations alive in the race is not larger than this value, the current iteration will stop and a new iteration will start, even if there is budget left to continue the current race. By default (when equal to 0), the value is calculated automatically as $\lfloor 2 + \log_{2}\Nparam \rfloor$, where $\Nparam$ is the number of non-fixed parameters to be tuned. \defparameter{nbConfigurations}{num-configurations}{0}% - The number of configurations that will be raced at each iteration. By default (when equal to 0), this value changes for each iteration and depends on \parameter{nbExperimentsPerIteration}, the iteration index and \parameter{mu}. The precise details are given in the \irace paper~\citep{LopDubPerStuBir2016irace}. + The number of configurations that will be raced at each iteration. By default (when equal to 0), this value changes for each iteration and depends on \parameter{nbExperimentsPerIteration}, the iteration index and \parameter{mu}. The precise details are given in the \irace paper~\citep{LopDubPerStuBir2016irace}. We recommend to use the default value. \defparameter{mu}{mu}{5}% - Parameter used to define the number of configurations to be sampled and evaluated at each iteration. The number of configurations will be calculated such that there is enough budget in each race to evaluate all configurations on at least $\mu + \min(5,j)$ training instances, where $j$ is the index of the current iteration. The value of $\mu$ will be adjusted to never be lower than the value of \parameter{firstTest}. We recommend to use the default value and, if needed, adjust \parameter{firstTest}and \parameter{eachTest}, instead. + Parameter used to define the number of configurations to be sampled and evaluated at each iteration. The number of configurations will be calculated such that there is enough budget in each race to evaluate all configurations on at least $\mu + \min(5,j)$ training instances, where $j$ is the index of the current iteration. The value of $\mu$ will be adjusted to never be lower than the value of \parameter{firstTest}. We recommend to use the default value and, if needed, adjust \parameter{firstTest}and \parameter{eachTest}, instead. \end{description} \subsection[Target algorithm parameters]{Target algorithm parameters} \begin{description} \defparameter[p]{parameterFile}{parameter-file}{./parameters.txt}% -File that contains the description of the parameters of the target algorithm. See \autoref{sec:target parameters}. + File that contains the description of the parameters of the target algorithm. See \autoref{sec:target parameters}. \end{description} \subsection[Target algorithm execution]{Target algorithm execution} \begin{description} \defparameter{targetRunner}{target-runner}{./target-runner}% -Executable or \aR function that evaluates a configuration of the target algorithm on a particular instance. See \autoref{sec:runner} for details. + Executable or \aR function that evaluates a configuration of the target algorithm on a particular instance. See \autoref{sec:runner} for details. \defparameter{targetRunnerLauncher}{target-runner-launcher}{}% -Executable that will be used to launch the target runner, when \parameter{targetRunner} cannot be executed directly (e.g., a Python script in Windows). + Executable that will be used to launch the target runner, when \parameter{targetRunner} cannot be executed directly (e.g., a Python script in Windows). \defparameter{targetCmdline}{target-cmdline}{\{configurationID\} \{instanceID\} \{seed\} \{instance\} \{bound\} \{targetRunnerArgs\}}% -Command-line arguments provided to \parameter{targetRunner} (or \parameter{targetRunnerLauncher} if defined). The substrings \code{\{configurationID\}}, \code{\{instanceID\}}, \code{\{seed\}}, \code{\{instance\}}, and \code{\{bound\}} will be replaced by their corresponding values. The substring \code{\{targetRunnerArgs\}} will be replaced by the concatenation of the switch and value of all active parameters of the particular configuration being evaluated. The substring \code{\{targetRunner\}}, if present, will be replaced by the value of \parameter{targetRunner} (useful when using \parameter{targetRunnerLauncher}). Example: + Command-line arguments provided to \parameter{targetRunner} (or \parameter{targetRunnerLauncher} if defined). The substrings \code{\{configurationID\}}, \code{\{instanceID\}}, \code{\{seed\}}, \code{\{instance\}}, and \code{\{bound\}} will be replaced by their corresponding values. The substring \code{\{targetRunnerArgs\}} will be replaced by the concatenation of the switch and value of all active parameters of the particular configuration being evaluated. The substring \code{\{targetRunner\}}, if present, will be replaced by the value of \parameter{targetRunner} (useful when using \parameter{targetRunnerLauncher}). Example: <>= targetRunner="./real_target_runner.py" @@ -126,44 +129,44 @@ targetCmdLine="-m {targetRunner} {configurationID} {instanceID}\ \defparameter{targetRunnerRetries}{target-runner-retries}{0}% -Number of times to retry a call to \parameter{targetRunner} if the call failed. + Number of times to retry a call to \parameter{targetRunner} if the call failed. \defparameter{targetRunnerTimeout}{target-runner-timeout}{0}% -Timeout in seconds of any \parameter{targetRunner} call (only applies to \code{target-runner} executables not to R functions), ignored if 0. + Timeout in seconds of any \parameter{targetRunner} call (only applies to \code{target-runner} executables not to R functions), ignored if 0. \defparameter{targetRunnerData}{}{}% -Optional data passed to \parameter{targetRunner}. This is ignored by the default \parameter{targetRunner} function, but it may be used by custom \parameter{targetRunner} functions to pass persistent data around. + Optional data passed to \parameter{targetRunner}. This is ignored by the default \parameter{targetRunner} function, but it may be used by custom \parameter{targetRunner} functions to pass persistent data around. \defparameter{targetRunnerParallel}{}{}% Optional \aR function to provide custom parallelization of \parameter{targetRunner}. See \autoref{sec:parallel} for more information. \defparameter{targetEvaluator}{target-evaluator}{}% -Optional script or \aR function that returns a numerical value for an experiment after all configurations have been executed on a given instance using \parameter{targetRunner}. See \autoref{sec:evaluator} for details. + Optional script or \aR function that returns a numerical value for an experiment after all configurations have been executed on a given instance using \parameter{targetRunner}. See \autoref{sec:evaluator} for details. \defparameter{deterministic}{deterministic}{0}% - Enable/disable deterministic target algorithm mode. If the target algorithm is deterministic, configurations will be evaluated only once per instance. See \autoref{sec:training} for more information. + Enable/disable deterministic target algorithm mode. If the target algorithm is deterministic, configurations will be evaluated only once per instance. See \autoref{sec:training} for more information. \begin{xwarningbox} If the number of instances provided is less than the value specified for the option \parameter{firstTest}, no statistical test will be performed. \end{xwarningbox} \defparameter{parallel}{parallel}{0}% -Number of calls of the \parameter{targetRunner} to execute in parallel. Values 0 or 1 mean no parallelization. For more information on parallelization, see \autoref{sec:parallel}. + Number of calls of the \parameter{targetRunner} to execute in parallel. Values 0 or 1 mean no parallelization. For more information on parallelization, see \autoref{sec:parallel}. \defparameter{loadBalancing}{load-balancing}{1}% - Enable/disable load-balancing when executing experiments in parallel. Load-balancing makes better use of computing resources, but increases communication overhead. If this overhead is large, disabling load-balancing may be faster. See \autoref{sec:parallel}. + Enable/disable load-balancing when executing experiments in parallel. Load-balancing makes better use of computing resources, but increases communication overhead. If this overhead is large, disabling load-balancing may be faster. See \autoref{sec:parallel}. \defparameter{mpi}{mpi}{0}% - Enable/disable use of \pkg{Rmpi} to execute the \parameter{targetRunner} in parallel using MPI protocol. When \parameter{mpi} is enabled, the option \parameter{parallel} is the number of slave nodes. See \autoref{sec:parallel}. + Enable/disable use of \pkg{Rmpi} to execute the \parameter{targetRunner} in parallel using MPI protocol. When \parameter{mpi} is enabled, the option \parameter{parallel} is the number of slave nodes. See \autoref{sec:parallel}. \defparameter{batchmode}{batchmode}{0}% - Specify how irace waits for jobs to finish when \parameter{targetRunner} submits jobs to a batch cluster: \code{sge}, \code{pbs}, \code{torque}, \code{slurm} or \code{htcondor} (\parameter{targetRunner} must submit jobs to the cluster using. for example, \code{qsub}). See \autoref{sec:parallel}. + Specify how irace waits for jobs to finish when \parameter{targetRunner} submits jobs to a batch cluster: \code{sge}, \code{pbs}, \code{torque}, \code{slurm} or \code{htcondor} (\parameter{targetRunner} must submit jobs to the cluster using. for example, \code{qsub}). See \autoref{sec:parallel}. \end{description} \subsection[Initial configurations]{Initial configurations} \begin{description} \defparameter{configurationsFile}{configurations-file}{}% -File containing a table of initial configurations. If empty or \code{NULL}, \irace will not use initial configurations. See \autoref{sec:initial}. + File containing a table of initial configurations. If empty or \code{NULL}, \irace will not use initial configurations. See \autoref{sec:initial}. \begin{xwarningbox} The provided configurations must not violate the constraints described in \parameter{parameterFile} and \parameter{forbiddenFile}. \end{xwarningbox} @@ -173,47 +176,50 @@ The provided configurations must not violate the constraints described in \param \subsection[Training instances]{Training instances} \begin{description} \defparameter{trainInstancesDir}{train-instances-dir}{}% -Directory where training instances are located; either absolute path or relative to current directory. See \autoref{sec:training}. + Directory where training instances are located; either absolute path or relative to current directory. See \autoref{sec:training}. \defparameter{trainInstancesFile}{train-instances-file}{}% -File that contains a list of instances and optionally additional parameters for them. See \autoref{sec:training}. + File that contains a list of instances and optionally additional parameters for them. See \autoref{sec:training}. \begin{xwarningbox} The list of instances in \parameter{trainInstancesFile} is interpreted as file-system paths relative to \parameter{trainInstancesDir}. When using an absolute path or instances that are not files, set \code{trainInstancesDir=""}. \end{xwarningbox} + \defparameter{blockSize}{block-size}{1}% + Number of training instances, that make up a 'block' in \parameter{trainInstancesFile}. Elimination of configurations will only be performed after evaluating a complete block and never in the middle of a block. Each block typically contains one instance from each instance class (type or family) and the block size is the number of classes. The value of \parameter{blockSize} will multiply \parameter{firstTest}, \parameter{eachTest} and \parameter{elitistNewInstances}. + \end{description} \subsection[Tuning budget]{Tuning budget} \begin{description} \defparameter{maxExperiments}{max-experiments}{0}% -The maximum number of runs (invocations of \parameter{targetRunner}) that will be performed. It determines the maximum budget of experiments for the tuning. See \autoref{sec:budget}. + The maximum number of runs (invocations of \parameter{targetRunner}) that will be performed. It determines the maximum budget of experiments for the tuning. See \autoref{sec:budget}. \defparameter{minExperiments}{min-experiments}{}% -The minimum number of runs (invocations of \parameter{targetRunner}) that will be performed. If this option is set, then \parameter{maxExperiments} is ignored and the actual budget will depend on the number of parameters and \parameter{minSurvival}, but it will not be smaller than this value. See \autoref{sec:budget}. + The minimum number of runs (invocations of \parameter{targetRunner}) that will be performed. If this option is set, then \parameter{maxExperiments} is ignored and the actual budget will depend on the number of parameters and \parameter{minSurvival}, but it will not be smaller than this value. See \autoref{sec:budget}. \defparameter{maxTime}{max-time}{0}% -The maximum total time for the runs of \parameter{targetRunner} that will be performed. The mean execution time of each run is estimated in order to calculate the maximum number of experiments (see option \parameter{budgetEstimation}). + The maximum total time for the runs of \parameter{targetRunner} that will be performed. The mean execution time of each run is estimated in order to calculate the maximum number of experiments (see option \parameter{budgetEstimation}). When \parameter{maxTime} is positive, then \parameter{targetRunner} \textbf{must} return the execution time as its second output. This value and the one returned by \parameter{targetRunner} must use the same units (seconds, minutes, iterations, evaluations, \ldots). See \autoref{sec:budget}. \defparameter{budgetEstimation}{budget-estimation}{0.05}% -Fraction (smaller than 1) of the budget used to estimate the mean execution time of a configuration. Only used when \parameter{maxTime} $> 0$. See \autoref{sec:budget}. + Fraction (smaller than 1) of the budget used to estimate the mean execution time of a configuration. Only used when \parameter{maxTime} $> 0$. See \autoref{sec:budget}. \defparameter{minMeasurableTime}{min-measurable-time}{0.01}% -Minimum time unit that is still (significantly) measureable. + Minimum time unit that is still (significantly) measureable. \end{description} \subsection[Statistical test]{Statistical test} \begin{description} \defparameter{testType}{test-type}{}% -Specifies the statistical test used for elimination: + Specifies the statistical test used for elimination: \begin{itemize} \item[] \code{F-test} (Friedman test) \item[] \code{t-test} (pairwise t-tests with no correction) \item[] \code{t-test-bonferroni} (t-test with Bonferroni's correction for multiple comparisons) \item[] \code{t-test-holm} (t-test with Holm's correction for multiple comparisons). \end{itemize} - We recommend to not use corrections for multiple comparisons because the test typically becomes too strict and the search stagnates. + We recommend to not use corrections for multiple comparisons because the test typically becomes too strict and the search stagnates. See \autoref{sec:stat test} for details about choosing the statistical test most appropriate for your scenario. \begin{xwarningbox} The default setting of \parameter{testType} is \code{F-test} unless the \parameter{capping} option is enabled in which case, the default setting is \code{t-test}. @@ -226,17 +232,20 @@ The value of \parameter{firstTest} must be a multiple of \parameter{eachTest}. \end{xwarningbox} \defparameter{eachTest}{each-test}{1}% -Specifies how many instances are evaluated between elimination tests. + Specifies how many instances are evaluated between elimination tests. \defparameter{confidence}{confidence}{0.95}% -Confidence level for the elimination test. + Confidence level for the elimination test. \end{description} \subsection[Adaptive capping]{Adaptive capping} \begin{description} \defparameter{capping}{capping}{}% -Enable the use of adaptive capping. Capping is enabled by default if \parameter{elitist} is active, $\parameter{maxTime} > 0$ and $\parameter{boundMax} > 0$. When using this option, \irace provides an execution bound to each target algorithm execution (See \autoref{sec:runner}). For more details about this option See \autoref{sec:capping}. + Enable the use of adaptive capping. Capping is enabled by default if \parameter{elitist} is active, $\parameter{maxTime} > 0$ and $\parameter{boundMax} > 0$. When using this option, \irace provides an execution bound to each target algorithm execution (See \autoref{sec:runner}). For more details about this option See \autoref{sec:capping}. + + \defparameter{cappingAfterFirstTest}{capping-after-first-test}{0}% + If set to 1, elimination due to capping only happens after \parameter{firstTest} instances are seen. \defparameter{cappingType}{capping-type}{median}% Specifies the measure used to define the execution bound: @@ -248,30 +257,30 @@ Enable the use of adaptive capping. Capping is enabled by default if \parameter{ \end{itemize} \defparameter{boundType}{bound-type}{candidate}% -Specifies how to calculate the performance of elite configurations for the execution bound: + Specifies how to calculate the performance of elite configurations for the execution bound: \begin{itemize} \item[] \code{candidate} (performance of candidates is aggregated across the instances already executed) \item[] \code{instance} (performance of candidates on each instance). \end{itemize} \defparameter{boundMax}{bound-max}{0}% -Maximum execution bound for \code{targetRunner}. It must be specified when capping is enabled. + Maximum execution bound for \parameter{targetRunner}. It must be specified when capping is enabled. \defparameter{boundDigits}{bound-digits}{0}% -Precision used for calculating the execution time. It must be specified when capping is enabled. + Precision used for calculating the execution time. It must be specified when capping is enabled. \defparameter{boundPar}{bound-par}{1}% -Penalty used for PARX. This value is used to penalize timed out executions, see \autoref{sec:capping}. + Penalty used for PARX. This value is used to penalize timed out executions, see \autoref{sec:capping}. \defparameter{boundAsTimeout}{bound-as-timeout}{1}% -Replace the configuration cost of bounded executions with \parameter{boundMax}. See \autoref{sec:capping}. + Replace the configuration cost of bounded executions with \parameter{boundMax}. See \autoref{sec:capping}. \end{description} \subsection[Recovery]{Recovery} \begin{description} \defparameter{recoveryFile}{recovery-file}{}% -Previously saved \irace log file that should be used to recover the execution of \irace; either absolute path or relative to the current directory. If empty or \code{NULL}, recovery is not performed. + Previously saved \irace log file that should be used to recover the execution of \irace; either absolute path or relative to the current directory. If empty or \code{NULL}, recovery is not performed. For more details about recovery, see \autoref{sec:recovery}. \end{description} @@ -279,18 +288,18 @@ For more details about recovery, see \autoref{sec:recovery}. \subsection[Testing]{Testing} \begin{description} \defparameter{-{}-only-test}{only-test}{}% -Run the configurations contained in the file provided as argument on the test instances. See \autoref{sec:testing}. + Run the configurations contained in the file provided as argument on the test instances. See \autoref{sec:testing}. \defparameter{testInstancesDir}{test-instances-dir}{}% -Directory where testing instances are located, either absolute or relative to the current directory. + Directory where testing instances are located, either absolute or relative to current directory. \defparameter{testInstancesFile}{test-instances-file}{}% -File containing a list of test instances and, optionally, additional parameters for them. + File containing a list of test instances and optionally additional parameters for them. \defparameter{testNbElites}{test-num-elites}{1}% - Number of elite configurations returned by irace that will be tested if test instances are provided. For more information about the testing, see \autoref{sec:testing}. + Number of elite configurations returned by irace that will be tested if test instances are provided. For more information about the testing, see \autoref{sec:testing}. \defparameter{testIterationElites}{test-iteration-elites}{0}% -Enable/disable testing the elite configurations found at each iteration. + Enable/disable testing the elite configurations found at each iteration. \end{description}