diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a85d1b5eba..9d866e3a8c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,21 +1,21 @@ We'd love you to contribute back to EasyBuild, and here's how you can do it: the branch - hack - pull request cycle. # License -Contributions can be made under the MIT or +Contributions can be made under the MIT or BSD licenses (in the three-clause and two-clause forms, though not the original four-clause form). Or alteratively the contributor must agree with following contributor agreement: ## Contributor Agreement. -In this case the contributor must agree that Ghent University shall have the irrevocable and perpetual right to make and +In this case the contributor must agree that Ghent University shall have the irrevocable and perpetual right to make and distribute copies of any Contribution, as well as to create and distribute collective works and derivative works of -any Contribution, under the Initial License or under any other open source license. +any Contribution, under the Initial License or under any other open source license. (as defined by The Open Source Initiative (OSI) http://opensource.org/). -Contributor shall identify each Contribution by placing the following notice in its source code adjacent to -Contributor's valid copyright notice: "Licensed to Ghent University under a Contributor Agreement." +Contributor shall identify each Contribution by placing the following notice in its source code adjacent to +Contributor's valid copyright notice: "Licensed to Ghent University under a Contributor Agreement." The currently acceptable license is GPLv2 or any other GPLv2 compatible license. -Ghent University understands and agrees that Contributor retains copyright in its Contributions. +Ghent University understands and agrees that Contributor retains copyright in its Contributions. Nothing in this Contributor Agreement shall be interpreted to prohibit Contributor from licensing its Contributions under different terms from the Initial License or this Contributor Agreement. @@ -31,7 +31,7 @@ You should also register an SSH public key, so you can easily clone, push to and ### Clone the easybuild-framework repository -Clone your fork of the easybuild-framework repository to your favorite workstation. +Clone your fork of the easybuild-framework repository to your favorite workstation. ```bash git clone git@github.com:YOUR\_GITHUB\_LOGIN/easybuild-framework.git @@ -51,7 +51,7 @@ git pull github_hpcugent develop ### Keep develop up-to-date -The _develop_ branch hosts the latest bleeding-edge version of easybuild-framework, and is merged into _master_ regularly (after thorough testing). +The _develop_ branch hosts the latest bleeding-edge version of easybuild-framework, and is merged into _master_ regularly (after thorough testing). Make sure you update it every time you create a feature branch (see below): @@ -82,7 +82,7 @@ git checkout Make sure to always base your features branches on _develop_, not on _master_! - + ## Hack @@ -108,7 +108,7 @@ When you've finished the implementation of a particular contribution, here's how ### Push your branch Push your branch to your easybuild-framework repository on GitHub: - + ```bash git push origin ``` @@ -127,7 +127,7 @@ Issue a pull request for your branch into the mair easybuild-framework repositor ### Issue pull request for existing ticket (from command line) If you're contributing code to an existing issue you can also convert the issue to a pull request by running -``` +``` GITHUBUSER=your_username && PASSWD=your_password && BRANCH=branch_name && ISSUE=issue_number && \ curl --user "$GITHUBUSER:$PASSWD" --request POST \ --data "{\"issue\": \"$ISSUE\", \"head\": \"$GITHUBUSER:$BRANCH\", \"base\": \"develop\"}" \ @@ -138,7 +138,7 @@ You might also want to look into [hub](https://github.com/defunkt/hub) for more ### Review process -A member of the EasyBuild team will then review your pull request, paying attention to what you're contributing, how you implemented it and [code style](Code style). +A member of the EasyBuild team will then review your pull request, paying attention to what you're contributing, how you implemented it and [code style](http://easybuild.readthedocs.org/en/latest/Code_style.html). Most likely, some remarks will be made on your pull request. Note that this is nothing personal, we're just trying to keep the EasyBuild codebase as high quality as possible. Even when an EasyBuild team member makes changes, the same public review process is followed. diff --git a/README.rst b/README.rst index 2cd7efc455..5b375ff615 100644 --- a/README.rst +++ b/README.rst @@ -1,54 +1,57 @@ -Build status - *master branch (Python 2.4, Python 2.6, Python 2.7)* - -.. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python24/badge/icon - :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python24/ -.. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master/badge/icon - :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master/ -.. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python27/badge/icon - :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python27/ - -Build status - *develop branch (Python 2.4, Python 2.6, Python 2.7)* - -.. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python24/badge/icon - :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python24/ -.. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop/badge/icon - :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop/ -.. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python27/badge/icon - :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python27/ - -EasyBuild: building software with ease --------------------------------------- - -The easybuild-framework package is the basis for EasyBuild -(http://hpcugent.github.com/easybuild), a software build and -installation framework written in Python that allows you to install -software in a structured, repeatable and robust way. - -This package contains the EasyBuild framework that supports the -implementation and use of so-called easyblocks, that implement the -software install procedure for a particular (group of) software +.. image:: http://hpcugent.github.io/easybuild/images/easybuild_logo_small.png + :align: center + +`EasyBuild `_ is a software build +and installation framework that allows you to manage (scientific) software +on High Performance Computing (HPC) systems in an efficient way. + +The **easybuild-framework** package is the core of EasyBuild. It +supports the implementation and use of so-called easyblocks which +implement the software install procedure for a particular (group of) software package(s). -The code of the easybuild-framework package is hosted on GitHub, along +The EasyBuild documentation is available at http://easybuild.readthedocs.org/. + +The EasyBuild framework source code is hosted on GitHub, along with an issue tracker for bug reports and feature requests, see http://github.com/hpcugent/easybuild-framework. -The EasyBuild documentation is available on the GitHub wiki of the -easybuild meta-package, see -http://github.com/hpcugent/easybuild/wiki/Home. - -Related packages: -- easybuild-easyblocks -(http://pypi.python.org/pypi/easybuild-easyblocks): a collection of -easyblocks that implement support for building and installing (groups -of) software packages. - -- easybuild-easyconfigs -(http://pypi.python.org/pypi/easybuild-easyconfigs): a collection of -example easyconfig files that specify which software to build, and using -which build options; these easyconfigs will be well tested with the -latest compatible versions of the easybuild-framework and -easybuild-easyblocks packages. - -The code in the vsc directory originally comes from VSC-tools -(https://github.com/hpcugent/VSC-tools). +Related Python packages: + +* **easybuild-easyblocks** + + * a collection of easyblocks that implement support for building and installing (groups of) software packages + * GitHub repository: http://github.com/hpcugent/easybuild-easyblocks + * package on PyPi: https://pypi.python.org/pypi/easybuild-easyblocks + +* **easybuild-easyconfigs** + + * a collection of example easyconfig files that specify which software to build, + and using which build options; these easyconfigs will be well tested + with the latest compatible versions of the easybuild-framework and easybuild-easyblocks packages + * GitHub repository: http://github.com/hpcugent/easybuild-easyconfigs + * PyPi: https://pypi.python.org/pypi/easybuild-easyconfigs + +The code in the ``vsc`` directory originally comes from the *vsc-base* package +(https://github.com/hpcugent/vsc-base). + + +*Build status overview:* + +* **master** branch *(Python 2.4, Python 2.6, Python 2.7)* + + .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python24/badge/icon + :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python24/ + .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master/badge/icon + :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master/ + .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python27/badge/icon + :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python27/ + +* **develop** branch *(Python 2.4, Python 2.6, Python 2.7)* + + .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python24/badge/icon + :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python24/ + .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop/badge/icon + :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop/ + .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python27/badge/icon + :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python27/ diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 3b7a82b7fd..6ce9d9ce17 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,6 +1,304 @@ This file contains a description of the major changes to the easybuild-framework EasyBuild package. For more detailed information, please see the git log. +These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html. + +v2.3.0 (September 2nd 2015) +--------------------------- + +feature + bugfix release +- requires vsc-base v2.2.4 or more recent (#1343) + - required for mk_rst_table function in vsc.utils.docs +- various other enhancements, including: + - add support for generating documentation for (generic) easyblocks in .rst format (#1317) + - preserve comments in easyconfig file in EasyConfig.dump() method (#1327) + - add --cleanup-tmpdir option (#1365) + - enables to preserve the used temporary directory via --disable-cleanup-tmpdir + - enhance EasyConfig.dump() to reformat dumped easyconfig according to style guidelines (#1345) + - add support for extracting .iso files using 7z (p7zip) (#1375) +- various bug fixes, including: + - correctly deal with special characters in template strings in EasyConfig.dump() method (#1323) + - rework easybuild.tools.module_generator module to avoid keeping state w.r.t. fake modules (#1348) + - fix dumping of hidden deps (#1354) + - fix use of --job with hidden dependencies: include --hidden in submitted job script when needed (#1356) + - fix ActiveMNS.det_full_module_name() for external modules (#1360) + - fix EasyConfig.all_dependencies definition, fix tracking of job dependencies (#1359, #1361) + - fix 'ModulesTool.exist' for hidden Lua module files (#1364) + - only call EasyBlock.sanity_check_step for non-extensions (#1366) + - this results in significant speedup when installing easyconfigs with lots of extensions, but also + results in checking the default sanity check paths if none were defined for extensions installed as a module + - fix using module naming schemes that were included via --include-module-naming-schemes (#1370) + +v2.2.0 (July 15th 2015) +----------------------- + +feature + bugfix release +- add support for using GC3Pie as a backend for --job (#1008) + - see also http://easybuild.readthedocs.org/en/latest/Submitting_jobs.html +- add support for --include-* configuration options to include additional easyblocks, toolchains, etc. (#1301) + - see http://easybuild.readthedocs.org/en/latest/Including_additional_Python_modules.html +- add (experimental) support for packaging installed software using FPM (#1224) + - see http://easybuild.readthedocs.org/en/latest/Packaging_support.html +- various other enhancements, including: + - use https for PyPI URL templates (#1286) + - add GNU toolchain definition (#1287) + - make bootstrap script more robust (#1289, #1325): + - exclude 'easyblocks' pkg from sys.path to avoid that setup.py for easybuild-easyblocks picks up wrong version + - undefine $EASYBUILD_BOOTSTRAP* environment variables, since they do not correspond with known config options + - improve error reporting/robustness in fix_broken_easyconfigs.py script (#1290) + - reset keep toolchain component class 'constants' every time (#1294) + - make --strict also a build option (#1295) + - fix purging of loaded modules in unit tests' setup method (#1297) + - promote MigrateFromEBToHMNS to a 'production' MNS (#1302) + - add support for --read-only-installdir and --group-writable-installdir configuration options (#1304) + - add support for *not* expanding relative paths in prepend_paths (#1310) + - enhance EasyConfig.dump() method to use easyconfig templates where possible (#1314, #1319, #1320, #1321) +- various bug fixes, including: + - fix issue with cleaning up (no) logfile if --logtostdout/-l is used (#1298) + - stop making ModulesTool class a singleton since it causes problems when multilple toolchains are in play (#1299) + - don't modify values of 'paths' list passed as argument to prepend_paths in ModuleGenerator (#1300) + - fix issue with EasyConfig.dump + cleanup (#1308, #1311) + - reenable (and fix) accidentally disabled test (#1316) + +v2.1.1 (May 18th 2015) +---------------------- + +bugfix release +- fix issue with missing load statements when --module-only is used, don't skip ready/prepare steps (#1276) +- enhance --search: only consider actual filename (not entire path), use regex syntax (#1281) +- various other bug fixes, including: + - fix generate_software_list.py script w.r.t. dependencies marked as external modules (#1273) + - only use $LMOD_CMD value if lmod binary can't be found in $PATH (#1275) + - fix location of module_only build option w.r.t. default value (#1277) + - fix combined use of --hide-deps and hiddendependencies (#1280) + - remove log handlers that were added during tests, to ensure effective cleanup of log files (#1282) + - this makes the unit test suite run ~3x faster! + - define $CRAYPE_LINK_TYPE if 'dynamic' toolchain option is enabled for Cray compiler wrappers (#1283) + +v2.1.0 (April 30th 2015) +------------------------ + +feature + bugfix release +- requires vsc-base v2.2.0 or more recent + - added support for LoggedException + - added support for add_flex action in GeneralOption + - added support to GeneralOption to act on unknown configuration environment variables +- add support for only (re)generating module files: --module-only (#1018) + - module naming scheme API is enhanced to include det_install_subdir method + - see http://easybuild.readthedocs.org/en/latest/Partial_installations.html#module-only +- add support for generating module files in Lua syntax (note: requires Lmod as modules tool) (#1060, #1255, #1256, #1270) + - see --module-syntax configuration option and http://easybuild.readthedocs.org/en/latest/Configuration.html#module-syntax +- deprecate log.error method in favor of raising EasyBuildError exception (#1218) + - see http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html#depr-error-reporting +- add support for using external modules as dependencies, and to provide metadata for external modules (#1230, #1265, #1267) + - see http://easybuild.readthedocs.org/en/latest/Using_external_modules.html +- add experimental support for Cray toolchains on top of PrgEnv modules: CrayGNU, CrayIntel, CrayCCE (#1234, #1268) + - see https://github.com/hpcugent/easybuild/wiki/EasyBuild-on-Cray for more information +- various other enhancements, including: + - clear list of checksums when using --try-software-version (#1169) + - sort the results of searching for files (e.g., --search output) (#1214) + - enhance test w.r.t. use of templates in cfgfile (#1217) + - define %(DEFAULT_REPOSITORYPATH)s template for cfgfiles (see eb --avail-cfgfile-constants) (#1220) + - also reset $LD_PRELOAD when running module commands, in case module defined $LD_PRELOAD (#1222) + - move location of 'module use' statements in generated module file (*after* 'module load' statements) (#1232) + - add support for --show-default-configfiles (#1240) + - see http://easybuild.readthedocs.org/en/latest/Configuration.html#default-configuration-files + - report error on missing configuration files, rather than ignoring them (#1240) + - see http://easybuild.readthedocs.org/en/latest/Configuration.html#configuration-env-vars + - clean up commit message used in easyconfig git repository (#1248) + - add --hide-deps configuration option to specify names of software that must be installed as hidden modules (#1250) + - see http://easybuild.readthedocs.org/en/latest/Manipulating_dependencies.html#hide-deps + - add support for appending/prepending to --robot-paths to avoid overwriting default robot search path (#1252) + - see also http://easybuild.readthedocs.org/en/latest/Using_the_EasyBuild_command_line.html#robot-search-path-prepend-append + - enable detection of use of unknown $EASYBUILD-prefixed environment variables (#1253) + - add --installpath-modules and --installpath-software configuration options (#1258) + - see http://easybuild.readthedocs.org/en/latest/Configuration.html#installpath + - use dedicated subdirectory in temporary directory for each test to ensure better cleanup (#1260) + - get rid of $PROFILEREAD hack when running commands, not needed anymore (#1264) +- various bug fixes, including: + - make bootstrap script robust against having vsc-base already available in Python search path (#1212, #1215) + - set default value for unpack_options easyconfig parameter to '', so self.cfg.update works on it (#1229) + - also copy rotated log files (#1238) + - fix parsing of --download-timeout value (#1242) + - make test_XDG_CONFIG_env_vars unit test robust against existing user config file in default location (#1259) + - fix minor robustness issues w.r.t. $XDG_CONFIG* and $PYTHONPATH in unit tests (#1262) + - fix issue with handling empty toolchain variables (#1263) + +v2.0.0 (March 6th 2015) +----------------------- + +feature + bugfix release +- requires vsc-base v2.0.3 or more recent + - avoid deprecation warnings w.r.t. use of 'message' attribute (hpcugent/vsc-base#155) + - fix typo in log message rendering --ignoreconfigfiles unusable (hpcugent/vsc-base#158) +- removed functionality that was deprecated for EasyBuild version 2.0 (#1143) + - see http://easybuild.readthedocs.org/en/latest/Removed-functionality.html + - the fix_broken_easyconfigs.py script can be used to update easyconfig files suffering from this (#1151, #1206, #1207) + - for more information about this script, see http://easybuild.readthedocs.org/en/latest/Useful-scripts.html#fix-broken-easyconfigs-py +- stop including a crippled copy of vsc-base, include vsc-base as a proper dependency instead (#1160, #1194) + - vsc-base is automatically installed as a dependency for easybuild-framework, if a Python installation tool is used + - see http://easybuild.readthedocs.org/en/latest/Installation.html#required-python-packages +- various other enhancements, including: + - add support for Linux/POWER systems (#1044) + - major cleanup in tools/systemtools.py + significantly enhanced tests (#1044) + - add support for 'eb -a rst', list available easyconfig parameters in ReST format (#1131) + - add support for specifying one or more easyconfigs in combination with --from-pr (#1132) + - see http://easybuild.readthedocs.org/en/latest/Integration_with_GitHub.html#using-easyconfigs-from-pull-requests-via-from-pr + - define __contains__ in EasyConfig class (#1155) + - restore support for downloading over a proxy (#1158) + - i.e., use urllib2 rather than urllib + - this involved sacrificing the download progress report (which was only visible in the log file) + - let mpi_family return None if MPI is not supported by a toolchain (#1164) + - include support for specifying system-level configuration files for EasyBuild via $XDG_CONFIG_DIRS (#1166) + - see http://easybuild.readthedocs.org/en/latest/Configuration.html#default-configuration-files + - make unit tests more robust (#1167, #1196) + - see http://easybuild.readthedocs.org/en/latest/Unit-tests.html + - add hierarchical module naming scheme categorizing modules by 'moduleclass' (#1176) + - enhance bootstrap script to allow bootstrapping using supplied tarballs (#1184) + - see http://easybuild.readthedocs.org/en/latest/Installation.html#advanced-bootstrapping-options + - disable updating of Lmod user cache by default, add configuration option --update-modules-tool-cache (#1185) + - for now, only the Lmod user cache can be updated using --update-modules-tool-cache + - use available which() function, rather than running 'which' via run_cmd (#1192) + - fix install-EasyBuild-develop.sh script w.r.t. vsc-base dependency (#1193) + - also consider robot search path when looking for specified easyconfigs (#1201) + - see http://easybuild.readthedocs.org/en/latest/Using_the_EasyBuild_command_line.html#specifying-easyconfigs +- various bug fixes, including: + - stop triggering deprecated/no longer support functionality in unit tests (#1126) + - fix from_pr test by including dummy easyblocks for HPL and ScaLAPACK (#1133) + - escape use of '%' in string with command line options with --job (#1135) + - fix handling specified patch level 0 (+ enhance tests for fetch_patches method) (#1139) + - fix formatting issues in generated easyconfig file obtained via --try-X (#1144) + - use log.error in tools/toolchain/toolchain.py where applicable (#1145) + - stop hardcoding /tmp in mpi_cmd_for function (#1146, #1200) + - correctly determine variable name for $EBEXTLIST when generating module file (#1156) + - do not ignore exit code of failing postinstall commands (#1157) + - fix rare case in which used easyconfig and copied easyconfig are the same (#1159) + - always filter hidden deps from list of dependencies (#1161) + - fix implementation of path_matches function in tools/filetools.py (#1163) + - make sure plain text keyring is used by unit tests (#1165) + - suppress creation of module symlinks for HierarchicalMNS (#1173) + - sort all lists obtained via glob.glob, since they are in arbitrary order (#1187) + - stop modifying $MODULEPATH directly in setUp/tearDown of toolchain tests (#1191) + +v1.16.2 (March 6th 2015) +------------------------ + +(no changes compared to v1.16.1, simple version bump to stay in sync with easybuild-easyblocks) + +v1.16.1 (December 19th 2014) +---------------------------- + +bugfix release +- fix functionality that is broken with --deprecated=2.0 or with $EASYBUILD_DEPRECATED=2.0 + - don't include easyconfig parameters for ConfigureMake in eb -a, since fallback is deprecated (#1123) + - correctly check software_license value type (#1124) + - fix generate_software_list.py script w.r.t. deprecated fallback to ConfigureMake (#1127) +- other bug fixes + - fix logging issues in tests, sync with vsc-base v2.0.0 (#1120) + +v1.16.0 (December 18th 2014) +---------------------------- + +feature + bugfix release +- deprecate automagic fallback to ConfigureMake easyblock (#1113) + - easyconfigs should specify easyblock = 'ConfigureMake' instead of relying on the fallback mechanism + - note: automagic fallback to ConfigureMake easyblock will be dropped in EasyBuild v2.0 + - see also http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html#configuremake-fallback +- stop triggering deprecated functionality, to enable use of --deprecated=2.0 (#1107, #1115, #1119) + - see http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html#configuremake-fallback for more information +- various other enhancements, including: + - add script to clean up gists created via --upload-test-report (#958) + - also use -xHost when using Intel compilers on AMD systems (as opposed to -msse3) (#960) + - add Python version check in eb command (#1046) + - take versionprefix into account in HierarchicalMNS module naming scheme (#1058) + - clean up and refactor main.py, move functionality to other modules (#1059, #1064, #1075, #1087) + - add check in download_file function for HTTP return code + show download progress report (#1066, #1090) + - include info log message with name and location of used easyblock (#1069) + - add toolchains definitions for gpsmpi, gpsolf, impich, intel-para, ipsmpi toolchains (#1072, #1073) + - support for Parastation MPI based toolchains + - enforce that hiddendependencies is a subset of dependencies (#1078) + - this is done to avoid that site-specific policies w.r.t. hidden modules slip into contributed easyconfigs + - enable use of --show_hidden for avail subcommand with recent Lmod versions (#1081) + - add --robot-paths configure option (#1080, #1093, #1095, #1114) + - support use of %(DEFAULT_ROBOT_PATHS)s template in EasyBuild configuration files (#1100) + - see also http://easybuild.readthedocs.org/en/latest/Using_the_EasyBuild_command_line.html#controlling-the-robot-search-path + - use -xHost rather than -xHOST, to match Intel documentation (#1084) + - update and cleanup README file (#1085) + - deprecate self.moduleGenerator in favor of self.module_generator in EasyBlock (#1088) + - also support MPICH MPI family in mpi_cmd_for function (#1098) + - update documentation references to point to http://easybuild.readthedocs.org (#1102) + - check for OS dependencies with both rpm and dpkg (if available) (#1111) +- various bug fixes, including: + - fix picking required software version specified by --software-version and clean up tweak.py (#1062, #1063) + - escape $ characters in module load message specified via modloadmsg easyconfig parameter) (#1068) + - take available hidden modules into account in dependency resolution (#1065) + - fix hard crash when using patch files with an empty list of sources (#1070) + - fix Intel MKL BLACS library being used for MPICH/MPICH2-based toolchains (#1072) + - fix regular expression in fetch_parameter_from_easyconfig_file function (#1096) + - don’t hardcode queue names when submitting a job (#1106) + - fix affiliation/mail address for Fotis in headers (#1105) + - filter out /dev/null entries in patch file in det_patched_files function (#1108) + - fix gmpolf toolchain definition, to have gmpich as MPI components (instead of gmpich2) (#1101) + - ‘MPICH’ refers to MPICH v3.x, while MPICH2 refers to MPICH(2) v2.x (MPICH v1.x is ancient/obsolete) + - note: this requires to reinstall the gmpolf module, using the updated easyconfig from easybuild-easyconfigs#1217 + +v1.15.2 (October 7th 2014) +-------------------------- + +bugfix release +- fix $MODULEPATH extensions for Clang/CUDA, to make goolfc/cgoolf compatible with HierarchicalMNS (#1050) +- include versionsuffix in module subdirectory with HierarchicalMNS (#1050, #1055) +- fix unit tests which were broken with bash patched for ShellShock bug (#1051) +- add definition of gimpi toolchain, required to make gimkl toolchain compatible with HierarchicalMNS (#1052) +- don't override COMPILER_MODULE_NAME obtained from ClangGCC in Clang-based toolchains (#1053) +- fix wrong code in path_to_top_of_module_tree function (#1054) + - because of this, load statements for compilers were potentially included in higher-level modules under HierarchicalMNS + +v1.15.1 (September 23rd 2014) +----------------------------- + +bugfix release +- take into account that multiple modules may be extending $MODULEPATH with the same path, + when determining path to top of module tree (see #1047) + - this bug caused a load statement for either icc or ifort to be included in higher-level + modules installed with an Intel-based compiler toolchain, under the HierarchicalMNS module naming scheme +- make HierarchicalMNS module naming scheme compatible with cgoolf and goolfc toolchain (#1049) +- add definition of iompi (sub)toolchain to make iomkl toolchain compatible with HierarchicalMNS (#1049) + +v1.15.0 (September 12th 2014) +----------------------------- + +feature + bugfix release +- various other enhancements, including: + - fetch extension sources in fetch_step to enhance --stop=fetch (#978) + - add iimpi toolchain definition (#993) + - prepend robot path with download location of files when --from-pr is used (#995) + - add support for excluding module path extensions from generated modules (#1003) + - see 'include_modpath_extensions' easyconfig parameter + - add support for installing hidden modules and using them as dependencies (#1009, #1021, #1023) + - see --hidden and 'hiddendependencies' easyconfig parameter + - stop relying on 'conflict' statement in module files to determine software name of toolchain components (#1017, #1037) + - instead, the 'is_short_modname_for' method defined by the module naming scheme implementation is queried + - improve error message generated for a missing easyconfig file (#1019) + - include path where tweaked easyconfigs are placed in robot path (#1032) + - indicate forced builds in --dry-run output (#1034) + - fix interaction between --force and --try-toolchain --robot (#1035) + - add --software option, disable recursion for --try-software(-X) (#1036) +- various bug fixes, including: + - fix HierarchicalMNS crashing when MPI library is installed with a dummy toolchain (#986) + - fix list of FFTW wrapper libraries for Intel MKL (#987) + - fix stability of unit tests (#988, #1027, #1033) + - make sure $SCALAPACK_INC_DIR (and $SCALAPACK_LIB_DIR) are defined when using imkl (#990) + - fix error message on missing FFTW wrapper libs (#992) + - fix duplicate toolchain elements in --list-toolchains output (#993) + - filter out load statements that extend the $MODULEPATH to make the module being installed available (#1016) + - fix conflict specification included in module files (#1017) + - avoid --from-pr crashing hard unless --robot is used (#1022) + - properly quote GCC version string in archived easyconfig (#1028) + - fix issue with --repositorypath not honoring --prefix (#1031) + - sync with latest vsc-base version to fix log order (#1039) + - increase # commits per page for --from-pr (#1040) + v1.14.0 (July 9th 2014) ----------------------- diff --git a/easybuild/__init__.py b/easybuild/__init__.py index 2b217b34bc..6bdfdf13a4 100644 --- a/easybuild/__init__.py +++ b/easybuild/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2014 Ghent University +# Copyright 2011-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easybuild_config.py b/easybuild/easybuild_config.py deleted file mode 100644 index 0d90c875a1..0000000000 --- a/easybuild/easybuild_config.py +++ /dev/null @@ -1,96 +0,0 @@ -# # -# Copyright 2009-2014 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# the Hercules foundation (http://www.herculesstichting.be/in_English) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# http://github.com/hpcugent/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -# # -""" -EasyBuild configuration file. - This is now frozen. - All new configuration should be done through the options parser. - This is deprecated and will be removed in 2.0 - -@author: Stijn De Weirdt (Ghent University) -@author: Dries Verdegem (Ghent University) -@author: Kenneth Hoste (Ghent University) -@author: Pieter De Baets (Ghent University) -@author: Jens Timmerman (Ghent University) -@author: Toon Willems (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) -""" - -# -# Developers, please do not add any new defaults or variables -# Use the config options -# - -import os -import tempfile - -import easybuild.tools.config as config - -# this should result in a MODULEPATH=($HOME/.local/easybuild|$EASYBUILDPREFIX)//all -if os.getenv('EASYBUILDPREFIX'): - prefix = os.getenv('EASYBUILDPREFIX') -else: - prefix = os.path.join(os.getenv('HOME'), ".local", "easybuild") - -# build/install/source paths configuration for EasyBuild -# build_path possibly overridden by EASYBUILDBUILDPATH -# install_path possibly overridden by EASYBUILDINSTALLPATH -build_path = os.path.join(prefix, 'build') -install_path = prefix -source_path = os.path.join(prefix, 'sources') - -# repository for eb files -# currently, EasyBuild supports the following repository types: - -# * `FileRepository`: a plain flat file repository. In this case, the `repositoryPath` contains the directory where the files are stored, -# * `GitRepository`: a _non-empty_ **bare** git repository (created with `git init --bare` or `git clone --bare`). -# Here, the `repositoryPath` contains the git repository location, which can be a directory or an URL. -# * `SvnRepository`: an SVN repository. In this case, the `repositoryPath` contains the subversion repository location, again, this can be a directory or an URL. - -# you have to set the `repository` variable inside the config like so: -# `repository = FileRepository(repositoryPath)` - -# optionally a subdir argument can be specified: -# `repository = FileRepository(repositoryPath, subdir)` -repository_path = os.path.join(prefix, 'ebfiles_repo') -repository = FileRepository(repository_path) # @UndefinedVariable (this file gets exec'ed, so ignore this) - -# log format: (dir, filename template) -# supported in template: name, version, data, time -log_format = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log") - -# set the path where log files will be stored -log_dir = tempfile.gettempdir() - -# define set of supported module classes -module_classes = ['base', 'bio', 'chem', 'compiler', 'lib', 'phys', 'tools', - 'cae', 'data', 'debugger', 'devel', 'ide', 'math', 'mpi', 'numlib', 'perf', 'system', 'vis'] - -# general cleanliness -del os, tempfile, config, prefix - -# -# Developers, please do not add any new defaults or variables -# Use the config options -# diff --git a/easybuild/framework/__init__.py b/easybuild/framework/__init__.py index 20b1669d93..bd67a8111f 100644 --- a/easybuild/framework/__init__.py +++ b/easybuild/framework/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 7e5ee5c023..ba52da9052 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -33,12 +33,12 @@ @author: Jens Timmerman (Ghent University) @author: Toon Willems (Ghent University) @author: Ward Poelmans (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ import copy import glob -import re +import inspect import os import shutil import stat @@ -46,34 +46,59 @@ import traceback from distutils.version import LooseVersion from vsc.utils import fancylogger +from vsc.utils.missing import get_class_for import easybuild.tools.environment as env from easybuild.tools import config, filetools -from easybuild.framework.easyconfig.easyconfig import (EasyConfig, ActiveMNS, ITERATE_OPTIONS, - fetch_parameter_from_easyconfig_file, get_class_for, get_easyblock_class, get_module_path, resolve_template) +from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR +from easybuild.framework.easyconfig.easyconfig import ITERATE_OPTIONS, EasyConfig, ActiveMNS +from easybuild.framework.easyconfig.easyconfig import get_easyblock_class, get_module_path, resolve_template +from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig from easybuild.framework.easyconfig.tools import get_paths_for from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP from easybuild.tools.build_details import get_build_stats from easybuild.tools.build_log import EasyBuildError, print_error, print_msg -from easybuild.tools.config import build_path, get_log_filename, get_repository, get_repositorypath, install_path -from easybuild.tools.config import log_path, read_only_installdir, source_paths, build_option -from easybuild.tools.environment import modify_env +from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath +from easybuild.tools.config import install_path, log_path, package_path, source_paths +from easybuild.tools.environment import restore_env from easybuild.tools.filetools import DEFAULT_CHECKSUM from easybuild.tools.filetools import adjust_permissions, apply_patch, convert_name, download_file, encode_class_name -from easybuild.tools.filetools import extract_file, mkdir, read_file, rmtree2 -from easybuild.tools.filetools import write_file, compute_checksum, verify_checksum +from easybuild.tools.filetools import extract_file, mkdir, move_logs, read_file, rmtree2 +from easybuild.tools.filetools import write_file, compute_checksum, verify_checksum, weld_paths from easybuild.tools.run import run_cmd from easybuild.tools.jenkins import write_to_xml -from easybuild.tools.module_generator import ModuleGenerator +from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX from easybuild.tools.modules import get_software_root, modules_tool +from easybuild.tools.package.utilities import package from easybuild.tools.repository.repository import init_repository from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME from easybuild.tools.systemtools import det_parallelism, use_group from easybuild.tools.utilities import remove_unwanted_chars from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION + +BUILD_STEP = 'build' +CLEANUP_STEP = 'cleanup' +CONFIGURE_STEP = 'configure' +EXTENSIONS_STEP = 'extensions' +FETCH_STEP = 'fetch' +MODULE_STEP = 'module' +PACKAGE_STEP = 'package' +PATCH_STEP = 'patch' +PERMISSIONS_STEP = 'permissions' +POSTPROC_STEP = 'postproc' +PREPARE_STEP = 'prepare' +READY_STEP = 'ready' +SANITYCHECK_STEP = 'sanitycheck' +SOURCE_STEP = 'source' +TEST_STEP = 'test' +TESTCASES_STEP = 'testcases' + +MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP] + + _log = fancylogger.getLogger('easyblock') @@ -93,15 +118,9 @@ def extra_options(extra=None): extra = {} if not isinstance(extra, dict): - _log.deprecated("Obtained value of type '%s' for extra, should be 'dict'" % type(extra), '2.0') - _log.debug("Converting extra_options value '%s' of type '%s' to a dict" % (extra, type(extra))) - extra = dict(extra) - - # to avoid breaking backward compatibility, we still need to return a list of tuples in EasyBuild v1.x - _log.deprecated("Returning list of tuples rather than a dict as return value of extra_options", '2.0') - res = extra.items() + _log.nosupport("Obtained 'extra' value of type '%s' in extra_options, should be 'dict'" % type(extra), '2.0') - return res + return extra # # INIT @@ -112,6 +131,9 @@ def __init__(self, ec): @param ec: a parsed easyconfig file (EasyConfig instance) """ + # keep track of original working directory, so we can go back there + self.orig_workdir = os.getcwd() + # list of patch/source files, along with checksums self.patches = [] self.src = [] @@ -131,7 +153,7 @@ def __init__(self, ec): # modules interface with default MODULEPATH self.modules_tool = modules_tool() # module generator - self.moduleGenerator = ModuleGenerator(self, fake=True) + self.module_generator = module_generator(self, fake=True) # modules footer self.modules_footer = None @@ -139,14 +161,14 @@ def __init__(self, ec): if modules_footer_path is not None: self.modules_footer = read_file(modules_footer_path) - # recursive unloading in modules - self.recursive_mod_unload = build_option('recursive_mod_unload') - # easyconfig for this application if isinstance(ec, EasyConfig): self.cfg = ec else: - _log.error("Value of incorrect type passed to EasyBlock constructor: %s ('%s')" % (type(ec), ec)) + raise EasyBuildError("Value of incorrect type passed to EasyBlock constructor: %s ('%s')", type(ec), ec) + + # determine install subdirectory, based on module name + self.install_subdir = None # indicates whether build should be performed in installation dir self.build_in_installdir = self.cfg['buildininstalldir'] @@ -157,30 +179,27 @@ def __init__(self, ec): self.logdebug = build_option('debug') self.postmsg = '' # allow a post message to be set, which can be shown as last output - # original environ will be set later - self.orig_environ = {} - # list of loaded modules self.loaded_modules = [] + # iterate configure/build/options + self.iter_opts = {} + + # sanity check fail error messages to report (if any) + self.sanity_check_fail_msgs = [] + # robot path self.robot_path = build_option('robot_path') # original module path self.orig_modulepath = os.getenv('MODULEPATH') - # keep track of original environment, so we restore it if needed - self.orig_environ = copy.deepcopy(os.environ) + # keep track of initial environment we start in, so we can restore it if needed + self.initial_environ = copy.deepcopy(os.environ) - # at the end of __init__, initialise the logging + # initialize logger self._init_log() - # iterate configure/build/options - self.iter_opts = {} - - # sanity check fail error messages to report (if any) - self.sanity_check_fail_msgs = [] - # should we keep quiet? self.silent = build_option('silent') @@ -194,6 +213,10 @@ def __init__(self, ec): if group_name is not None: self.group = use_group(group_name) + # generate build/install directories + self.gen_builddir() + self.gen_installdir() + self.log.info("Init completed for application name %s version %s" % (self.name, self.version)) # INIT/CLOSE LOG @@ -211,6 +234,10 @@ def _init_log(self): self.log.info(this_is_easybuild()) + this_module = inspect.getmodule(self) + self.log.info("This is easyblock %s from module %s (%s)", + self.__class__.__name__, this_module.__name__, this_module.__file__) + def close_log(self): """ Shutdown the logger. @@ -238,7 +265,7 @@ def get_checksum_for(self, checksums, filename=None, index=None): elif checksums is None: return None else: - self.log.error("Invalid type for checksums (%s), should be list, tuple or None." % type(checksums)) + raise EasyBuildError("Invalid type for checksums (%s), should be list, tuple or None.", type(checksums)) def fetch_sources(self, list_of_sources, checksums=None): """ @@ -267,54 +294,60 @@ def fetch_sources(self, list_of_sources, checksums=None): 'finalpath': self.builddir, }) else: - self.log.error('No file found for source %s' % source) + raise EasyBuildError('No file found for source %s', source) self.log.info("Added sources: %s" % self.src) - def fetch_patches(self, list_of_patches, extension=False, checksums=None): + def fetch_patches(self, patch_specs=None, extension=False, checksums=None): """ Add a list of patches. All patches will be checked if a file exists (or can be located) """ + if patch_specs is None: + patch_specs = self.cfg['patches'] patches = [] - for index, patch_entry in enumerate(list_of_patches): + for index, patch_spec in enumerate(patch_specs): # check if the patches can be located copy_file = False suff = None level = None - if isinstance(patch_entry, (list, tuple)): - if not len(patch_entry) == 2: - self.log.error("Unknown patch specification '%s', only two-element lists/tuples are supported!" % patch_entry) - pf = patch_entry[0] - - if isinstance(patch_entry[1], int): - level = patch_entry[1] - elif isinstance(patch_entry[1], basestring): + if isinstance(patch_spec, (list, tuple)): + if not len(patch_spec) == 2: + raise EasyBuildError("Unknown patch specification '%s', only 2-element lists/tuples are supported!", + str(patch_spec)) + patch_file = patch_spec[0] + + # this *must* be of typ int, nothing else + # no 'isinstance(..., int)', since that would make True/False also acceptable + if type(patch_spec[1]) == int: + level = patch_spec[1] + elif isinstance(patch_spec[1], basestring): # non-patch files are assumed to be files to copy - if not patch_entry[0].endswith('.patch'): + if not patch_spec[0].endswith('.patch'): copy_file = True - suff = patch_entry[1] + suff = patch_spec[1] else: - self.log.error("Wrong patch specification '%s', only int and string are supported as second element!" % patch_entry) + raise EasyBuildError("Wrong patch spec '%s', only int/string are supported as 2nd element", + str(patch_spec)) else: - pf = patch_entry + patch_file = patch_spec - path = self.obtain_file(pf, extension=extension) + path = self.obtain_file(patch_file, extension=extension) if path: - self.log.debug('File %s found for patch %s' % (path, patch_entry)) + self.log.debug('File %s found for patch %s' % (path, patch_spec)) patchspec = { - 'name': pf, + 'name': patch_file, 'path': path, - 'checksum': self.get_checksum_for(checksums, filename=pf, index=index), + 'checksum': self.get_checksum_for(checksums, filename=patch_file, index=index), } if suff: if copy_file: patchspec['copy'] = suff else: patchspec['sourcepath'] = suff - if level: + if level is not None: patchspec['level'] = level if extension: @@ -322,7 +355,7 @@ def fetch_patches(self, list_of_patches, extension=False, checksums=None): else: self.patches.append(patchspec) else: - self.log.error('No file found for patch %s' % patch_entry) + raise EasyBuildError('No file found for patch %s', patch_spec) if extension: self.log.info("Fetched extension patches: %s" % patches) @@ -356,9 +389,9 @@ def fetch_extension_sources(self): ext_options = ext[2] if not isinstance(ext_options, dict): - self.log.error("Unexpected type (non-dict) for 3rd element of %s" % ext) + raise EasyBuildError("Unexpected type (non-dict) for 3rd element of %s", ext) elif len(ext) > 3: - self.log.error('Extension specified in unknown format (list/tuple too long)') + raise EasyBuildError('Extension specified in unknown format (list/tuple too long)') ext_src = { 'name': ext_name, @@ -387,9 +420,9 @@ def fetch_extension_sources(self): if verify_checksum(src_fn, fn_checksum): self.log.info('Checksum for ext source %s verified' % fn) else: - self.log.error('Checksum for ext source %s failed' % fn) + raise EasyBuildError('Checksum for ext source %s failed', fn) - ext_patches = self.fetch_patches(ext_options.get('patches', []), extension=True) + ext_patches = self.fetch_patches(patch_specs=ext_options.get('patches', []), extension=True) if ext_patches: self.log.debug('Found patches for extension %s: %s' % (ext_name, ext_patches)) ext_src.update({'patches': ext_patches}) @@ -401,20 +434,20 @@ def fetch_extension_sources(self): if verify_checksum(ext_patch, checksum): self.log.info('Checksum for extension patch %s verified' % ext_patch) else: - self.log.error('Checksum for extension patch %s failed' % ext_patch) + raise EasyBuildError('Checksum for extension patch %s failed', ext_patch) else: self.log.debug('No patches found for extension %s.' % ext_name) exts_sources.append(ext_src) else: - self.log.error("Source for extension %s not found.") + raise EasyBuildError("Source for extension %s not found.", ext) elif isinstance(ext, basestring): exts_sources.append({'name': ext}) else: - self.log.error("Extension specified in unknown format (not a string/list/tuple)") + raise EasyBuildError("Extension specified in unknown format (not a string/list/tuple)") return exts_sources @@ -454,7 +487,7 @@ def obtain_file(self, filename, extension=False, urls=None): return fullpath except IOError, err: - self.log.exception("Downloading file %s from url %s to %s failed: %s" % (filename, url, fullpath, err)) + raise EasyBuildError("Downloading file %s from url %s to %s failed: %s", filename, url, fullpath, err) else: # try and find file in various locations @@ -466,9 +499,9 @@ def obtain_file(self, filename, extension=False, urls=None): # always consider robot + easyconfigs install paths as a fall back (e.g. for patch files, test cases, ...) common_filepaths = [] - if self.robot_path is not None: + if self.robot_path: common_filepaths.extend(self.robot_path) - common_filepaths.extend(get_paths_for("easyconfigs", robot_path=self.robot_path)) + common_filepaths.extend(get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=self.robot_path)) for path in ebpath + common_filepaths + srcpaths: # create list of candidate filepaths @@ -559,7 +592,8 @@ def obtain_file(self, filename, extension=False, urls=None): else: failedpaths.append(fullurl) - self.log.error("Couldn't find file %s anywhere, and downloading it didn't work either...\nPaths attempted (in order): %s " % (filename, ', '.join(failedpaths))) + raise EasyBuildError("Couldn't find file %s anywhere, and downloading it didn't work either... " + "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) # # GETTER/SETTER UTILITY FUNCTIONS @@ -588,10 +622,24 @@ def toolchain(self): @property def full_mod_name(self): """ - Toolchain used to build this easyblock + Full module name (including subdirectory in module install path) """ return self.cfg.full_mod_name + @property + def short_mod_name(self): + """ + Short module name (not including subdirectory in module install path) + """ + return self.cfg.short_mod_name + + @property + def moduleGenerator(self): + """ + Module generator (DEPRECATED, use self.module_generator instead). + """ + self.log.nosupport("self.moduleGenerator is replaced by self.module_generator", '2.0') + # # DIRECTORY UTILITY FUNCTIONS # @@ -624,8 +672,8 @@ def make_builddir(self): if not self.build_in_installdir: # self.builddir should be already set by gen_builddir() if not self.builddir: - self.log.error("self.builddir not set, make sure gen_builddir() is called first!") - self.log.debug("Creating the build directory %s (cleanup: %s)" % (self.builddir, self.cfg['cleanupoldbuild'])) + raise EasyBuildError("self.builddir not set, make sure gen_builddir() is called first!") + self.log.debug("Creating the build directory %s (cleanup: %s)", self.builddir, self.cfg['cleanupoldbuild']) else: self.log.info("Changing build dir to %s" % self.installdir) self.builddir = self.installdir @@ -646,12 +694,12 @@ def gen_installdir(self): Generate the name of the installation directory. """ basepath = install_path() - if basepath: - installdir = os.path.join(basepath, self.full_mod_name) - self.installdir = os.path.abspath(installdir) + self.install_subdir = ActiveMNS().det_install_subdir(self.cfg) + self.installdir = os.path.join(os.path.abspath(basepath), self.install_subdir) + self.log.info("Install dir set to %s" % self.installdir) else: - self.log.error("Can't set installation directory") + raise EasyBuildError("Can't set installation directory") def make_installdir(self, dontcreate=None): """ @@ -678,7 +726,7 @@ def make_dir(self, dir_name, clean, dontcreateinstalldir=False): rmtree2(dir_name) self.log.info("Removed old directory %s" % dir_name) except OSError, err: - self.log.exception("Removal of old directory %s failed: %s" % (dir_name, err)) + raise EasyBuildError("Removal of old directory %s failed: %s", dir_name, err) else: try: timestamp = time.strftime("%Y%m%d-%H%M%S") @@ -686,7 +734,7 @@ def make_dir(self, dir_name, clean, dontcreateinstalldir=False): shutil.move(dir_name, backupdir) self.log.info("Moved old directory %s to %s" % (dir_name, backupdir)) except OSError, err: - self.log.exception("Moving old directory to backup %s %s failed: %s" % (dir_name, backupdir, err)) + raise EasyBuildError("Moving old directory to backup %s %s failed: %s", dir_name, backupdir, err) if dontcreateinstalldir: olddir = dir_name @@ -716,29 +764,32 @@ def make_devel_module(self, create_in_builddir=False): # load fake module fake_mod_data = self.load_fake_module(purge=True) - mod_gen = ModuleGenerator(self) - header = "#%Module\n" - - env_txt = "" - for (key, val) in env.get_changes().items(): - # check if non-empty string - # TODO: add unset for empty vars? - if val.strip(): - env_txt += mod_gen.set_environment(key, val) + header = self.module_generator.MODULE_HEADER + if header: + header += '\n' - load_txt = "" + load_lines = [] # capture all the EBDEVEL vars # these should be all the dependencies and we should load them + recursive_unload = self.cfg['recursive_module_unload'] for key in os.environ: # legacy support - if key.startswith(DEVEL_ENV_VAR_NAME_PREFIX) or key.startswith("SOFTDEVEL"): - if key.startswith("SOFTDEVEL"): - self.log.deprecated("Environment variable SOFTDEVEL* being relied on", "2.0") + if key.startswith(DEVEL_ENV_VAR_NAME_PREFIX): if not key.endswith(convert_name(self.name, upper=True)): path = os.environ[key] if os.path.isfile(path): mod_name = path.rsplit(os.path.sep, 1)[-1] - load_txt += mod_gen.load_module(mod_name, recursive_unload=self.recursive_mod_unload) + load_statement = self.module_generator.load_module(mod_name, recursive_unload=recursive_unload) + load_lines.append(load_statement) + elif key.startswith('SOFTDEVEL'): + self.log.nosupport("Environment variable SOFTDEVEL* being relied on", '2.0') + + env_lines = [] + for (key, val) in env.get_changes().items(): + # check if non-empty string + # TODO: add unset for empty vars? + if val.strip(): + env_lines.append(self.module_generator.set_environment(key, val)) if create_in_builddir: output_dir = self.builddir @@ -749,7 +800,8 @@ def make_devel_module(self, create_in_builddir=False): filename = os.path.join(output_dir, ActiveMNS().det_devel_module_filename(self.cfg)) self.log.debug("Writing devel module to %s" % filename) - write_file(filename, header + load_txt + env_txt) + txt = ''.join([header] + load_lines + env_lines) + write_file(filename, txt) # cleanup: unload fake module, remove fake module dir self.clean_up_fake_module(fake_mod_data) @@ -758,7 +810,18 @@ def make_module_dep(self): """ Make the dependencies for the module file. """ - load = unload = '' + deps = [] + mns = ActiveMNS() + + # include load statements for toolchain, either directly or for toolchain dependencies + if self.toolchain.name != DUMMY_TOOLCHAIN_NAME: + if mns.expand_toolchain_load(): + mod_names = self.toolchain.toolchain_dep_mods + deps.extend(mod_names) + self.log.debug("Adding toolchain components as module dependencies: %s" % mod_names) + else: + deps.append(self.toolchain.det_short_module_name()) + self.log.debug("Adding toolchain %s as a module dependency" % deps[-1]) # include load/unload statements for dependencies builddeps = self.cfg.builddependencies() @@ -767,65 +830,90 @@ def make_module_dep(self): if not dep in builddeps: modname = dep['short_mod_name'] self.log.debug("Adding %s as a module dependency" % modname) - load += self.moduleGenerator.load_module(modname, recursive_unload=self.recursive_mod_unload) - unload += self.moduleGenerator.unload_module(modname) + deps.append(modname) else: self.log.debug("Skipping build dependency %s" % str(dep)) - # include load statements for toolchain, either directly or for toolchain dependencies - # purposely after dependencies which may be critical, - # e.g. when unloading a module in a hierarchical naming scheme - if self.toolchain.name != DUMMY_TOOLCHAIN_NAME: - if ActiveMNS().expand_toolchain_load(): - mod_names = self.toolchain.toolchain_dependencies - else: - mod_names = [self.toolchain.det_short_module_name()] - for mod_name in mod_names: - load += self.moduleGenerator.load_module(mod_name, recursive_unload=self.recursive_mod_unload) - unload += self.moduleGenerator.unload_module(mod_name) + self.log.debug("Full list of dependencies: %s" % deps) + + # exclude dependencies that extend $MODULEPATH and form the path to the top of the module tree (if any) + mod_install_path = os.path.join(install_path('mod'), build_option('suffix_modules_path')) + full_mod_subdir = os.path.join(mod_install_path, self.cfg.mod_subdir) + init_modpaths = mns.det_init_modulepaths(self.cfg) + top_paths = [mod_install_path] + [os.path.join(mod_install_path, p) for p in init_modpaths] + excluded_deps = self.modules_tool.path_to_top_of_module_tree(top_paths, self.cfg.short_mod_name, + full_mod_subdir, deps) + + deps = [d for d in deps if d not in excluded_deps] + self.log.debug("List of retained dependencies: %s" % deps) + recursive_unload = self.cfg['recursive_module_unload'] + loads = [self.module_generator.load_module(d, recursive_unload=recursive_unload) for d in deps] + unloads = [self.module_generator.unload_module(d) for d in deps[::-1]] # Force unloading any other modules if self.cfg['moduleforceunload']: - return unload + load + return ''.join(unloads) + ''.join(loads) else: - return load + return ''.join(loads) def make_module_description(self): """ Create the module description. """ - return self.moduleGenerator.get_description() + return self.module_generator.get_description() def make_module_extra(self): """ Sets optional variables (EBROOT, MPI tuning variables). """ - txt = "\n" + lines = [''] # EBROOT + EBVERSION + EBDEVEL - environment_name = convert_name(self.name, upper=True) - txt += self.moduleGenerator.set_environment(ROOT_ENV_VAR_NAME_PREFIX + environment_name, "$root") - txt += self.moduleGenerator.set_environment(VERSION_ENV_VAR_NAME_PREFIX + environment_name, self.version) - devel_path = os.path.join("$root", log_path(), ActiveMNS().det_devel_module_filename(self.cfg)) - txt += self.moduleGenerator.set_environment(DEVEL_ENV_VAR_NAME_PREFIX + environment_name, devel_path) + env_name = convert_name(self.name, upper=True) + + lines.append(self.module_generator.set_environment(ROOT_ENV_VAR_NAME_PREFIX + env_name, '', relpath=True)) + lines.append(self.module_generator.set_environment(VERSION_ENV_VAR_NAME_PREFIX + env_name, self.version)) + + devel_path = os.path.join(log_path(), ActiveMNS().det_devel_module_filename(self.cfg)) + devel_path_envvar = DEVEL_ENV_VAR_NAME_PREFIX + env_name + lines.append(self.module_generator.set_environment(devel_path_envvar, devel_path, relpath=True)) - txt += "\n" + lines.append('\n') for (key, value) in self.cfg['modextravars'].items(): - txt += self.moduleGenerator.set_environment(key, value) + lines.append(self.module_generator.set_environment(key, value)) + for (key, value) in self.cfg['modextrapaths'].items(): if isinstance(value, basestring): value = [value] elif not isinstance(value, (tuple, list)): - self.log.error("modextrapaths dict value %s (type: %s) is not a list or tuple" % (value, type(value))) - txt += self.moduleGenerator.prepend_paths(key, value) + raise EasyBuildError("modextrapaths dict value %s (type: %s) is not a list or tuple", + value, type(value)) + lines.append(self.module_generator.prepend_paths(key, value)) + if self.cfg['modloadmsg']: - txt += self.moduleGenerator.msg_on_load(self.cfg['modloadmsg']) + lines.append(self.module_generator.msg_on_load(self.cfg['modloadmsg'])) + if self.cfg['modtclfooter']: - txt += self.moduleGenerator.add_tcl_footer(self.cfg['modtclfooter']) + if isinstance(self.module_generator, ModuleGeneratorTcl): + self.log.debug("Including Tcl footer in module: %s", self.cfg['modtclfooter']) + lines.extend([self.cfg['modtclfooter'], '\n']) + else: + self.log.warning("Not including footer in Tcl syntax in non-Tcl module file: %s", + self.cfg['modtclfooter']) + + if self.cfg['modluafooter']: + if isinstance(self.module_generator, ModuleGeneratorLua): + self.log.debug("Including Lua footer in module: %s", self.cfg['modluafooter']) + lines.extend([self.cfg['modluafooter'], '\n']) + else: + self.log.warning("Not including footer in Lua syntax in non-Lua module file: %s", + self.cfg['modluafooter']) + for (key, value) in self.cfg['modaliases'].items(): - txt += self.moduleGenerator.set_alias(key, value) + lines.append(self.module_generator.set_alias(key, value)) - self.log.debug("make_module_extra added this: %s" % txt) + txt = ''.join(lines) + self.log.debug("make_module_extra added this: %s", txt) return txt @@ -834,46 +922,50 @@ def make_module_extra_extensions(self): Sets optional variables for extensions. """ # add stuff specific to individual extensions - txt = self.module_extra_extensions + lines = [self.module_extra_extensions] # set environment variable that specifies list of extensions if self.exts_all: exts_list = ','.join(['%s-%s' % (ext['name'], ext.get('version', '')) for ext in self.exts_all]) - txt += self.moduleGenerator.set_environment('EBEXTSLIST%s' % self.name.upper(), exts_list) + env_var_name = convert_name(self.name, upper=True) + lines.append(self.module_generator.set_environment('EBEXTSLIST%s' % env_var_name, exts_list)) - return txt + return ''.join(lines) def make_module_footer(self): """ Insert a footer section in the modulefile, primarily meant for contextual information """ - txt = '\n# Built with EasyBuild version %s\n' % VERBOSE_VERSION + footer = [self.module_generator.comment("Built with EasyBuild version %s" % VERBOSE_VERSION)] # add extra stuff for extensions (if any) if self.cfg['exts_list']: - txt += self.make_module_extra_extensions() + footer.append(self.make_module_extra_extensions()) # include modules footer if one is specified if self.modules_footer is not None: self.log.debug("Including specified footer into module: '%s'" % self.modules_footer) - txt += self.modules_footer + footer.append(self.modules_footer) - return txt + return ''.join(footer) def make_module_extend_modpath(self): """ Include prepend-path statements for extending $MODULEPATH. """ - top_modpath = install_path('mod') - modpath_exts = ActiveMNS().det_modpath_extensions(self.cfg) txt = '' - if modpath_exts: + if self.cfg['include_modpath_extensions']: + top_modpath = install_path('mod') mod_path_suffix = build_option('suffix_modules_path') + modpath_exts = ActiveMNS().det_modpath_extensions(self.cfg) + self.log.debug("Including module path extensions returned by module naming scheme: %s" % modpath_exts) full_path_modpath_extensions = [os.path.join(top_modpath, mod_path_suffix, ext) for ext in modpath_exts] # module path extensions must exist, otherwise loading this module file will fail for modpath_extension in full_path_modpath_extensions: mkdir(modpath_extension, parents=True) - txt = self.moduleGenerator.use(full_path_modpath_extensions) + txt = self.module_generator.use(full_path_modpath_extensions) + else: + self.log.debug("Not including module path extensions, as specified.") return txt def make_module_req(self): @@ -882,26 +974,25 @@ def make_module_req(self): """ requirements = self.make_module_req_guess() - if os.path.exists(self.installdir): + lines = [] + if os.path.isdir(self.installdir): try: - cwd = os.getcwd() os.chdir(self.installdir) except OSError, err: - self.log.error("Failed to change to %s: %s" % (self.installdir, err)) + raise EasyBuildError("Failed to change to %s: %s", self.installdir, err) - txt = "\n" + lines.append('\n') for key in sorted(requirements): for path in requirements[key]: - paths = glob.glob(path) + paths = sorted(glob.glob(path)) if paths: - txt += self.moduleGenerator.prepend_paths(key, paths) + lines.append(self.module_generator.prepend_paths(key, paths)) try: - os.chdir(cwd) + os.chdir(self.orig_workdir) except OSError, err: - self.log.error("Failed to change back to %s: %s" % (cwd, err)) - else: - txt = "" - return txt + raise EasyBuildError("Failed to change back to %s: %s", self.orig_workdir, err) + + return ''.join(lines) def make_module_req_guess(self): """ @@ -928,7 +1019,7 @@ def load_module(self, mod_paths=None, purge=True): mod_paths = [] all_mod_paths = mod_paths + ActiveMNS().det_init_modulepaths(self.cfg) mods = [self.full_mod_name] - self.modules_tool.load(mods, mod_paths=all_mod_paths, purge=purge, orig_env=self.orig_environ) + self.modules_tool.load(mods, mod_paths=all_mod_paths, purge=purge, init_env=self.initial_environ) else: self.log.warning("Not loading module, since self.full_mod_name is not set.") @@ -936,40 +1027,37 @@ def load_fake_module(self, purge=False): """ Create and load fake module. """ - - # take a copy of the environment before loading the fake module, so we can restore it - orig_env = copy.deepcopy(os.environ) + # take a copy of the current environment before loading the fake module, so we can restore it + env = copy.deepcopy(os.environ) # create fake module - fake_mod_path = self.make_module_step(True) + fake_mod_path = self.make_module_step(fake=True) # load fake module - modtool = modules_tool() - modtool.prepend_module_path(fake_mod_path) + self.modules_tool.prepend_module_path(fake_mod_path) self.load_module(purge=purge) - return (fake_mod_path, orig_env) + return (fake_mod_path, env) def clean_up_fake_module(self, fake_mod_data): """ Clean up fake module. """ - fake_mod_path, orig_env = fake_mod_data + fake_mod_path, env = fake_mod_data # unload module and remove temporary module directory # self.full_mod_name might not be set (e.g. during unit tests) if fake_mod_path and self.full_mod_name is not None: try: - modtool = modules_tool() - modtool.unload([self.full_mod_name]) - modtool.remove_module_path(fake_mod_path) + self.modules_tool.unload([self.full_mod_name]) + self.modules_tool.remove_module_path(fake_mod_path) rmtree2(os.path.dirname(fake_mod_path)) except OSError, err: - self.log.error("Failed to clean up fake module dir %s: %s" % (fake_mod_path, err)) + raise EasyBuildError("Failed to clean up fake module dir %s: %s", fake_mod_path, err) elif self.full_mod_name is None: self.log.warning("Not unloading module, since self.full_mod_name is not set.") # restore original environment - modify_env(os.environ, orig_env) + restore_env(env) def load_dependency_modules(self): """Load dependency modules.""" @@ -997,9 +1085,9 @@ def skip_extensions(self): self.cfg.enable_templating = True if not exts_filter or len(exts_filter) == 0: - self.log.error("Skipping of extensions, but no exts_filter set in easyconfig") + raise EasyBuildError("Skipping of extensions, but no exts_filter set in easyconfig") elif isinstance(exts_filter, basestring) or len(exts_filter) != 2: - self.log.error('exts_filter should be a list or tuple of ("command","input")') + raise EasyBuildError('exts_filter should be a list or tuple of ("command","input")') cmdtmpl = exts_filter[0] cmdinputtmpl = exts_filter[1] if not self.exts: @@ -1018,14 +1106,13 @@ def skip_extensions(self): 'src': ext.get('source'), } - deprecated_msg = "Providing 'name' and 'version' keys for extensions, should use 'ext_name', 'ext_version'" - self.log.deprecated(deprecated_msg, '2.0') - tmpldict.update({ - 'name': modname, - 'version': ext.get('version'), - }) + try: + cmd = cmdtmpl % tmpldict + except KeyError, err: + msg = "KeyError occured on completing extension filter template: %s; " + msg += "'name'/'version' keys are no longer supported, should use 'ext_name'/'ext_version' instead" + self.log.nosupport(msg, '2.0') - cmd = cmdtmpl % tmpldict if cmdinputtmpl: stdin = cmdinputtmpl % tmpldict (cmdstdouterr, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, regexp=False) @@ -1052,21 +1139,32 @@ def guess_start_dir(self): -- if abspath: use that -- else, treat it as subdir for regular procedure """ - tmpdir = '' + start_dir = '' if self.cfg['start_dir']: - tmpdir = self.cfg['start_dir'] + start_dir = self.cfg['start_dir'] - if not os.path.isabs(tmpdir): + if not os.path.isabs(start_dir): if len(self.src) > 0 and not self.skip and self.src[0]['finalpath']: - self.cfg['start_dir'] = os.path.join(self.src[0]['finalpath'], tmpdir) + topdir = self.src[0]['finalpath'] + else: + topdir = self.builddir + + abs_start_dir = os.path.join(topdir, start_dir) + if topdir.endswith(start_dir) and not os.path.exists(abs_start_dir): + self.cfg['start_dir'] = topdir else: - self.cfg['start_dir'] = os.path.join(self.builddir, tmpdir) + if os.path.exists(abs_start_dir): + self.cfg['start_dir'] = abs_start_dir + else: + raise EasyBuildError("Specified start dir %s does not exist", abs_start_dir) + + self.log.info("Using %s as start dir", self.cfg['start_dir']) try: os.chdir(self.cfg['start_dir']) - self.log.debug("Changed to real build directory %s" % (self.cfg['start_dir'])) + self.log.debug("Changed to real build directory %s (start_dir)" % self.cfg['start_dir']) except OSError, err: - self.log.exception("Can't change to real build directory %s: %s" % (self.cfg['start_dir'], err)) + raise EasyBuildError("Can't change to real build directory %s: %s", self.cfg['start_dir'], err) def handle_iterate_opts(self): """Handle options relevant during iterated part of build/install procedure.""" @@ -1110,6 +1208,20 @@ def check_readiness_step(self): """ Verify if all is ok to start build. """ + # set level of parallelism for build + par = build_option('parallel') + if self.cfg['parallel']: + if par is None: + par = self.cfg['parallel'] + self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par) + else: + par = min(int(par), int(self.cfg['parallel'])) + self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par) + else: + self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par) + self.cfg['parallel'] = det_parallelism(par=par, maxpar=self.cfg['maxparallel']) + self.log.info("Setting parallelism: %s" % self.cfg['parallel']) + # check whether modules are loaded loadedmods = self.modules_tool.loaded_modules() if len(loadedmods) > 0: @@ -1120,18 +1232,18 @@ def check_readiness_step(self): if not len(self.cfg.dependencies()) == len(self.toolchain.dependencies): self.log.debug("dep %s (%s)" % (len(self.cfg.dependencies()), self.cfg.dependencies())) self.log.debug("tc.dep %s (%s)" % (len(self.toolchain.dependencies), self.toolchain.dependencies)) - self.log.error('Not all dependencies have a matching toolchain version') + raise EasyBuildError('Not all dependencies have a matching toolchain version') # check if the application is not loaded at the moment (root, env_var) = get_software_root(self.name, with_env_var=True) if root: - self.log.error("Module is already loaded (%s is set), installation cannot continue." % env_var) + raise EasyBuildError("Module is already loaded (%s is set), installation cannot continue.", env_var) # check if main install needs to be skipped # - if a current module can be found, skip is ok # -- this is potentially very dangerous if self.cfg['skip']: - if self.modules_tool.exists(self.full_mod_name): + if self.modules_tool.exist([self.full_mod_name])[0]: self.skip = True self.log.info("Module %s found." % self.full_mod_name) self.log.info("Going to skip actual main build and potential existing extensions. Expert only.") @@ -1145,12 +1257,15 @@ def fetch_step(self, skip_checksums=False): # check EasyBuild version easybuild_version = self.cfg['easybuild_version'] if not easybuild_version: - self.log.warn("Easyconfig does not specify an EasyBuild-version (key 'easybuild_version')! Assuming the latest version") + self.log.warn("Easyconfig does not specify an EasyBuild-version (key 'easybuild_version')! " + "Assuming the latest version") else: if LooseVersion(easybuild_version) < VERSION: - self.log.warn("EasyBuild-version %s is older than the currently running one. Proceed with caution!" % easybuild_version) + self.log.warn("EasyBuild-version %s is older than the currently running one. Proceed with caution!", + easybuild_version) elif LooseVersion(easybuild_version) > VERSION: - self.log.error("EasyBuild-version %s is newer than the currently running one. Aborting!" % easybuild_version) + raise EasyBuildError("EasyBuild-version %s is newer than the currently running one. Aborting!", + easybuild_version) # fetch sources if self.cfg['sources']: @@ -1158,6 +1273,10 @@ def fetch_step(self, skip_checksums=False): else: self.log.info('no sources provided') + # fetch extensions + if len(self.cfg['exts_list']) > 0: + self.exts = self.fetch_extension_sources() + # fetch patches if self.cfg['patches']: if isinstance(self.cfg['checksums'], (list, tuple)): @@ -1165,7 +1284,7 @@ def fetch_step(self, skip_checksums=False): patches_checksums = self.cfg['checksums'][len(self.cfg['sources']):] else: patches_checksums = self.cfg['checksums'] - self.fetch_patches(self.cfg['patches'], checksums=patches_checksums) + self.fetch_patches(checksums=patches_checksums) else: self.log.info('no patches provided') @@ -1176,15 +1295,11 @@ def fetch_step(self, skip_checksums=False): fil[DEFAULT_CHECKSUM] = check_sum self.log.info("%s checksum for %s: %s" % (DEFAULT_CHECKSUM, fil['path'], fil[DEFAULT_CHECKSUM])) - # set level of parallelism for build - self.cfg['parallel'] = det_parallelism(self.cfg['parallel'], self.cfg['maxparallel']) - self.log.info("Setting parallelism: %s" % self.cfg['parallel']) - # create parent dirs in install and modules path already # this is required when building in parallel mod_path_suffix = build_option('suffix_modules_path') mod_symlink_paths = ActiveMNS().det_module_symlink_paths(self.cfg) - parent_subdir = os.path.dirname(self.full_mod_name) + parent_subdir = os.path.dirname(self.install_subdir) pardirs = [ os.path.join(install_path(), parent_subdir), os.path.join(install_path('mod'), mod_path_suffix, parent_subdir), @@ -1201,7 +1316,7 @@ def checksum_step(self): for fil in self.src + self.patches: ok = verify_checksum(fil['path'], fil['checksum']) if not ok: - self.log.error("Checksum verification for %s using %s failed." % (fil['path'], fil['checksum'])) + raise EasyBuildError("Checksum verification for %s using %s failed.", fil['path'], fil['checksum']) else: self.log.info("Checksum verification for %s using %s passed." % (fil['path'], fil['checksum'])) @@ -1215,7 +1330,7 @@ def extract_step(self): if srcdir: self.src[self.src.index(src)]['finalpath'] = srcdir else: - self.log.error("Unpacking source %s failed" % src['name']) + raise EasyBuildError("Unpacking source %s failed", src['name']) def patch_step(self, beginpath=None): """ @@ -1224,36 +1339,46 @@ def patch_step(self, beginpath=None): for patch in self.patches: self.log.info("Applying patch %s" % patch['name']) - copy = False - # default: patch first source - srcind = 0 - if 'source' in patch: - srcind = patch['source'] - srcpathsuffix = '' - if 'sourcepath' in patch: - srcpathsuffix = patch['sourcepath'] - elif 'copy' in patch: - srcpathsuffix = patch['copy'] - copy = True + # patch source at specified index (first source if not specified) + srcind = patch.get('source', 0) + # if patch level is specified, use that (otherwise let apply_patch derive patch level) + level = patch.get('level', None) + # determine suffix of source path to apply patch in (if any) + srcpathsuffix = patch.get('sourcepath', patch.get('copy', '')) + # determine whether 'patch' file should be copied rather than applied + copy_patch = 'copy' in patch and not 'sourcepath' in patch - if not beginpath: - beginpath = self.src[srcind]['finalpath'] + self.log.debug("Source index: %s; patch level: %s; source path suffix: %s; copy patch: %s", + srcind, level, srcpathsuffix, copy) - src = os.path.abspath("%s/%s" % (beginpath, srcpathsuffix)) + if beginpath is None: + try: + beginpath = self.src[srcind]['finalpath'] + self.log.debug("Determine begin path for patch %s: %s" % (patch['name'], beginpath)) + except IndexError, err: + raise EasyBuildError("Can't apply patch %s to source at index %s of list %s: %s", + patch['name'], srcind, self.src, err) + else: + self.log.debug("Using specified begin path for patch %s: %s" % (patch['name'], beginpath)) - level = None - if 'level' in patch: - level = patch['level'] + # detect partial overlap between paths + src = os.path.abspath(weld_paths(beginpath, srcpathsuffix)) + self.log.debug("Applying patch %s in path %s", patch, src) - if not apply_patch(patch['path'], src, copy=copy, level=level): - self.log.error("Applying patch %s failed" % patch['name']) + if not apply_patch(patch['path'], src, copy=copy_patch, level=level): + raise EasyBuildError("Applying patch %s failed", patch['name']) def prepare_step(self): """ Pre-configure step. Set's up the builddir just before starting configure """ + # clean environment, undefine any unwanted environment variables that may be harmful self.cfg['unwanted_env_vars'] = env.unset_env_vars(self.cfg['unwanted_env_vars']) + + # prepare toolchain: load toolchain module and dependencies, set up build environment self.toolchain.prepare(self.cfg['onlytcmod']) + + # guess directory to start configure/build/install process in, and move there self.guess_start_dir() def configure_step(self): @@ -1283,7 +1408,7 @@ def install_step(self): """Install built software (abstract method).""" raise NotImplementedError - def extensions_step(self): + def extensions_step(self, fetch=False): """ After make install, run this. - only if variable len(exts_list) > 0 @@ -1300,7 +1425,9 @@ def extensions_step(self): self.prepare_for_extensions() - self.exts = self.fetch_extension_sources() + if fetch: + self.exts = self.fetch_extension_sources() + self.exts_all = self.exts[:] # retain a copy of all extensions, regardless of filtering/skipping if self.skip: @@ -1314,21 +1441,11 @@ def extensions_step(self): # we really need a default class if not exts_defaultclass: self.clean_up_fake_module(fake_mod_data) - self.log.error("ERROR: No default extension class set for %s" % self.name) + raise EasyBuildError("ERROR: No default extension class set for %s", self.name) # obtain name and module path for default extention class - legacy = False if hasattr(exts_defaultclass, '__iter__'): - # LEGACY: module path is explicitely specified - self.log.warning("LEGACY: using specified module path for default class (will be deprecated soon)") - default_class_modpath = exts_defaultclass[0] - default_class = exts_defaultclass[1] - derived_mod_path = get_module_path(default_class, generic=True) - if not default_class_modpath == derived_mod_path: - msg = "Specified module path for default class %s " % default_class_modpath - msg += "doesn't match derived path %s" % derived_mod_path - self.log.warning(msg) - legacy = True + self.log.nosupport("Module path for default class is explicitly defined", '2.0') elif isinstance(exts_defaultclass, basestring): # proper way: derive module path from specified class name @@ -1336,47 +1453,29 @@ def extensions_step(self): default_class_modpath = get_module_path(default_class, generic=True) else: - self.log.error("Improper default extension class specification, should be list/tuple or string.") + raise EasyBuildError("Improper default extension class specification, should be list/tuple or string.") # get class instances for all extensions for ext in self.exts: self.log.debug("Starting extension %s" % ext['name']) - # always go back to build dir to avoid running stuff from a dir that no longer exists - os.chdir(self.builddir) + # always go back to original work dir to avoid running stuff from a dir that no longer exists + os.chdir(self.orig_workdir) - inst = None - - # try instantiating extension-specific class - class_name = encode_class_name(ext['name']) # use the same encoding as get_class + cls, inst = None, None + class_name = encode_class_name(ext['name']) mod_path = get_module_path(class_name) - if not os.path.exists("%s.py" % mod_path): - self.log.deprecated("Determine module path based on software name", "2.0") - mod_path = get_module_path(ext['name'], decode=False) + # try instantiating extension-specific class try: - cls = get_class_for(mod_path, class_name) - inst = cls(self, ext) - except (ImportError, NameError), err: - self.log.debug("Failed to use class %s from %s for extension %s: %s" % (class_name, - mod_path, - ext['name'], - err)) - - # LEGACY: try and use default module path for getting extension class instance - if inst is None and legacy: - try: - msg = "Failed to use derived module path for %s, " % class_name - msg += "considering specified module path as (legacy) fallback." - self.log.debug(msg) - mod_path = default_class_modpath - cls = get_class_for(mod_path, class_name) + # no error when importing class fails, in case we run into an existing easyblock + # with a similar name (e.g., Perl Extension 'GO' vs 'Go' for which 'EB_Go' is available) + cls = get_easyblock_class(None, name=ext['name'], default_fallback=False, error_on_failed_import=False) + self.log.debug("Obtained class %s for extension %s" % (cls, ext['name'])) + if cls is not None: inst = cls(self, ext) - except (ImportError, NameError), err: - self.log.debug("Failed to use class %s from %s for extension %s: %s" % (class_name, - mod_path, - ext['name'], - err)) + except (ImportError, NameError), err: + self.log.debug("Failed to use extension-specific class for extension %s: %s" % (ext['name'], err)) # alternative attempt: use class specified in class map (if any) if inst is None and ext['name'] in exts_classmap: @@ -1387,24 +1486,22 @@ def extensions_step(self): cls = get_class_for(mod_path, class_name) inst = cls(self, ext) except (ImportError, NameError), err: - self.log.error("Failed to load specified class %s for extension %s: %s" % (class_name, - ext['name'], - err)) + raise EasyBuildError("Failed to load specified class %s for extension %s: %s", + class_name, ext['name'], err) # fallback attempt: use default class - if not inst is None: - self.log.debug("Installing extension %s with class %s (from %s)" % (ext['name'], class_name, mod_path)) - else: + if inst is None: try: cls = get_class_for(default_class_modpath, default_class) self.log.debug("Obtained class %s for installing extension %s" % (cls, ext['name'])) inst = cls(self, ext) - tup = (ext['name'], default_class, default_class_modpath) - self.log.debug("Installing extension %s with default class %s (from %s)" % tup) + self.log.debug("Installing extension %s with default class %s (from %s)", + ext['name'], default_class, default_class_modpath) except (ImportError, NameError), err: - msg = "Also failed to use default class %s from %s for extension %s: %s, giving up" % \ - (default_class, default_class_modpath, ext['name'], err) - self.log.error(msg) + raise EasyBuildError("Also failed to use default class %s from %s for extension %s: %s, giving up", + default_class, default_class_modpath, ext['name'], err) + else: + self.log.debug("Installing extension %s with class %s (from %s)" % (ext['name'], class_name, mod_path)) # real work inst.prerun() @@ -1420,45 +1517,43 @@ def extensions_step(self): self.clean_up_fake_module(fake_mod_data) def package_step(self): - """Package software (e.g. into an RPM).""" - pass + """Package installed software (e.g., into an RPM), if requested, using selected package tool.""" + + if build_option('package'): + + pkgtype = build_option('package_type') + pkgdir_dest = os.path.abspath(package_path()) + opt_force = build_option('force') + + self.log.info("Generating %s package in %s", pkgtype, pkgdir_dest) + pkgdir_src = package(self) + + mkdir(pkgdir_dest) + + for src_file in glob.glob(os.path.join(pkgdir_src, "*.%s" % pkgtype)): + dest_file = os.path.join(pkgdir_dest, os.path.basename(src_file)) + if os.path.exists(dest_file) and not opt_force: + raise EasyBuildError("Unable to copy package %s to %s (already exists).", src_file, dest_file) + else: + self.log.info("Copied package %s to %s", src_file, pkgdir_dest) + shutil.copy(src_file, pkgdir_dest) + + else: + self.log.info("Skipping package step (not enabled)") def post_install_step(self): """ Do some postprocessing - run post install commands if any were specified - - set file permissions .... - Installing user must be member of the group that it is changed to """ if self.cfg['postinstallcmds'] is not None: # make sure we have a list of commands if not isinstance(self.cfg['postinstallcmds'], (list, tuple)): - self.log.error("Invalid value for 'postinstallcmds', should be list or tuple of strings.") + raise EasyBuildError("Invalid value for 'postinstallcmds', should be list or tuple of strings.") for cmd in self.cfg['postinstallcmds']: if not isinstance(cmd, basestring): - self.log.error("Invalid element in 'postinstallcmds', not a string: %s" % cmd) - run_cmd(cmd, simple=True, log_ok=False, log_all=False) - - if self.group is not None: - # remove permissions for others, and set group ID - try: - perms = stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH - adjust_permissions(self.installdir, perms, add=False, recursive=True, group_id=self.group[1], - relative=True, ignore_errors=True) - except EasyBuildError, err: - self.log.error("Unable to change group permissions of file(s): %s" % err) - self.log.info("Successfully made software only available for group %s (gid %s)" % self.group) - - if read_only_installdir(): - # remove write permissions for everyone - perms = stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH - adjust_permissions(self.installdir, perms, add=False, recursive=True, relative=True, ignore_errors=True) - self.log.info("Successfully removed write permissions recursively for *EVERYONE* on install dir.") - else: - # remove write permissions for group and other to protect installation - perms = stat.S_IWGRP | stat.S_IWOTH - adjust_permissions(self.installdir, perms, add=False, recursive=True, relative=True, ignore_errors=True) - self.log.info("Successfully removed write permissions recursively for group/other on install dir.") + raise EasyBuildError("Invalid element in 'postinstallcmds', not a string: %s", cmd) + run_cmd(cmd, simple=True, log_ok=True, log_all=True) def sanity_check_step(self, custom_paths=None, custom_commands=None, extension=False): """ @@ -1468,7 +1563,7 @@ def sanity_check_step(self, custom_paths=None, custom_commands=None, extension=F """ # supported/required keys in for sanity check paths, along with function used to check the paths path_keys_and_check = { - 'files': lambda fp: os.path.exists(fp), # files must exist + 'files': lambda fp: os.path.exists(fp) and not os.path.isdir(fp), # files must exist and not be a directory 'dirs': lambda dp: os.path.isdir(dp) and os.listdir(dp), # directories must exist and be non-empty } # prepare sanity check paths @@ -1492,19 +1587,20 @@ def sanity_check_step(self, custom_paths=None, custom_commands=None, extension=F lenvals = [len(x) for x in paths.values()] req_keys = sorted(path_keys_and_check.keys()) if not ks == req_keys or sum(valnottypes) > 0 or sum(lenvals) == 0: - self.log.error("Incorrect format for sanity_check_paths (should have %s keys, " - "values should be lists (at least one non-empty))." % '/'.join(req_keys)) + raise EasyBuildError("Incorrect format for sanity_check_paths (should (only) have %s keys, " + "values should be lists (at least one non-empty)).", ','.join(req_keys)) for key, check_fn in path_keys_and_check.items(): for xs in paths[key]: if isinstance(xs, basestring): xs = (xs,) elif not isinstance(xs, tuple): - self.log.error("Unsupported type '%s' encountered in %s, not a string or tuple" % (key, type(xs))) + raise EasyBuildError("Unsupported type '%s' encountered in %s, not a string or tuple", + key, type(xs)) found = False for name in xs: path = os.path.join(self.installdir, name) - if os.path.exists(path): + if check_fn(path): self.log.debug("Sanity check: found %s %s in %s" % (key[:-1], name, self.installdir)) found = True break @@ -1525,7 +1621,11 @@ def sanity_check_step(self, custom_paths=None, custom_commands=None, extension=F self.log.warning("Sanity check: %s" % self.sanity_check_fail_msgs[-1]) # chdir to installdir (better environment for running tests) - os.chdir(self.installdir) + if os.path.isdir(self.installdir): + try: + os.chdir(self.installdir) + except OSError, err: + raise EasyBuildError("Failed to move to installdir %s: %s", self.installdir, err) # run sanity check commands commands = self.cfg['sanity_check_commands'] @@ -1575,7 +1675,7 @@ def sanity_check_step(self, custom_paths=None, custom_commands=None, extension=F # pass or fail if self.sanity_check_fail_msgs: - self.log.error("Sanity check failed: %s" % ', '.join(self.sanity_check_fail_msgs)) + raise EasyBuildError("Sanity check failed: %s", ', '.join(self.sanity_check_fail_msgs)) else: self.log.debug("Sanity check passed!") @@ -1588,7 +1688,7 @@ def cleanup_step(self): """ if not self.build_in_installdir and build_option('cleanup_builddir'): try: - os.chdir(build_path()) # make sure we're out of the dir we're removing + os.chdir(self.orig_workdir) # make sure we're out of the dir we're removing self.log.info("Cleaning up builddir %s (in %s)" % (self.builddir, os.getcwd())) @@ -1602,7 +1702,7 @@ def cleanup_step(self): base = os.path.dirname(base) except OSError, err: - self.log.exception("Cleaning up builddir %s failed: %s" % (self.builddir, err)) + raise EasyBuildError("Cleaning up builddir %s failed: %s", self.builddir, err) if not build_option('cleanup_builddir'): self.log.info("Keeping builddir %s" % self.builddir) @@ -1613,36 +1713,71 @@ def make_module_step(self, fake=False): """ Generate a module file. """ - self.moduleGenerator.set_fake(fake) - modpath = self.moduleGenerator.prepare() + modpath = self.module_generator.prepare(fake=fake) - txt = '' - txt += self.make_module_description() - txt += self.make_module_extend_modpath() + txt = self.make_module_description() txt += self.make_module_dep() + txt += self.make_module_extend_modpath() txt += self.make_module_req() txt += self.make_module_extra() txt += self.make_module_footer() - write_file(self.moduleGenerator.filename, txt) + mod_filepath = self.module_generator.get_module_filepath(fake=fake) + write_file(mod_filepath, txt) - self.log.info("Added modulefile: %s" % (self.moduleGenerator.filename)) + self.log.info("Module file %s written: %s", mod_filepath, txt) - self.modules_tool.update() - self.moduleGenerator.create_symlinks() + # only update after generating final module file + if not fake: + self.modules_tool.update() + + mod_symlink_paths = ActiveMNS().det_module_symlink_paths(self.cfg) + self.module_generator.create_symlinks(mod_symlink_paths, fake=fake) if not fake: self.make_devel_module() return modpath + def permissions_step(self): + """ + Finalize installation procedure: adjust permissions as configured, change group ownership (if requested). + Installing user must be member of the group that it is changed to. + """ + if self.group is not None: + # remove permissions for others, and set group ID + try: + perms = stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH + adjust_permissions(self.installdir, perms, add=False, recursive=True, group_id=self.group[1], + relative=True, ignore_errors=True) + except EasyBuildError, err: + raise EasyBuildError("Unable to change group permissions of file(s): %s", err) + self.log.info("Successfully made software only available for group %s (gid %s)" % self.group) + + if build_option('read_only_installdir'): + # remove write permissions for everyone + perms = stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH + adjust_permissions(self.installdir, perms, add=False, recursive=True, relative=True, ignore_errors=True) + self.log.info("Successfully removed write permissions recursively for *EVERYONE* on install dir.") + + elif build_option('group_writable_installdir'): + # enable write permissions for group + perms = stat.S_IWGRP + adjust_permissions(self.installdir, perms, add=True, recursive=True, relative=True, ignore_errors=True) + self.log.info("Successfully enabled write permissions recursively for group on install dir.") + + else: + # remove write permissions for group and other + perms = stat.S_IWGRP | stat.S_IWOTH + adjust_permissions(self.installdir, perms, add=False, recursive=True, relative=True, ignore_errors=True) + self.log.info("Successfully removed write permissions recursively for group/other on install dir.") + def test_cases_step(self): """ Run provided test cases. """ for test in self.cfg['tests']: - # Current working dir no longer exists - os.chdir(self.installdir) + os.chdir(self.orig_workdir) if os.path.isabs(test): path = test else: @@ -1651,13 +1786,13 @@ def test_cases_step(self): if os.path.exists(path): break if not os.path.exists(path): - self.log.error("Test specifies invalid path: %s" % path) + raise EasyBuildError("Test specifies invalid path: %s", path) try: self.log.debug("Running test %s" % path) run_cmd(path, log_all=True, simple=True) except EasyBuildError, err: - self.log.exception("Running test %s failed: %s" % (path, err)) + raise EasyBuildError("Running test %s failed: %s", path, err) def update_config_template_run_step(self): """Update the the easyconfig template dictionary with easyconfig.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP names""" @@ -1666,23 +1801,47 @@ def update_config_template_run_step(self): self.cfg.template_values[name[0]] = str(getattr(self, name[0], None)) self.cfg.generate_template_values() - def run_step(self, step, methods, skippable=False): - """ - Run step, returns false when execution should be stopped - """ + def _skip_step(self, step, skippable): + """Dedice whether or not to skip the specified step.""" + module_only = build_option('module_only') + force = build_option('force') + skip = False + + # skip step if specified as individual (skippable) step if skippable and (self.skip or step in self.cfg['skipsteps']): - self.log.info("Skipping %s step" % step) + self.log.info("Skipping %s step (skip: %s, skipsteps: %s)", step, self.skip, self.cfg['skipsteps']) + skip = True + + # skip step when only generating module file + # * still run sanity check without use of force + # * always run ready & prepare step to set up toolchain + deps + elif module_only and not step in MODULE_ONLY_STEPS: + self.log.info("Skipping %s step (only generating module)", step) + skip = True + + # allow skipping sanity check too when only generating module and force is used + elif module_only and step == SANITYCHECK_STEP and force: + self.log.info("Skipping %s step because of forced module-only mode", step) + skip = True + else: - self.log.info("Starting %s step" % step) - # update the config templates - self.update_config_template_run_step() + self.log.debug("Not skipping %s step (skippable: %s, skip: %s, skipsteps: %s, module_only: %s, force: %s", + step, skippable, self.skip, self.cfg['skipsteps'], module_only, force) - for m in methods: - self.log.info("Running method %s part of step %s" % ('_'.join(m.func_code.co_names), step)) - m(self) + return skip + + def run_step(self, step, methods): + """ + Run step, returns false when execution should be stopped + """ + self.log.info("Starting %s step", step) + self.update_config_template_run_step() + for m in methods: + self.log.info("Running method %s part of step %s" % ('_'.join(m.func_code.co_names), step)) + m(self) if self.cfg['stop'] == step: - self.log.info("Stopping after %s step." % step) + self.log.info("Stopping after %s step.", step) raise StopException(step) @staticmethod @@ -1697,20 +1856,18 @@ def get_step(tag, descr, substeps, skippable, initial=True): # list of substeps for steps that are slightly different from 2nd iteration onwards ready_substeps = [ (False, lambda x: x.check_readiness_step()), - (False, lambda x: x.gen_builddir()), - (False, lambda x: x.gen_installdir()), (True, lambda x: x.make_builddir()), (True, lambda x: env.reset_changes()), (True, lambda x: x.handle_iterate_opts()), ] - ready_step_spec = lambda initial: get_step('ready', "creating build dir, resetting environment", + ready_step_spec = lambda initial: get_step(READY_STEP, "creating build dir, resetting environment", ready_substeps, False, initial=initial) source_substeps = [ (False, lambda x: x.checksum_step()), (True, lambda x: x.extract_step()), ] - source_step_spec = lambda initial: get_step('source', "unpacking", source_substeps, True, initial=initial) + source_step_spec = lambda initial: get_step(SOURCE_STEP, "unpacking", source_substeps, True, initial=initial) def prepare_step_spec(initial): """Return prepare step specification.""" @@ -1718,7 +1875,7 @@ def prepare_step_spec(initial): substeps = [lambda x: x.prepare_step()] else: substeps = [lambda x: x.guess_start_dir()] - return ('prepare', 'preparing', substeps, False) + return (PREPARE_STEP, 'preparing', substeps, False) install_substeps = [ (False, lambda x: x.stage_install_step()), @@ -1730,14 +1887,14 @@ def prepare_step_spec(initial): # format for step specifications: (stop_name: (description, list of functions, skippable)) # core steps that are part of the iterated loop - patch_step_spec = ('patch', 'patching', [lambda x: x.patch_step()], True) - configure_step_spec = ('configure', 'configuring', [lambda x: x.configure_step()], True) - build_step_spec = ('build', 'building', [lambda x: x.build_step()], True) - test_step_spec = ('test', 'testing', [lambda x: x.test_step()], True) + patch_step_spec = (PATCH_STEP, 'patching', [lambda x: x.patch_step()], True) + configure_step_spec = (CONFIGURE_STEP, 'configuring', [lambda x: x.configure_step()], True) + build_step_spec = (BUILD_STEP, 'building', [lambda x: x.build_step()], True) + test_step_spec = (TEST_STEP, 'testing', [lambda x: x.test_step()], True) # part 1: pre-iteration + first iteration steps_part1 = [ - ('fetch', 'fetching files', [lambda x: x.fetch_step()], False), + (FETCH_STEP, 'fetching files', [lambda x: x.fetch_step()], False), ready_step_spec(True), source_step_spec(True), patch_step_spec, @@ -1762,19 +1919,20 @@ def prepare_step_spec(initial): ] * (iteration_count - 1) # part 3: post-iteration part steps_part3 = [ - ('extensions', 'taking care of extensions', [lambda x: x.extensions_step()], False), - ('package', 'packaging', [lambda x: x.package_step()], True), - ('postproc', 'postprocessing', [lambda x: x.post_install_step()], True), - ('sanitycheck', 'sanity checking', [lambda x: x.sanity_check_step()], False), - ('cleanup', 'cleaning up', [lambda x: x.cleanup_step()], False), - ('module', 'creating module', [lambda x: x.make_module_step()], False), + (EXTENSIONS_STEP, 'taking care of extensions', [lambda x: x.extensions_step()], False), + (POSTPROC_STEP, 'postprocessing', [lambda x: x.post_install_step()], True), + (SANITYCHECK_STEP, 'sanity checking', [lambda x: x.sanity_check_step()], False), + (CLEANUP_STEP, 'cleaning up', [lambda x: x.cleanup_step()], False), + (MODULE_STEP, 'creating module', [lambda x: x.make_module_step()], False), + (PERMISSIONS_STEP, 'permissions', [lambda x: x.permissions_step()], False), + (PACKAGE_STEP, 'packaging', [lambda x: x.package_step()], False), ] # full list of steps, included iterated steps steps = steps_part1 + steps_part2 + steps_part3 if run_test_cases: - steps.append(('testcases', 'running test cases', [ + steps.append((TESTCASES_STEP, 'running test cases', [ lambda x: x.load_module(), lambda x: x.test_cases_step(), ], False)) @@ -1793,9 +1951,12 @@ def run_all_steps(self, run_test_cases): print_msg("building and installing %s..." % self.full_mod_name, self.log, silent=self.silent) try: - for (stop_name, descr, step_methods, skippable) in steps: - print_msg("%s..." % descr, self.log, silent=self.silent) - self.run_step(stop_name, step_methods, skippable=skippable) + for (step_name, descr, step_methods, skippable) in steps: + if self._skip_step(step_name, skippable): + print_msg("%s [skipped]" % descr, self.log, silent=self.silent) + else: + print_msg("%s..." % descr, self.log, silent=self.silent) + self.run_step(step_name, step_methods) except StopException: pass @@ -1804,38 +1965,39 @@ def run_all_steps(self, run_test_cases): return True -def build_and_install_one(module, orig_environ): +def build_and_install_one(ecdict, init_env): """ Build the software - @param module: dictionary contaning parsed easyconfig + metadata - @param orig_environ: original environment (used to reset environment) + @param ecdict: dictionary contaning parsed easyconfig + metadata + @param init_env: original environment (used to reset environment) """ silent = build_option('silent') - spec = module['spec'] + spec = ecdict['spec'] + rawtxt = ecdict['ec'].rawtxt + name = ecdict['ec']['name'] print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent) # restore original environment _log.info("Resetting environment") filetools.errors_found_in_log = 0 - modify_env(os.environ, orig_environ) + restore_env(init_env) cwd = os.getcwd() # load easyblock easyblock = build_option('easyblock') if not easyblock: - easyblock = fetch_parameter_from_easyconfig_file(spec, 'easyblock') + easyblock = fetch_parameters_from_easyconfig(rawtxt, ['easyblock'])[0] - name = module['ec']['name'] try: app_class = get_easyblock_class(easyblock, name=name) - app = app_class(module['ec']) + app = app_class(ecdict['ec']) _log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock)) except EasyBuildError, err: - tup = (name, easyblock, err.msg) - print_error("Failed to get application instance for %s (easyblock: %s): %s" % tup, silent=silent) + print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg), + silent=silent) # application settings stop = build_option('stop') @@ -1877,6 +2039,9 @@ def build_and_install_one(module, orig_environ): new_log_dir = os.path.dirname(app.logfile) else: new_log_dir = os.path.join(app.installdir, config.log_path()) + if build_option('read_only_installdir'): + # temporarily re-enable write permissions for copying log/easyconfig to install dir + adjust_permissions(new_log_dir, stat.S_IWUSR, add=True, recursive=False) # collect build stats _log.info("Collecting build stats...") @@ -1888,9 +2053,9 @@ def build_and_install_one(module, orig_environ): # upload spec to central repository currentbuildstats = app.cfg['buildstats'] repo = init_repository(get_repository(), get_repositorypath()) - if 'original_spec' in module: + if 'original_spec' in ecdict: block = det_full_ec_version(app.cfg) + ".block" - repo.add_easyconfig(module['original_spec'], app.name, block, buildstats, currentbuildstats) + repo.add_easyconfig(ecdict['original_spec'], app.name, block, buildstats, currentbuildstats) repo.add_easyconfig(spec, app.name, det_full_ec_version(app.cfg), buildstats, currentbuildstats) repo.commit("Built %s" % app.full_mod_name) del repo @@ -1903,21 +2068,24 @@ def build_and_install_one(module, orig_environ): # cleanup logs app.close_log() - try: - mkdir(new_log_dir, parents=True) - log_fn = os.path.basename(get_log_filename(app.name, app.version)) - application_log = os.path.join(new_log_dir, log_fn) - shutil.move(app.logfile, application_log) - _log.debug("Moved log file %s to %s" % (app.logfile, application_log)) - except (IOError, OSError), err: - print_error("Failed to move log file %s to new log file %s: %s" % (app.logfile, application_log, err)) + log_fn = os.path.basename(get_log_filename(app.name, app.version)) + application_log = os.path.join(new_log_dir, log_fn) + move_logs(app.logfile, application_log) try: newspec = os.path.join(new_log_dir, "%s-%s.eb" % (app.name, det_full_ec_version(app.cfg))) - shutil.copy(spec, newspec) - _log.debug("Copied easyconfig file %s to %s" % (spec, newspec)) + # only copy if the files are not the same file already (yes, it happens) + if os.path.exists(newspec) and os.path.samefile(spec, newspec): + _log.debug("Not copying easyconfig file %s to %s since files are identical" % (spec, newspec)) + else: + shutil.copy(spec, newspec) + _log.debug("Copied easyconfig file %s to %s" % (spec, newspec)) except (IOError, OSError), err: - print_error("Failed to move easyconfig %s to log dir %s: %s" % (spec, new_log_dir, err)) + print_error("Failed to copy easyconfig %s to %s: %s" % (spec, newspec, err)) + + if build_option('read_only_installdir'): + # take away user write permissions (again) + adjust_permissions(new_log_dir, stat.S_IWUSR, add=False, recursive=False) # build failed else: @@ -1950,22 +2118,24 @@ def build_and_install_one(module, orig_environ): return (success, application_log, errormsg) -def get_easyblock_instance(easyconfig): + +def get_easyblock_instance(ecdict): """ Get an instance for this easyconfig @param easyconfig: parsed easyconfig (EasyConfig instance) returns an instance of EasyBlock (or subclass thereof) """ - spec = easyconfig['spec'] - name = easyconfig['ec']['name'] + spec = ecdict['spec'] + rawtxt = ecdict['ec'].rawtxt + name = ecdict['ec']['name'] # handle easyconfigs with custom easyblocks # determine easyblock specification from easyconfig file, if any - easyblock = fetch_parameter_from_easyconfig_file(spec, 'easyblock') + easyblock = fetch_parameters_from_easyconfig(rawtxt, ['easyblock'])[0] app_class = get_easyblock_class(easyblock, name=name) - return app_class(easyconfig['ec']) + return app_class(ecdict['ec']) def build_easyconfigs(easyconfigs, output_dir, test_results): @@ -2008,6 +2178,10 @@ def perform_step(step, obj, method, logfile): apps.append(instance) base_dir = os.getcwd() + + # keep track of environment right before initiating builds + # note: may be different from ORIG_OS_ENVIRON, since EasyBuild may have defined additional env vars itself by now + # e.g. via easyconfig.handle_allowed_system_deps base_env = copy.deepcopy(os.environ) succes = [] @@ -2022,7 +2196,7 @@ def perform_step(step, obj, method, logfile): # start with a clean slate os.chdir(base_dir) - modify_env(os.environ, base_env) + restore_env(base_env) steps = EasyBlock.get_steps(iteration_count=app.det_iter_cnt()) @@ -2036,21 +2210,7 @@ def perform_step(step, obj, method, logfile): # close log and move it app.close_log() - try: - # retain old logs - if os.path.exists(applog): - i = 0 - old_applog = "%s.%d" % (applog, i) - while os.path.exists(old_applog): - i += 1 - old_applog = "%s.%d" % (applog, i) - shutil.move(applog, old_applog) - _log.info("Moved existing log file %s to %s" % (applog, old_applog)) - - shutil.move(app.logfile, applog) - _log.info("Log file moved to %s" % applog) - except IOError, err: - print_error("Failed to move log file %s to new log file %s: %s" % (app.logfile, applog, err)) + move_logs(app.logfile, applog) if app not in build_stopped: # gather build stats diff --git a/easybuild/framework/easyconfig/__init__.py b/easybuild/framework/easyconfig/__init__.py index b8f643bab3..906b4eb0d3 100644 --- a/easybuild/framework/easyconfig/__init__.py +++ b/easybuild/framework/easyconfig/__init__.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -31,5 +31,8 @@ from easybuild.framework.easyconfig.default import ALL_CATEGORIES globals().update(ALL_CATEGORIES) +# subdirectory (of 'easybuild' dir) in which easyconfig files are located in a package +EASYCONFIGS_PKG_SUBDIR = 'easyconfigs' + # is used in some tools from easybuild.framework.easyconfig.easyconfig import EasyConfig diff --git a/easybuild/framework/easyconfig/constants.py b/easybuild/framework/easyconfig/constants.py index 7183c0c9fb..501db4dcca 100644 --- a/easybuild/framework/easyconfig/constants.py +++ b/easybuild/framework/easyconfig/constants.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,8 +37,12 @@ _log = fancylogger.getLogger('easyconfig.constants', fname=False) + +EXTERNAL_MODULE_MARKER = 'EXTERNAL_MODULE' + # constants that can be used in easyconfig EASYCONFIG_CONSTANTS = { + 'EXTERNAL_MODULE': (EXTERNAL_MODULE_MARKER, "External module marker"), 'SYS_PYTHON_VERSION': (platform.python_version(), "System Python version (platform.python_version())"), 'OS_TYPE': (get_os_type(), "System type (e.g. 'Linux' or 'Darwin')"), 'OS_NAME': (get_os_name(), "System name (e.g. 'fedora' or 'RHEL')"), diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 20640e0af7..e69e646b81 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -35,37 +35,30 @@ """ from vsc.utils import fancylogger -from easybuild.tools.ordereddict import OrderedDict +from easybuild.tools.build_log import EasyBuildError + _log = fancylogger.getLogger('easyconfig.default', fname=False) # we use a tuple here so we can sort them based on the numbers -HIDDEN = "HIDDEN" -MANDATORY = "MANDATORY" -CUSTOM = "CUSTOM" -TOOLCHAIN = "TOOLCHAIN" -BUILD = "BUILD" -FILEMANAGEMENT = "FILEMANAGEMENT" -DEPENDENCIES = "DEPENDENCIES" -LICENSE = "LICENSE" -EXTENSIONS = "EXTENSIONS" -MODULES = "MODULES" -OTHER = "OTHER" - ALL_CATEGORIES = { - HIDDEN: (-1, 'hidden'), - MANDATORY: (0, 'mandatory'), - CUSTOM: (1, 'easyblock-specific'), - TOOLCHAIN: (2, 'toolchain'), - BUILD: (3, 'build'), - FILEMANAGEMENT: (4, 'file-management'), - DEPENDENCIES: (5, 'dependencies'), - LICENSE: (6, 'license'), - EXTENSIONS: (7, 'extensions'), - MODULES: (8, 'modules'), - OTHER: (9, 'other'), + 'HIDDEN': (-1, 'hidden'), + 'MANDATORY': (0, 'mandatory'), + 'CUSTOM': (1, 'easyblock-specific'), + 'TOOLCHAIN': (2, 'toolchain'), + 'BUILD': (3, 'build'), + 'FILEMANAGEMENT': (4, 'file-management'), + 'DEPENDENCIES': (5, 'dependencies'), + 'LICENSE': (6, 'license'), + 'EXTENSIONS': (7, 'extensions'), + 'MODULES': (8, 'modules'), + 'OTHER': (9, 'other'), } +# define constants so they can be used below +# avoid that pylint complains about unknown variables in this file +# pylint: disable=E0602 +globals().update(ALL_CATEGORIES) # List of tuples. Each tuple has the following format (key, [default, help text, category]) DEFAULT_CONFIG = { @@ -92,7 +85,8 @@ 'buildopts': ['', 'Extra options passed to make step (default already has -j X)', BUILD], 'checksums': [[], "Checksums for sources and patches", BUILD], 'configopts': ['', 'Extra options passed to configure (default already has --prefix)', BUILD], - 'easyblock': ['ConfigureMake', "EasyBlock to use for building", BUILD], + 'easyblock': [None, "EasyBlock to use for building; if set to None, an easyblock is selected " + "based on the software name", BUILD], 'easybuild_version': [None, "EasyBuild-version this spec-file was written for", BUILD], 'installopts': ['', 'Extra options for installation', BUILD], 'maxparallel': [None, 'Max degree of parallelism', BUILD], @@ -116,7 +110,7 @@ 'stop': [None, 'Keyword to halt the build process after a certain step.', BUILD], 'tests': [[], ("List of test-scripts to run after install. A test script should return a " "non-zero exit status to fail"), BUILD], - 'unpack_options': [None, "Extra options for unpacking source", BUILD], + 'unpack_options': ['', "Extra options for unpacking source", BUILD], 'unwanted_env_vars': [[], "List of environment variables that shouldn't be set during build", BUILD], 'versionprefix': ['', ('Additional prefix for software version ' '(placed before version and toolchain name)'), BUILD], @@ -143,6 +137,7 @@ 'allow_system_deps': [[], "Allow listed system dependencies (format: (, ))", DEPENDENCIES], 'builddependencies': [[], "List of build dependencies", DEPENDENCIES], 'dependencies': [[], "List of dependencies", DEPENDENCIES], + 'hiddendependencies': [[], "List of dependencies available as hidden modules", DEPENDENCIES], 'osdependencies': [[], "OS dependencies that should be present on the system", DEPENDENCIES], # LICENSE easyconfig parameters @@ -160,14 +155,18 @@ 'exts_list': [[], 'List with extensions added to the base installation', EXTENSIONS], # MODULES easyconfig parameters + 'modaliases': [{}, "Aliases to be defined in module file", MODULES], 'modextrapaths': [{}, "Extra paths to be prepended in module file", MODULES], 'modextravars': [{}, "Extra environment variables to be added to module file", MODULES], 'modloadmsg': [{}, "Message that should be printed when generated module is loaded", MODULES], + 'modluafooter': ["", "Footer to include in generated module file (Lua syntax)", MODULES], + 'modaltsoftname': [None, "Module name to use (rather than using software name", MODULES], 'modtclfooter': ["", "Footer to include in generated module file (Tcl syntax)", MODULES], - 'modaliases': [{}, "Aliases to be defined in module file", MODULES], 'moduleclass': ['base', 'Module class to be used for this software', MODULES], 'moduleforceunload': [False, 'Force unload of all modules when loading the extension', MODULES], 'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES], + 'include_modpath_extensions': [True, "Include $MODULEPATH extensions specified by module naming scheme.", MODULES], + 'recursive_module_unload': [False, 'Recursive unload of all dependencies when unloading module', MODULES], # OTHER easyconfig parameters 'buildstats': [None, "A list of dicts with build statistics", OTHER], @@ -183,32 +182,10 @@ def sorted_categories(): return categories -def convert_to_help(opts, has_default=False): - """ - Converts the given list to a mapping of category -> [(name, help)] (OrderedDict) - @param: has_default, if False, add the DEFAULT_CONFIG list - """ - mapping = OrderedDict() - if isinstance(opts, dict): - opts = opts.items() - if not has_default: - defs = [(k, [def_val, descr, ALL_CATEGORIES[cat]]) for k, (def_val, descr, cat) in DEFAULT_CONFIG.items()] - opts = defs + opts - - # sort opts - opts.sort() - - for cat in sorted_categories(): - mapping[cat[1]] = [(opt[0], "%s (default: %s)" % (opt[1][1], opt[1][0])) - for opt in opts if opt[1][2] == cat] - - return mapping - - def get_easyconfig_parameter_default(param): """Get default value for given easyconfig parameter.""" if param not in DEFAULT_CONFIG: - _log.error("Unkown easyconfig parameter: %s (known: %s)" % (param, sorted(DEFAULT_CONFIG.keys()))) + raise EasyBuildError("Unkown easyconfig parameter: %s (known: %s)", param, sorted(DEFAULT_CONFIG.keys())) else: _log.debug("Returning default value for easyconfig parameter %s: %s" % (param, DEFAULT_CONFIG[param][0])) return DEFAULT_CONFIG[param][0] diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 45caac97cc..227b0b1f6e 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -40,83 +40,65 @@ import os import re from vsc.utils import fancylogger -from vsc.utils.missing import any, nub +from vsc.utils.missing import get_class_for, nub from vsc.utils.patterns import Singleton import easybuild.tools.environment as env from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, get_module_naming_scheme -from easybuild.tools.filetools import decode_class_name, encode_class_name, read_file +from easybuild.tools.filetools import decode_class_name, encode_class_name, read_file, write_file from easybuild.tools.module_naming_scheme import DEVEL_MODULE_SUFFIX from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version -from easybuild.tools.module_naming_scheme.utilities import is_valid_module_name +from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name +from easybuild.tools.ordereddict import OrderedDict from easybuild.tools.systemtools import check_os_dependency from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME, DUMMY_TOOLCHAIN_VERSION from easybuild.tools.toolchain.utilities import get_toolchain -from easybuild.tools.utilities import remove_unwanted_chars +from easybuild.tools.utilities import quote_py_str, quote_str, remove_unwanted_chars from easybuild.framework.easyconfig import MANDATORY -from easybuild.framework.easyconfig.default import DEFAULT_CONFIG, ALL_CATEGORIES, get_easyconfig_parameter_default +from easybuild.framework.easyconfig.constants import EXTERNAL_MODULE_MARKER +from easybuild.framework.easyconfig.default import DEFAULT_CONFIG from easybuild.framework.easyconfig.format.convert import Dependency from easybuild.framework.easyconfig.format.one import retrieve_blocks_in_spec from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT, License -from easybuild.framework.easyconfig.parser import EasyConfigParser -from easybuild.framework.easyconfig.templates import template_constant_dict +from easybuild.framework.easyconfig.parser import DEPRECATED_PARAMETERS, REPLACED_PARAMETERS +from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig +from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, template_constant_dict _log = fancylogger.getLogger('easyconfig.easyconfig', fname=False) - # add license here to make it really MANDATORY (remove comment in default) -_log.deprecated('Mandatory license not enforced', '2.0') MANDATORY_PARAMS = ['name', 'version', 'homepage', 'description', 'toolchain'] # set of configure/build/install options that can be provided as lists for an iterated build ITERATE_OPTIONS = ['preconfigopts', 'configopts', 'prebuildopts', 'buildopts', 'preinstallopts', 'installopts'] -# map of deprecated easyconfig parameters, and their replacements -DEPRECATED_OPTIONS = { - 'license': ('software_license', '2.0'), - 'makeopts': ('buildopts', '2.0'), - 'premakeopts': ('prebuildopts', '2.0'), -} + +try: + import autopep8 + HAVE_AUTOPEP8 = True +except ImportError as err: + _log.warning("Failed to import autopep8, dumping easyconfigs with reformatting enabled will not work: %s", err) + HAVE_AUTOPEP8 = False + _easyconfig_files_cache = {} _easyconfigs_cache = {} -def handle_deprecated_easyconfig_parameter(ec_method): - """Decorator to handle deprecated easyconfig parameters.""" +def handle_deprecated_or_replaced_easyconfig_parameters(ec_method): + """Decorator to handle deprecated/replaced easyconfig parameters.""" def new_ec_method(self, key, *args, **kwargs): - """Map deprecated easyconfig parameters to the new correct parameter.""" - # map name of deprecated easyconfig parameter to new name - if key in DEPRECATED_OPTIONS: + """Check whether any replace easyconfig parameters are still used""" + # map deprecated parameters to their replacements, issue deprecation warning(/error) + if key in DEPRECATED_PARAMETERS: depr_key = key - key, ver = DEPRECATED_OPTIONS[depr_key] + key, ver = DEPRECATED_PARAMETERS[depr_key] _log.deprecated("Easyconfig parameter '%s' is deprecated, use '%s' instead." % (depr_key, key), ver) - - # make sure that value for software_license has correct type, convert if needed - if key == 'software_license': - # key 'license' will already be mapped to 'software_license' above - lic = self._config['software_license'] - if not isinstance(lic, License): - self.log.deprecated('Type for software_license must to be instance of License (sub)class', '2.0') - lic_type = type(lic) - - class LicenseLegacy(License, lic_type): - """A special License class to deal with legacy license paramters""" - DESCRICPTION = ("Internal-only, legacy closed license class to deprecate license parameter." - " (DO NOT USE).") - HIDDEN = False - - def __init__(self, *args): - if len(args) > 0: - lic_type.__init__(self, args[0]) - License.__init__(self) - lic = LicenseLegacy(lic) - EASYCONFIG_LICENSES_DICT[lic.name] = lic - self._config['software_license'] = lic - + if key in REPLACED_PARAMETERS: + _log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0') return ec_method(self, key, *args, **kwargs) return new_ec_method @@ -127,62 +109,61 @@ class EasyConfig(object): Class which handles loading, reading, validation of easyconfigs """ - def __init__(self, path, extra_options=None, build_specs=None, validate=True): + def __init__(self, path, extra_options=None, build_specs=None, validate=True, hidden=None, rawtxt=None, + auto_convert_value_types=True): """ initialize an easyconfig. - @param path: path to easyconfig file to be parsed + @param path: path to easyconfig file to be parsed (ignored if rawtxt is specified) @param extra_options: dictionary with extra variables that can be set for this specific instance @param build_specs: dictionary of build specifications (see EasyConfig class, default: {}) @param validate: indicates whether validation should be performed (note: combined with 'validate' build option) + @param hidden: indicate whether corresponding module file should be installed hidden ('.'-prefixed) + @param rawtxt: raw contents of easyconfig file + @param auto_convert_value_types: indicates wether types of easyconfig values should be automatically converted + in case they are wrong """ self.template_values = None self.enable_templating = True # a boolean to control templating self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) - if not os.path.isfile(path): - self.log.error("EasyConfig __init__ expected a valid path") + if path is not None and not os.path.isfile(path): + raise EasyBuildError("EasyConfig __init__ expected a valid path") + + # read easyconfig file contents (or use provided rawtxt), so it can be passed down to avoid multiple re-reads + self.path = None + if rawtxt is None: + self.path = path + self.rawtxt = read_file(path) + self.log.debug("Raw contents from supplied easyconfig file %s: %s" % (path, self.rawtxt)) + else: + self.rawtxt = rawtxt + self.log.debug("Supplied raw easyconfig contents: %s" % self.rawtxt) # use legacy module classes as default self.valid_module_classes = build_option('valid_module_classes') if self.valid_module_classes is not None: self.log.info("Obtained list of valid module classes: %s" % self.valid_module_classes) - # replace the category name with the category - self._config = {} - for k, [def_val, descr, cat] in copy.deepcopy(DEFAULT_CONFIG).items(): - self._config[k] = [def_val, descr, ALL_CATEGORIES[cat]] + self._config = copy.deepcopy(DEFAULT_CONFIG) + # obtain name and easyblock specifications from raw easyconfig contents + self.software_name, self.easyblock = fetch_parameters_from_easyconfig(self.rawtxt, ['name', 'easyblock']) + + # determine line of extra easyconfig parameters if extra_options is None: - name = fetch_parameter_from_easyconfig_file(path, 'name') - easyblock = fetch_parameter_from_easyconfig_file(path, 'easyblock') - app_class = get_easyblock_class(easyblock, name=name) - self.extra_options = app_class.extra_options() + easyblock_class = get_easyblock_class(self.easyblock, name=self.software_name) + self.extra_options = easyblock_class.extra_options() else: self.extra_options = extra_options if not isinstance(self.extra_options, dict): - if isinstance(self.extra_options, (list, tuple,)): - typ = type(self.extra_options) - self.log.deprecated("Specified extra_options should be of type 'dict', found type '%s'" % typ, '2.0') - tup = (self.extra_options, type(self.extra_options)) - self.log.debug("Converting extra_options value '%s' of type '%s' to a dict" % tup) - self.extra_options = dict(self.extra_options) - else: - tup = (type(self.extra_options), self.extra_options) - self.log.error("extra_options parameter passed is of incorrect type: %s ('%s')" % tup) - - # map deprecated params to new names if they occur in extra_options - for key, val in self.extra_options.items(): - if key in DEPRECATED_OPTIONS: - new_key, depr_ver = DEPRECATED_OPTIONS[key] - self.log.deprecated("Found deprecated key '%s', should use '%s' instead." % (key, new_key), depr_ver) - self.extra_options[new_key] = self.extra_options[key] - self.log.debug("Set '%s' with value of deprecated '%s': %s" % (new_key, key, self.extra_options[key])) - del self.extra_options[key] - self._config.update(self.extra_options) - - self.path = path + tup = (type(self.extra_options), self.extra_options) + self.log.nosupport("extra_options return value should be of type 'dict', found '%s': %s" % tup, '2.0') + + # deep copy to make sure self.extra_options remains unchanged + self._config.update(copy.deepcopy(self.extra_options)) + self.mandatory = MANDATORY_PARAMS[:] # extend mandatory keys @@ -202,8 +183,12 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True): 'stop': self.valid_stops, } + self.external_modules_metadata = build_option('external_modules_metadata') + # parse easyconfig file self.build_specs = build_specs + self.parser = EasyConfigParser(filename=self.path, rawcontent=self.rawtxt, + auto_convert_value_types=auto_convert_value_types) self.parse() # handle allowed system dependencies @@ -214,18 +199,33 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True): if self.validation: self.validate(check_osdeps=build_option('check_osdeps')) - # set module info + # filter hidden dependencies from list of dependencies + self.filter_hidden_deps() + + # list of *all* dependencies, including hidden/build deps & toolchain, but excluding filtered deps + self.all_dependencies = copy.deepcopy(self.dependencies()) + if self.toolchain.name != DUMMY_TOOLCHAIN_NAME: + self.all_dependencies.append(self.toolchain.as_dict()) + + # keep track of whether the generated module file should be hidden + if hidden is None: + hidden = build_option('hidden') + self.hidden = hidden + + # set installdir/module info mns = ActiveMNS() self.full_mod_name = mns.det_full_module_name(self) self.short_mod_name = mns.det_short_module_name(self) self.mod_subdir = mns.det_module_subdir(self) + self.software_license = None + def copy(self): """ Return a copy of this EasyConfig instance. """ # create a new EasyConfig instance - ec = EasyConfig(self.path, validate=self.validation) + ec = EasyConfig(self.path, validate=self.validation, hidden=self.hidden, rawtxt=self.rawtxt) # take a copy of the actual config dictionary (which already contains the extra options) ec._config = copy.deepcopy(self._config) @@ -241,7 +241,7 @@ def update(self, key, value): elif isinstance(prev_value, list): self[key] = prev_value + value else: - self.log.error("Can't update configuration value for %s, because it's not a string or list." % key) + raise EasyBuildError("Can't update configuration value for %s, because it's not a string or list.", key) def parse(self): """ @@ -254,40 +254,44 @@ def parse(self): # build a new dictionary with only the expected keys, to pass as named arguments to get_config_dict() arg_specs = self.build_specs else: - self.log.error("Specifications should be specified using a dictionary, got %s" % type(self.build_specs)) + raise EasyBuildError("Specifications should be specified using a dictionary, got %s", + type(self.build_specs)) self.log.debug("Obtained specs dict %s" % arg_specs) - parser = EasyConfigParser(self.path) - parser.set_specifications(arg_specs) - local_vars = parser.get_config_dict() + self.log.info("Parsing easyconfig file %s with rawcontent: %s" % (self.path, self.rawtxt)) + self.parser.set_specifications(arg_specs) + local_vars = self.parser.get_config_dict() self.log.debug("Parsed easyconfig as a dictionary: %s" % local_vars) - # validate mandatory keys - # TODO: remove this code. this is now (also) checked in the format (see validate_pyheader) - missing_keys = [key for key in self.mandatory if key not in local_vars] - if missing_keys: - self.log.error("mandatory variables %s not provided in %s" % (missing_keys, self.path)) + # make sure all mandatory parameters are defined + # this includes both generic mandatory parameters and software-specific parameters defined via extra_options + missing_mandatory_keys = [key for key in self.mandatory if key not in local_vars] + if missing_mandatory_keys: + raise EasyBuildError("mandatory parameters not provided in %s: %s", self.path, missing_mandatory_keys) # provide suggestions for typos possible_typos = [(key, difflib.get_close_matches(key.lower(), self._config.keys(), 1, 0.85)) - for key in local_vars if key not in self._config] + for key in local_vars if key not in self] typos = [(key, guesses[0]) for (key, guesses) in possible_typos if len(guesses) == 1] if typos: - self.log.error("You may have some typos in your easyconfig file: %s" % - ', '.join(["%s -> %s" % typo for typo in typos])) + raise EasyBuildError("You may have some typos in your easyconfig file: %s", + ', '.join(["%s -> %s" % typo for typo in typos])) # we need toolchain to be set when we call _parse_dependency for key in ['toolchain'] + local_vars.keys(): # validations are skipped, just set in the config # do not store variables we don't need - if key in self._config.keys() + DEPRECATED_OPTIONS.keys(): + if key in self._config.keys(): if key in ['builddependencies', 'dependencies']: self[key] = [self._parse_dependency(dep) for dep in local_vars[key]] + elif key in ['hiddendependencies']: + self[key] = [self._parse_dependency(dep, hidden=True) for dep in local_vars[key]] else: self[key] = local_vars[key] - tup = (key, self[key], type(self[key])) - self.log.info("setting config option %s: value %s (type: %s)" % tup) + self.log.info("setting config option %s: value %s (type: %s)", key, self[key], type(self[key])) + elif key in REPLACED_PARAMETERS: + _log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0') else: self.log.debug("Ignoring unknown config option %s (value: %s)" % (key, local_vars[key])) @@ -301,16 +305,19 @@ def parse(self): def handle_allowed_system_deps(self): """Handle allowed system dependencies.""" for (name, version) in self['allow_system_deps']: - env.setvar(get_software_root_env_var_name(name), name) # root is set to name, not an actual path - env.setvar(get_software_version_env_var_name(name), version) # version is expected to be something that makes sense + # root is set to name, not an actual path + env.setvar(get_software_root_env_var_name(name), name) + # version is expected to be something that makes sense + env.setvar(get_software_version_env_var_name(name), version) def validate(self, check_osdeps=True): """ - Validate this EasyConfig - - check certain variables - TODO: move more into here + Validate this easyonfig + - ensure certain easyconfig parameters are set to a known value (see self.validations) + - check OS dependencies + - check license """ - self.log.info("Validating easy block") + self.log.info("Validating easyconfig") for attr in self.validations: self._validate(attr, self.validations[attr]) @@ -322,8 +329,8 @@ def validate(self, check_osdeps=True): self.log.info("Checking skipsteps") if not isinstance(self._config['skipsteps'][0], (list, tuple,)): - self.log.error('Invalid type for skipsteps. Allowed are list or tuple, got %s (%s)' % - (type(self._config['skipsteps'][0]), self._config['skipsteps'][0])) + raise EasyBuildError('Invalid type for skipsteps. Allowed are list or tuple, got %s (%s)', + type(self._config['skipsteps'][0]), self._config['skipsteps'][0]) self.log.info("Checking build option lists") self.validate_iterate_opts_lists() @@ -333,17 +340,17 @@ def validate(self, check_osdeps=True): def validate_license(self): """Validate the license""" - lic = self._config['software_license'][0] + lic = self['software_license'] if lic is None: - self.log.deprecated('Mandatory license not enforced', '2.0') # when mandatory, remove this possibility if 'software_license' in self.mandatory: - self.log.error('License is mandatory') - elif not isinstance(lic, License): - self.log.error('License %s has to be a License subclass instance, found classname %s.' % - (lic, lic.__class__.__name__)) - elif not lic.name in EASYCONFIG_LICENSES_DICT: - self.log.error('Invalid license %s (classname: %s).' % (lic.name, lic.__class__.__name__)) + raise EasyBuildError("Software license is mandatory, but 'software_license' is undefined") + elif lic in EASYCONFIG_LICENSES_DICT: + # create License instance + self.software_license = EASYCONFIG_LICENSES_DICT[lic]() + else: + known_licenses = ', '.join(sorted(EASYCONFIG_LICENSES_DICT.keys())) + raise EasyBuildError("Invalid license %s (known licenses: %s)", lic, known_licenses) # TODO, when GROUP_SOURCE and/or GROUP_BINARY is True # check the owner of source / binary (must match 'group' parameter from easyconfig) @@ -361,13 +368,14 @@ def validate_os_deps(self): if isinstance(dep, basestring): dep = (dep,) elif not isinstance(dep, tuple): - self.log.error("Non-tuple value type for OS dependency specification: %s (type %s)" % (dep, type(dep))) + raise EasyBuildError("Non-tuple value type for OS dependency specification: %s (type %s)", + dep, type(dep)) if not any([check_os_dependency(cand_dep) for cand_dep in dep]): not_found.append(dep) if not_found: - self.log.error("One or more OS dependencies were not found: %s" % not_found) + raise EasyBuildError("One or more OS dependencies were not found: %s", not_found) else: self.log.info("OS dependencies ok: %s" % self['osdependencies']) @@ -386,7 +394,7 @@ def validate_iterate_opts_lists(self): # anticipate changes in available easyconfig parameters (e.g. makeopts -> buildopts?) if self.get(opt, None) is None: - self.log.error("%s not available in self.cfg (anymore)?!" % opt) + raise EasyBuildError("%s not available in self.cfg (anymore)?!", opt) # keep track of list, supply first element as first option to handle if isinstance(self[opt], (list, tuple)): @@ -395,22 +403,50 @@ def validate_iterate_opts_lists(self): # make sure that options that specify lists have the same length list_opt_lengths = [length for (opt, length) in opt_counts if length > 1] if len(nub(list_opt_lengths)) > 1: - self.log.error("Build option lists for iterated build should have same length: %s" % opt_counts) + raise EasyBuildError("Build option lists for iterated build should have same length: %s", opt_counts) return True + def filter_hidden_deps(self): + """ + Filter hidden dependencies from list of dependencies. + """ + dep_mod_names = [dep['full_mod_name'] for dep in self['dependencies']] + + faulty_deps = [] + for hidden_dep in self['hiddendependencies']: + # check whether hidden dep is a listed dep using *visible* module name, not hidden one + hidden_mod_name = ActiveMNS().det_full_module_name(hidden_dep) + visible_mod_name = ActiveMNS().det_full_module_name(hidden_dep, force_visible=True) + if visible_mod_name in dep_mod_names: + self['dependencies'] = [d for d in self['dependencies'] if d['full_mod_name'] != visible_mod_name] + self.log.debug("Removed dependency matching hidden dependency %s" % hidden_dep) + elif hidden_mod_name in dep_mod_names: + self['dependencies'] = [d for d in self['dependencies'] if d['full_mod_name'] != hidden_mod_name] + self.log.debug("Hidden dependency %s is already marked to be installed as hidden module", hidden_dep) + else: + # hidden dependencies must also be included in list of dependencies; + # this is done to try and make easyconfigs portable w.r.t. site-specific policies with minimal effort, + # i.e. by simply removing the 'hiddendependencies' specification + self.log.warning("Hidden dependency %s not in list of dependencies" % visible_mod_name) + faulty_deps.append(visible_mod_name) + + if faulty_deps: + raise EasyBuildError("Hidden dependencies with visible module names %s not in list of dependencies: %s", + faulty_deps, dep_mod_names) + def dependencies(self): """ Returns an array of parsed dependencies (after filtering, if requested) dependency = {'name': '', 'version': '', 'dummy': (False|True), 'versionsuffix': '', 'toolchain': ''} """ - deps = self['dependencies'] + self.builddependencies() + deps = self['dependencies'] + self['builddependencies'] + self['hiddendependencies'] # if filter-deps option is provided we "clean" the list of dependencies for # each processed easyconfig to remove the unwanted dependencies + self.log.debug("Dependencies BEFORE filtering: %s" % deps) filter_deps = build_option('filter_deps') if filter_deps: - self.log.debug("Dependencies BEFORE filtering: %s" % deps) filtered_deps = [] for dep in deps: if dep['name'] not in filter_deps: @@ -448,7 +484,7 @@ def toolchain(self): returns the Toolchain used """ if self._toolchain is None: - self._toolchain = get_toolchain(self['toolchain'], self['toolchainopts'], ActiveMNS()) + self._toolchain = get_toolchain(self['toolchain'], self['toolchainopts'], mns=ActiveMNS()) tc_dict = self._toolchain.as_dict() self.log.debug("Initialized toolchain: %s (opts: %s)" % (tc_dict, self['toolchainopts'])) return self._toolchain @@ -457,52 +493,37 @@ def dump(self, fp): """ Dump this easyconfig to file, with the given filename. """ - eb_file = file(fp, "w") - - def to_str(x): - """Return quoted version of x""" - if isinstance(x, basestring): - if '\n' in x or ('"' in x and "'" in x): - return '"""%s"""' % x - elif "'" in x: - return '"%s"' % x - else: - return "'%s'" % x - else: - return "%s" % x - - # ordered groups of keys to obtain a nice looking easyconfig file - grouped_keys = [ - ["name", "version", "versionprefix", "versionsuffix"], - ["homepage", "description"], - ["toolchain", "toolchainopts"], - ["source_urls", "sources"], - ["patches"], - ["dependencies"], - ["parallel", "maxparallel"], - ["osdependencies"] - ] - - # print easyconfig parameters ordered and in groups specified above - ebtxt = [] - printed_keys = [] - for group in grouped_keys: - for key1 in group: - val = self._config[key1][0] - for key2, [def_val, _, _] in DEFAULT_CONFIG.items(): - # only print parameters that are different from the default value - if key1 == key2 and val != def_val: - ebtxt.append("%s = %s" % (key1, to_str(val))) - printed_keys.append(key1) - ebtxt.append("") - - # print other easyconfig parameters at the end - for key, [val, _, _] in DEFAULT_CONFIG.items(): - if not key in printed_keys and val != self._config[key][0]: - ebtxt.append("%s = %s" % (key, to_str(self._config[key][0]))) - - eb_file.write('\n'.join(ebtxt)) - eb_file.close() + orig_enable_templating = self.enable_templating + + # templated values should be dumped unresolved + self.enable_templating = False + + # build dict of default values + default_values = dict([(key, DEFAULT_CONFIG[key][0]) for key in DEFAULT_CONFIG]) + default_values.update(dict([(key, self.extra_options[key][0]) for key in self.extra_options])) + + self.generate_template_values() + templ_const = dict([(quote_py_str(const[1]), const[0]) for const in TEMPLATE_CONSTANTS]) + + # reverse map of templates longer than 2 characters, to inject template values where possible, sorted on length + keys = sorted(self.template_values, key=lambda k: len(self.template_values[k]), reverse=True) + templ_val = OrderedDict([(self.template_values[k], k) for k in keys if len(self.template_values[k]) > 2]) + + ectxt = self.parser.dump(self, default_values, templ_const, templ_val) + self.log.debug("Dumped easyconfig: %s", ectxt) + + if build_option('dump_autopep8'): + autopep8_opts = { + 'aggressive': 1, # enable non-whitespace changes, but don't be too aggressive + 'max_line_length': 120, + } + self.log.info("Reformatting dumped easyconfig using autopep8 (options: %s)", autopep8_opts) + ectxt = autopep8.fix_code(ectxt, options=autopep8_opts) + self.log.debug("Dumped easyconfig after autopep8 reformatting: %s", ectxt) + + write_file(fp, ectxt.strip()) + + self.enable_templating = orig_enable_templating def _validate(self, attr, values): # private method """ @@ -512,10 +533,10 @@ def _validate(self, attr, values): # private method if values is None: values = [] if self[attr] and self[attr] not in values: - self.log.error("%s provided '%s' is not valid: %s" % (attr, self[attr], values)) + raise EasyBuildError("%s provided '%s' is not valid: %s", attr, self[attr], values) # private method - def _parse_dependency(self, dep): + def _parse_dependency(self, dep, hidden=False): """ parses the dependency into a usable dict with a common format dep can be a dict, a tuple or a list. @@ -524,26 +545,41 @@ def _parse_dependency(self, dep): of these attributes, 'name' and 'version' are mandatory output dict contains these attributes: - ['name', 'version', 'versionsuffix', 'dummy', 'toolchain', 'short_mod_name', 'full_mod_name'] + ['name', 'version', 'versionsuffix', 'dummy', 'toolchain', 'short_mod_name', 'full_mod_name', 'hidden', + 'external_module'] + + @param hidden: indicate whether corresponding module file should be installed hidden ('.'-prefixed) """ # convert tuple to string otherwise python might complain about the formatting self.log.debug("Parsing %s as a dependency" % str(dep)) attr = ['name', 'version', 'versionsuffix', 'toolchain'] dependency = { - 'dummy': False, - 'full_mod_name': None, # full module name - 'short_mod_name': None, # short module name - 'name': '', # software name - 'toolchain': None, - 'version': '', + # full/short module names + 'full_mod_name': None, + 'short_mod_name': None, + # software name, version, versionsuffix + 'name': None, + 'version': None, 'versionsuffix': '', + # toolchain with which this dependency is installed + 'toolchain': None, + # boolean indicating whether we're dealing with a dummy toolchain for this dependency + 'dummy': False, + # boolean indicating whether the module for this dependency is (to be) installed hidden + 'hidden': hidden, + # boolean indicating whether this dependency should be resolved via an external module + 'external_module': False, + # metadata in case this is an external module; + # provides information on what this module represents (software name/version, install prefix, ...) + 'external_module_metadata': {}, } if isinstance(dep, dict): dependency.update(dep) # make sure 'dummy' key is handled appropriately if 'dummy' in dep and not 'toolchain' in dep: dependency['toolchain'] = dep['dummy'] + elif isinstance(dep, Dependency): dependency['name'] = dep.name() dependency['version'] = dep.version() @@ -553,12 +589,35 @@ def _parse_dependency(self, dep): toolchain = dep.toolchain() if toolchain is not None: dependency['toolchain'] = toolchain + elif isinstance(dep, (list, tuple)): - # try and convert to list - dep = list(dep) - dependency.update(dict(zip(attr, dep))) + if dep and dep[-1] == EXTERNAL_MODULE_MARKER: + if len(dep) == 2: + dependency['external_module'] = True + dependency['short_mod_name'] = dep[0] + dependency['full_mod_name'] = dep[0] + if dep[0] in self.external_modules_metadata: + dependency['external_module_metadata'].update(self.external_modules_metadata[dep[0]]) + self.log.info("Updated dependency info with available metadata for external module %s: %s", + dep[0], dependency['external_module_metadata']) + else: + self.log.info("No metadata available for external module %s", dep[0]) + else: + raise EasyBuildError("Incorrect external dependency specification: %s", dep) + else: + # non-external dependency: tuple (or list) that specifies name/version(/versionsuffix(/toolchain)) + dependency.update(dict(zip(attr, dep))) + else: - self.log.error('Dependency %s of unsupported type: %s.' % (dep, type(dep))) + raise EasyBuildError("Dependency %s of unsupported type: %s", dep, type(dep)) + + if dependency['external_module']: + self.log.debug("Returning parsed external dependency: %s", dependency) + return dependency + + # check whether this dependency should be hidden according to --hide-deps + if build_option('hide_deps'): + dependency['hidden'] |= dependency['name'] in build_option('hide_deps') # dependency inherits toolchain, unless it's specified to have a custom toolchain tc = copy.deepcopy(self['toolchain']) @@ -572,27 +631,29 @@ def _parse_dependency(self, dep): if len(tc_spec) == 2: tc = {'name': tc_spec[0], 'version': tc_spec[1]} else: - self.log.error("List/tuple value for toolchain should have two elements (%s)" % str(tc_spec)) + raise EasyBuildError("List/tuple value for toolchain should have two elements (%s)", str(tc_spec)) elif isinstance(tc_spec, dict): if 'name' in tc_spec and 'version' in tc_spec: tc = copy.deepcopy(tc_spec) else: - self.log.error("Found toolchain spec as dict with required 'name'/'version' keys: %s" % tc_spec) + raise EasyBuildError("Found toolchain spec as dict with wrong keys (no name/version): %s", tc_spec) else: - self.log.error("Unsupported type for toolchain spec encountered: %s => %s" % (tc_spec, type(tc_spec))) + raise EasyBuildError("Unsupported type for toolchain spec encountered: %s (%s)", tc_spec, type(tc_spec)) + self.log.debug("Derived toolchain to use for dependency %s, based on toolchain spec %s: %s", dep, tc_spec, tc) dependency['toolchain'] = tc # make sure 'dummy' value is set correctly dependency['dummy'] = dependency['toolchain']['name'] == DUMMY_TOOLCHAIN_NAME # validations - if not dependency['name']: - self.log.error("Dependency specified without name: %s" % dependency) + if dependency['name'] is None: + raise EasyBuildError("Dependency specified without name: %s", dependency) - if not dependency['version']: - self.log.error("Dependency specified without version: %s" % dependency) + if dependency['version'] is None: + raise EasyBuildError("Dependency specified without version: %s", dependency) + # set module names dependency['short_mod_name'] = ActiveMNS().det_short_module_name(dependency) dependency['full_mod_name'] = ActiveMNS().det_full_module_name(dependency) @@ -625,33 +686,43 @@ def _generate_template_values(self, ignore=None, skip_lower=True): if v is None: del self.template_values[k] - @handle_deprecated_easyconfig_parameter + @handle_deprecated_or_replaced_easyconfig_parameters + def __contains__(self, key): + """Check whether easyconfig parameter is defined""" + return key in self._config + + @handle_deprecated_or_replaced_easyconfig_parameters def __getitem__(self, key): - """ - will return the value without the help text - """ - value = self._config[key][0] + """Return value of specified easyconfig parameter (without help text, etc.)""" + value = None + if key in self._config: + value = self._config[key][0] + else: + raise EasyBuildError("Use of unknown easyconfig parameter '%s' when getting parameter value", key) + if self.enable_templating: if self.template_values is None or len(self.template_values) == 0: self.generate_template_values() - return resolve_template(value, self.template_values) - else: - return value + value = resolve_template(value, self.template_values) - @handle_deprecated_easyconfig_parameter + return value + + @handle_deprecated_or_replaced_easyconfig_parameters def __setitem__(self, key, value): - """ - sets the value of key in config. - help text is untouched - """ - self._config[key][0] = value + """Set value of specified easyconfig parameter (help text & co is left untouched)""" + if key in self._config: + self._config[key][0] = value + else: + raise EasyBuildError("Use of unknown easyconfig parameter '%s' when setting parameter value to '%s'", + key, value) + @handle_deprecated_or_replaced_easyconfig_parameters def get(self, key, default=None): """ Gets the value of a key in the config, with 'default' as fallback. """ - if key in self._config: - return self.__getitem__(key) + if key in self: + return self[key] else: return default @@ -673,54 +744,14 @@ def asdict(self): def det_installversion(version, toolchain_name, toolchain_version, prefix, suffix): """Deprecated 'det_installversion' function, to determine exact install version, based on supplied parameters.""" old_fn = 'framework.easyconfig.easyconfig.det_installversion' - _log.deprecated('Use module_generator.det_full_ec_version instead of %s' % old_fn, '2.0') - cfg = { - 'version': version, - 'toolchain': {'name': toolchain_name, 'version': toolchain_version}, - 'versionprefix': prefix, - 'versionsuffix': suffix, - } - return det_full_ec_version(cfg) - - -def fetch_parameter_from_easyconfig_file(path, param): - """Fetch parameter specification from given easyconfig file.""" - # check whether easyblock is specified in easyconfig file - # note: we can't rely on value for 'easyblock' in parsed easyconfig, it may be the default value - reg = re.compile(r"^\s*%s\s*=\s*(?P\S.*)\s*$" % param, re.M) - txt = read_file(path) - res = reg.search(txt) - if res: - return res.group('param').strip("'\"") - else: - return None + _log.nosupport('Use det_full_ec_version from easybuild.tools.module_generator instead of %s' % old_fn, '2.0') -def get_class_for(modulepath, class_name): - """ - Get class for a given class name and easyblock module path. - """ - # try to import specified module path, reraise ImportError if it occurs - try: - m = __import__(modulepath, globals(), locals(), ['']) - except ImportError, err: - raise ImportError(err) - # try to import specified class name from specified module path, throw ImportError if this fails - try: - c = getattr(m, class_name) - except AttributeError, err: - raise ImportError("Failed to import %s from %s: %s" % (class_name, modulepath, err)) - return c - - -def get_easyblock_class(easyblock, name=None): +def get_easyblock_class(easyblock, name=None, default_fallback=True, error_on_failed_import=True): """ Get class for a particular easyblock (or use default) """ - - def_class = get_easyconfig_parameter_default('easyblock') - def_mod_path = get_module_path(def_class, generic=True) - + cls = None try: if easyblock: # something was specified, lets parse it @@ -729,8 +760,8 @@ def get_easyblock_class(easyblock, name=None): # figure out if full path was specified or not if es: modulepath = '.'.join(es) - tup = (class_name, modulepath) - _log.info("Assuming that full easyblock module path was specified (class: %s, modulepath: %s)" % tup) + _log.info("Assuming that full easyblock module path was specified (class: %s, modulepath: %s)", + class_name, modulepath) cls = get_class_for(modulepath, class_name) else: # if we only get the class name, most likely we're dealing with a generic easyblock @@ -750,9 +781,22 @@ def get_easyblock_class(easyblock, name=None): class_name = encode_class_name(name) # modulepath will be the namespace + encoded modulename (from the classname) modulepath = get_module_path(class_name) - if not os.path.exists("%s.py" % modulepath): - _log.deprecated("Determine module path based on software name", "2.0") - modulepath = get_module_path(name, decode=False) + modulepath_imported = False + try: + __import__(modulepath, globals(), locals(), ['']) + modulepath_imported = True + except ImportError, err: + _log.debug("Failed to import module '%s': %s" % (modulepath, err)) + + # check if determining module path based on software name would have resulted in a different module path + if modulepath_imported: + _log.debug("Module path '%s' found" % modulepath) + else: + _log.debug("No module path '%s' found" % modulepath) + modulepath_bis = get_module_path(name, decode=False) + _log.debug("Module path determined based on software name: %s" % modulepath_bis) + if modulepath_bis != modulepath: + _log.nosupport("Determining module path based on software name", '2.0') # try and find easyblock try: @@ -760,25 +804,39 @@ def get_easyblock_class(easyblock, name=None): cls = get_class_for(modulepath, class_name) _log.info("Successfully obtained %s class instance from %s" % (class_name, modulepath)) except ImportError, err: - # when an ImportError occurs, make sure that it's caused by not finding the easyblock module, # and not because of a broken import statement in the easyblock module error_re = re.compile(r"No module named %s" % modulepath.replace("easybuild.easyblocks.", '')) _log.debug("error regexp: %s" % error_re.pattern) if error_re.match(str(err)): - # no easyblock could be found, so fall back to default class. - _log.warning("Failed to import easyblock for %s, falling back to default class %s: error: %s" % \ - (class_name, (def_mod_path, def_class), err)) - cls = get_class_for(def_mod_path, def_class) + if default_fallback: + # no easyblock could be found, so fall back to ConfigureMake (NO LONGER SUPPORTED) + legacy_fallback_easyblock = 'ConfigureMake' + def_mod_path = get_module_path(legacy_fallback_easyblock, generic=True) + depr_msg = "Fallback to default easyblock %s (from %s)" % (legacy_fallback_easyblock, def_mod_path) + depr_msg += "; use \"easyblock = '%s'\" in easyconfig file?" % legacy_fallback_easyblock + _log.nosupport(depr_msg, '2.0') else: - _log.error("Failed to import easyblock for %s because of module issue: %s" % (class_name, err)) + if error_on_failed_import: + raise EasyBuildError("Failed to import easyblock for %s because of module issue: %s", + class_name, err) + else: + _log.debug("Failed to import easyblock for %s, but ignoring it: %s" % (class_name, err)) + + if cls is not None: + _log.info("Successfully obtained class '%s' for easyblock '%s' (software name '%s')", + cls.__name__, easyblock, name) + else: + _log.debug("No class found for easyblock '%s' (software name '%s', default fallback: %s", + easyblock, name, default_fallback) - tup = (cls.__name__, easyblock, name) - _log.info("Successfully obtained class '%s' for easyblock '%s' (software name '%s')" % tup) return cls + except EasyBuildError, err: + # simply reraise rather than wrapping it into another error + raise err except Exception, err: - _log.error("Failed to obtain class for %s easyblock (not available?): %s" % (easyblock, err)) + raise EasyBuildError("Failed to obtain class for %s easyblock (not available?): %s", easyblock, err) def get_module_path(name, generic=False, decode=True): @@ -859,21 +917,25 @@ def resolve_template(value, tmpl_dict): return value -def process_easyconfig(path, build_specs=None, validate=True, parse_only=False): +def process_easyconfig(path, build_specs=None, validate=True, parse_only=False, hidden=None): """ Process easyconfig, returning some information for each block @param path: path to easyconfig file @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) @param validate: whether or not to perform validation + @param hidden: indicate whether corresponding module file should be installed hidden ('.'-prefixed) """ blocks = retrieve_blocks_in_spec(path, build_option('only_blocks')) + if hidden is None: + hidden = build_option('hidden') + # only cache when no build specifications are involved (since those can't be part of a dict key) cache_key = None if build_specs is None: - cache_key = (path, validate, parse_only) + cache_key = (path, validate, hidden, parse_only) if cache_key in _easyconfigs_cache: - return copy.deepcopy(_easyconfigs_cache[cache_key]) + return [e.copy() for e in _easyconfigs_cache[cache_key]] easyconfigs = [] for spec in blocks: @@ -882,10 +944,9 @@ def process_easyconfig(path, build_specs=None, validate=True, parse_only=False): # create easyconfig try: - ec = EasyConfig(spec, build_specs=build_specs, validate=validate) + ec = EasyConfig(spec, build_specs=build_specs, validate=validate, hidden=hidden) except EasyBuildError, err: - msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg) - _log.exception(msg) + raise EasyBuildError("Failed to process easyconfig %s: %s", spec, err.msg) name = ec['name'] @@ -897,21 +958,28 @@ def process_easyconfig(path, build_specs=None, validate=True, parse_only=False): if not parse_only: # also determine list of dependencies, module name (unless only parsed easyconfigs are requested) easyconfig.update({ - 'spec': spec, + 'spec': ec.path, 'short_mod_name': ec.short_mod_name, 'full_mod_name': ec.full_mod_name, 'dependencies': [], 'builddependencies': [], + 'hiddendependencies': [], + 'hidden': hidden, }) if len(blocks) > 1: easyconfig['original_spec'] = path # add build dependencies - for dep in ec.builddependencies(): + for dep in ec['builddependencies']: _log.debug("Adding build dependency %s for app %s." % (dep, name)) easyconfig['builddependencies'].append(dep) - # add dependencies (including build dependencies) + # add hidden dependencies + for dep in ec['hiddendependencies']: + _log.debug("Adding hidden dependency %s for app %s." % (dep, name)) + easyconfig['hiddendependencies'].append(dep) + + # add dependencies (including build & hidden dependencies) for dep in ec.dependencies(): _log.debug("Adding dependency %s for app %s." % (dep, name)) easyconfig['dependencies'].append(dep) @@ -922,11 +990,8 @@ def process_easyconfig(path, build_specs=None, validate=True, parse_only=False): _log.debug("Adding toolchain %s as dependency for app %s." % (dep, name)) easyconfig['dependencies'].append(dep) - # this is used by the parallel builder - easyconfig['unresolved_deps'] = copy.deepcopy(easyconfig['dependencies']) - if cache_key is not None: - _easyconfigs_cache[cache_key] = copy.deepcopy(easyconfigs) + _easyconfigs_cache[cache_key] = [e.copy() for e in easyconfigs] return easyconfigs @@ -956,9 +1021,9 @@ def robot_find_easyconfig(name, version): _log.debug("Obtained easyconfig path from cache for %s: %s" % (key, _easyconfig_files_cache[key])) return _easyconfig_files_cache[key] paths = build_option('robot_path') + if not paths: + raise EasyBuildError("No robot path specified, which is required when looking for easyconfigs (use --robot)") if not isinstance(paths, (list, tuple)): - if paths is None: - _log.error("No robot path specified, which is required when looking for easyconfigs (use --robot)") paths = [paths] # candidate easyconfig paths for path in paths: @@ -989,7 +1054,8 @@ def __init__(self, *args, **kwargs): if sel_mns in avail_mnss: self.mns = avail_mnss[sel_mns]() else: - self.log.error("Selected module naming scheme %s could not be found in %s" % (sel_mns, avail_mnss.keys())) + raise EasyBuildError("Selected module naming scheme %s could not be found in %s", + sel_mns, avail_mnss.keys()) def requires_full_easyconfig(self, keys): """Check whether specified list of easyconfig parameters is sufficient for active module naming scheme.""" @@ -997,55 +1063,101 @@ def requires_full_easyconfig(self, keys): def check_ec_type(self, ec): """ - Query module naming scheme using specified method and argument. - Obtain and pass a full parsed easyconfig file if provided keys are insufficient. + Obtain a full parsed easyconfig file to pass to naming scheme methods if provided keys are insufficient. """ if not isinstance(ec, EasyConfig) and self.requires_full_easyconfig(ec.keys()): self.log.debug("A parsed easyconfig is required by the module naming scheme, so finding one for %s" % ec) # fetch/parse easyconfig file if deemed necessary eb_file = robot_find_easyconfig(ec['name'], det_full_ec_version(ec)) if eb_file is not None: - parsed_ec = process_easyconfig(eb_file, parse_only=True) + parsed_ec = process_easyconfig(eb_file, parse_only=True, hidden=ec['hidden']) if len(parsed_ec) > 1: self.log.warning("More than one parsed easyconfig obtained from %s, only retaining first" % eb_file) self.log.debug("Full list of parsed easyconfigs: %s" % parsed_ec) ec = parsed_ec[0]['ec'] else: - self.log.error("Failed to find an easyconfig file when determining module name for: %s" % ec) + raise EasyBuildError("Failed to find easyconfig file '%s-%s.eb' when determining module name for: %s", + ec['name'], det_full_ec_version(ec), ec) return ec - def det_full_module_name(self, ec): + def _det_module_name_with(self, mns_method, ec, force_visible=False): """ - Determine full module name by selected module naming scheme, based on supplied easyconfig. + Determine module name using specified module naming scheme method, based on supplied easyconfig. Returns a string representing the module name, e.g. 'GCC/4.6.3', 'Python/2.7.5-ictce-4.1.13', with the following requirements: - module name is specified as a relative path - string representing module name has length > 0 - module name only contains printable characters (string.printable, except carriage-control chars) """ - self.log.debug("Determining full module name for %s" % ec) - mod_name = self.mns.det_full_module_name(self.check_ec_type(ec)) - - if not is_valid_module_name(mod_name): - self.log.error("%s is not a valid full module name" % str(mod_name)) + """ + Returns a string representing the module name, e.g. 'GCC/4.6.3', 'Python/2.7.5-ictce-4.1.13', + with the following requirements: + - module name is specified as a relative path + - string representing module name has length > 0 + - module name only contains printable characters (string.printable, except carriage-control chars) + """ + ec = self.check_ec_type(ec) + + # replace software name with desired replacement (if specified) + orig_name = None + if ec.get('modaltsoftname', None): + orig_name = ec['name'] + ec['name'] = ec['modaltsoftname'] + self.log.info("Replaced software name '%s' with '%s' when determining module name", orig_name, ec['name']) else: - self.log.debug("Obtained valid full module name %s" % mod_name) + self.log.debug("No alternative software name specified to determine module name with") - return mod_name + mod_name = mns_method(ec) - def det_devel_module_filename(self, ec): - """Determine devel module filename.""" - return self.mns.det_full_module_name(self.check_ec_type(ec)).replace(os.path.sep, '-') + DEVEL_MODULE_SUFFIX + # restore original software name if it was tampered with + if orig_name is not None: + ec['name'] = orig_name - def det_short_module_name(self, ec): - """Determine module name according to module naming scheme.""" - self.log.debug("Determining module name for %s" % ec) - mod_name = self.mns.det_short_module_name(self.check_ec_type(ec)) if not is_valid_module_name(mod_name): - self.log.error("%s is not a valid module name" % str(mod_name)) + raise EasyBuildError("%s is not a valid module name", str(mod_name)) + + # check whether module name should be hidden or not + # ec may be either a dict or an EasyConfig instance, 'force_visible' argument overrules + if (ec.get('hidden', False) or getattr(ec, 'hidden', False)) and not force_visible: + mod_name = det_hidden_modname(mod_name) + + return mod_name + + def det_full_module_name(self, ec, force_visible=False): + """Determine full module name by selected module naming scheme, based on supplied easyconfig.""" + self.log.debug("Determining full module name for %s (force_visible: %s)" % (ec, force_visible)) + if ec.get('external_module', False): + # external modules have the module name readily available, and may lack the info required by the MNS + mod_name = ec['full_mod_name'] + self.log.debug("Full module name for external module: %s", mod_name) else: - self.log.debug("Obtained valid module name %s" % mod_name) + mod_name = self._det_module_name_with(self.mns.det_full_module_name, ec, force_visible=force_visible) + self.log.debug("Obtained valid full module name %s", mod_name) + return mod_name + + def det_install_subdir(self, ec): + """Determine name of software installation subdirectory.""" + self.log.debug("Determining software installation subdir for %s", ec) + subdir = self.mns.det_install_subdir(self.check_ec_type(ec)) + self.log.debug("Obtained subdir %s", subdir) + return subdir + + def det_devel_module_filename(self, ec, force_visible=False): + """Determine devel module filename.""" + modname = self.det_full_module_name(ec, force_visible=force_visible) + return modname.replace(os.path.sep, '-') + DEVEL_MODULE_SUFFIX + + def det_short_module_name(self, ec, force_visible=False): + """Determine short module name according to module naming scheme.""" + self.log.debug("Determining short module name for %s (force_visible: %s)" % (ec, force_visible)) + mod_name = self._det_module_name_with(self.mns.det_short_module_name, ec, force_visible=force_visible) + self.log.debug("Obtained valid short module name %s" % mod_name) + + # sanity check: obtained module name should pass the 'is_short_modname_for' check + if not self.is_short_modname_for(mod_name, ec.get('modaltsoftname', None) or ec['name']): + raise EasyBuildError("is_short_modname_for('%s', '%s') for active module naming scheme returns False", + mod_name, ec['name']) return mod_name def det_module_subdir(self, ec): @@ -1081,3 +1193,9 @@ def expand_toolchain_load(self): This is useful when toolchains are not exposed to users. """ return self.mns.expand_toolchain_load() + + def is_short_modname_for(self, short_modname, name): + """ + Determine whether the specified (short) module name is a module for software with the specified name. + """ + return self.mns.is_short_modname_for(short_modname, name) diff --git a/easybuild/framework/easyconfig/format/__init__.py b/easybuild/framework/easyconfig/format/__init__.py index d9929739b2..0daff8a9cc 100644 --- a/easybuild/framework/easyconfig/format/__init__.py +++ b/easybuild/framework/easyconfig/format/__init__.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/convert.py b/easybuild/framework/easyconfig/format/convert.py index 4f03522275..df3259a871 100644 --- a/easybuild/framework/easyconfig/format/convert.py +++ b/easybuild/framework/easyconfig/format/convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2014 Ghent University +# Copyright 2014-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 4003f5682d..648af65c05 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -32,14 +32,17 @@ import copy import re from vsc.utils import fancylogger -from vsc.utils.missing import get_subclasses, any +from vsc.utils.missing import get_subclasses from easybuild.framework.easyconfig.format.version import EasyVersion, OrderedVersionOperators from easybuild.framework.easyconfig.format.version import ToolchainVersionOperator, VersionOperator from easybuild.framework.easyconfig.format.convert import Dependency +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.configobj import Section +INDENT_4SPACES = ' ' * 4 + # format is mandatory major.minor FORMAT_VERSION_KEYWORD = "EASYCONFIGFORMAT" FORMAT_VERSION_TEMPLATE = "%(major)s.%(minor)s" @@ -47,6 +50,29 @@ FORMAT_VERSION_REGEXP = re.compile(r'^#\s+%s\s*(?P\d+)\.(?P\d+)\s*$' % FORMAT_VERSION_KEYWORD, re.M) FORMAT_DEFAULT_VERSION = EasyVersion('1.0') +DEPENDENCY_PARAMETERS = ['builddependencies', 'dependencies', 'hiddendependencies'] + +# values for these keys will not be templated in dump() +EXCLUDED_KEYS_REPLACE_TEMPLATES = ['description', 'easyblock', 'homepage', 'name', 'toolchain', 'version'] + +# ordered groups of keys to obtain a nice looking easyconfig file +GROUPED_PARAMS = [ + ['easyblock'], + ['name', 'version', 'versionprefix', 'versionsuffix'], + ['homepage', 'description'], + ['toolchain', 'toolchainopts'], + ['sources', 'source_urls'], + ['patches'], + DEPENDENCY_PARAMETERS, + ['osdependencies'], + ['preconfigopts', 'configopts'], + ['prebuildopts', 'buildopts'], + ['preinstallopts', 'installopts'], + ['parallel', 'maxparallel'], +] +LAST_PARAMS = ['sanity_check_paths', 'moduleclass'] + + _log = fancylogger.getLogger('easyconfig.format.format', fname=False) @@ -59,7 +85,7 @@ def get_format_version(txt): maj_min = res.groupdict() format_version = EasyVersion(FORMAT_VERSION_TEMPLATE % maj_min) except (KeyError, TypeError), err: - _log.error("Failed to get version from match %s: %s" % (res.groups(), err)) + raise EasyBuildError("Failed to get version from match %s: %s", res.groups(), err) return format_version @@ -242,13 +268,13 @@ def parse_sections(self, toparse, current): new_value = [] for dep_name, dep_val in value.items(): if isinstance(dep_val, Section): - self.log.error("Unsupported nested section '%s' found in dependencies section" % dep_name) + raise EasyBuildError("Unsupported nested section '%s' in dependencies section", dep_name) else: # FIXME: parse the dependency specification for version, toolchain, suffix, etc. dep = Dependency(dep_val, name=dep_name) if dep.name() is None or dep.version() is None: - tmpl = "Failed to find name/version in parsed dependency: %s (dict: %s)" - self.log.error(tmpl % (dep, dict(dep))) + raise EasyBuildError("Failed to find name/version in parsed dependency: %s (dict: %s)", + dep, dict(dep)) new_value.append(dep) tmpl = 'Converted dependency section %s to %s, passed it to parent section (or default)' @@ -268,7 +294,7 @@ def parse_sections(self, toparse, current): else: self.log.debug("Not a %s section marker" % marker_type.__name__) if not new_key: - self.log.error("Unsupported section marker '%s'" % key) + raise EasyBuildError("Unsupported section marker '%s'", key) # parse value as a section, recursively new_value = self.parse_sections(value, current.get_nested_dict()) @@ -296,10 +322,10 @@ def parse_sections(self, toparse, current): # remove possible surrounding whitespace (some people add space after comma) new_value = [value_type(x.strip()) for x in value] if False in [x.is_valid() for x in new_value]: - self.log.error("Failed to parse '%s' as list of %s" % (value, value_type.__name__)) + raise EasyBuildError("Failed to parse '%s' as list of %s", value, value_type.__name__) else: - tup = (key, value, type(value)) - self.log.error('Bug: supported but unknown key %s with non-string value: %s, type %s' % tup) + raise EasyBuildError('Bug: supported but unknown key %s with non-string value: %s, type %s', + key, value, type(value)) self.log.debug("Converted value '%s' for key '%s' into new value '%s'" % (value, key, new_value)) current[key] = new_value @@ -334,7 +360,7 @@ def parse(self, configobj): self.supported = self.sections.pop(self.SECTION_MARKER_SUPPORTED) for key, value in self.supported.items(): if not key in self.VERSION_OPERATOR_VALUE_TYPES: - self.log.error('Unsupported key %s in %s section' % (key, self.SECTION_MARKER_SUPPORTED)) + raise EasyBuildError('Unsupported key %s in %s section', key, self.SECTION_MARKER_SUPPORTED) self.sections['%s' % key] = value for key, supported_key, fn_name in [('version', 'versions', 'get_version_str'), @@ -344,7 +370,7 @@ def parse(self, configobj): first = self.supported[supported_key][0] f_val = getattr(first, fn_name)() if f_val is None: - self.log.error("First %s %s can't be used as default (%s returned None)" % (key, first, fn_name)) + raise EasyBuildError("First %s %s can't be used as default (%s returned None)", key, first, fn_name) else: self.log.debug('Using first %s (%s) as default %s' % (key, first, f_val)) self.default[key] = f_val @@ -438,8 +464,7 @@ def _squash_netsed_dict(self, key, nested_dict, squashed, sanity, vt_tuple): tc_overops.add(key) if key.test(tcname, tcversion): - tup = (tcname, tcversion, key) - self.log.debug("Found matching marker for specified toolchain '%s, %s': %s" % tup) + self.log.debug("Found matching marker for specified toolchain '%s, %s': %s", tcname, tcversion, key) # TODO remove when unifying add_toolchina with .add() tmp_squashed = self._squash(vt_tuple, nested_dict, sanity) res_sections.update(tmp_squashed.result) @@ -456,7 +481,7 @@ def _squash_netsed_dict(self, key, nested_dict, squashed, sanity, vt_tuple): else: self.log.debug('Found non-matching version marker %s. Ignoring this (nested) section.' % key) else: - self.log.error("Unhandled section marker '%s' (type '%s')" % (key, type(key))) + raise EasyBuildError("Unhandled section marker '%s' (type '%s')", key, type(key)) return res_sections @@ -479,8 +504,8 @@ def _squash_versop(self, key, value, squashed, sanity, vt_tuple): tmp_tc_oversops = {} # temporary, only for conflict checking for tcversop in value: tc_overops = tmp_tc_oversops.setdefault(tcversop.tc_name, OrderedVersionOperators()) - tup = (tcversop, tc_overops, tcname, tcversion) - self.log.debug('Add tcversop %s to tc_overops %s tcname %s tcversion %s' % tup) + self.log.debug("Add tcversop %s to tc_overops %s tcname %s tcversion %s", + tcversop, tc_overops, tcname, tcversion) tc_overops.add(tcversop) # test non-conflicting list if tcversop.test(tcname, tcversion): matching_toolchains.append(tcversop) @@ -507,7 +532,7 @@ def _squash_versop(self, key, value, squashed, sanity, vt_tuple): self.log.debug('No matching versions, removing the whole current key %s' % key) return Squashed() else: - self.log.error('Unexpected VERSION_OPERATOR_VALUE_TYPES key %s value %s' % (key, value)) + raise EasyBuildError('Unexpected VERSION_OPERATOR_VALUE_TYPES key %s value %s', key, value) return None @@ -520,11 +545,11 @@ def get_version_toolchain(self, version=None, tcname=None, tcversion=None): version = self.default['version'] self.log.debug("No version specified, using default %s" % version) else: - self.log.error("No version specified, no default found.") + raise EasyBuildError("No version specified, no default found.") elif version in versions: self.log.debug("Version '%s' is supported in easyconfig." % version) else: - self.log.error("Version '%s' not supported in easyconfig (only %s)" % (version, versions)) + raise EasyBuildError("Version '%s' not supported in easyconfig (only %s)", version, versions) tcnames = [tc.tc_name for tc in self.supported['toolchains']] if tcname is None: @@ -532,11 +557,11 @@ def get_version_toolchain(self, version=None, tcname=None, tcversion=None): tcname = self.default['toolchain']['name'] self.log.debug("No toolchain name specified, using default %s" % tcname) else: - self.log.error("No toolchain name specified, no default found.") + raise EasyBuildError("No toolchain name specified, no default found.") elif tcname in tcnames: self.log.debug("Toolchain '%s' is supported in easyconfig." % tcname) else: - self.log.error("Toolchain '%s' not supported in easyconfig (only %s)" % (tcname, tcnames)) + raise EasyBuildError("Toolchain '%s' not supported in easyconfig (only %s)", tcname, tcnames) tcs = [tc for tc in self.supported['toolchains'] if tc.tc_name == tcname] if tcversion is None: @@ -544,17 +569,16 @@ def get_version_toolchain(self, version=None, tcname=None, tcversion=None): tcversion = self.default['toolchain']['version'] self.log.debug("No toolchain version specified, using default %s" % tcversion) else: - self.log.error("No toolchain version specified, no default found.") + raise EasyBuildError("No toolchain version specified, no default found.") elif any([tc.test(tcname, tcversion) for tc in tcs]): self.log.debug("Toolchain '%s' version '%s' is supported in easyconfig" % (tcname, tcversion)) else: - tup = (tcname, tcversion, tcs) - self.log.error("Toolchain '%s' version '%s' not supported in easyconfig (only %s)" % tup) + raise EasyBuildError("Toolchain '%s' version '%s' not supported in easyconfig (only %s)", + tcname, tcversion, tcs) - tup = (version, tcname, tcversion) - self.log.debug('version %s, tcversion %s, tcname %s' % tup) + self.log.debug('version %s, tcversion %s, tcname %s', version, tcname, tcversion) - return tup + return (version, tcname, tcversion) def get_specs_for(self, version=None, tcname=None, tcversion=None): """ @@ -578,9 +602,10 @@ def __init__(self): self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) if not len(self.VERSION) == len(FORMAT_VERSION_TEMPLATE.split('.')): - self.log.error('Invalid version number %s (incorrect length)' % self.VERSION) + raise EasyBuildError('Invalid version number %s (incorrect length)', self.VERSION) self.rawtext = None # text version of the easyconfig + self.comments = {} # comments in easyconfig file self.header = None # easyconfig header (e.g., format version, license, ...) self.docstring = None # easyconfig docstring (e.g., author, maintainer, ...) @@ -603,10 +628,14 @@ def parse(self, txt, **kwargs): """Parse the txt according to this format. This is highly version specific""" raise NotImplementedError - def dump(self): + def dump(self, ecfg, default_values, templ_const, templ_val): """Dump easyconfig according to this format. This is higly version specific""" raise NotImplementedError + def extract_comments(self, rawtxt): + """Extract comments from raw content.""" + raise NotImplementedError + def get_format_version_classes(version=None): """Return the (usable) subclasses from EasyConfigFormat that have a matching version.""" diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index a4cc036826..cce947f3f8 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -36,16 +36,50 @@ import tempfile from vsc.utils import fancylogger -from easybuild.framework.easyconfig.format.format import FORMAT_DEFAULT_VERSION, get_format_version +from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS, EXCLUDED_KEYS_REPLACE_TEMPLATES +from easybuild.framework.easyconfig.format.format import FORMAT_DEFAULT_VERSION, GROUPED_PARAMS, INDENT_4SPACES +from easybuild.framework.easyconfig.format.format import LAST_PARAMS, get_format_version from easybuild.framework.easyconfig.format.pyheaderconfigobj import EasyConfigFormatConfigObj from easybuild.framework.easyconfig.format.version import EasyVersion -from easybuild.tools.build_log import print_msg +from easybuild.framework.easyconfig.templates import to_template_str +from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.filetools import write_file +from easybuild.tools.utilities import quote_py_str + + +# dependency parameters always need to be reformatted, to correctly deal with dumping parsed dependencies +REFORMAT_FORCED_PARAMS = ['sanity_check_paths'] + DEPENDENCY_PARAMETERS +REFORMAT_SKIPPED_PARAMS = ['toolchain', 'toolchainopts'] +REFORMAT_THRESHOLD_LENGTH = 100 # only reformat lines that would be longer than this amount of characters +REFORMAT_ORDERED_ITEM_KEYS = { + 'sanity_check_paths': ['files', 'dirs'], +} _log = fancylogger.getLogger('easyconfig.format.one', fname=False) +def dump_dependency(dep, toolchain): + """Dump parsed dependency in tuple format""" + + if dep['external_module']: + res = "(%s, EXTERNAL_MODULE)" % quote_py_str(dep['full_mod_name']) + else: + # mininal spec: (name, version) + tup = (dep['name'], dep['version']) + if dep['toolchain'] != toolchain: + if dep['dummy']: + tup += (dep['versionsuffix'], True) + else: + tup += (dep['versionsuffix'], (dep['toolchain']['name'], dep['toolchain']['version'])) + + elif dep['versionsuffix']: + tup += (dep['versionsuffix'],) + + res = str(tup) + return res + + class FormatOneZero(EasyConfigFormatConfigObj): """Support for easyconfig format 1.x""" VERSION = EasyVersion('1.0') @@ -71,14 +105,14 @@ def get_config_dict(self): spec_tc_version = spec_tc.get('version', None) cfg = self.pyheader_localvars if spec_version is not None and not spec_version == cfg['version']: - self.log.error('Requested version %s not available, only %s' % (spec_version, cfg['version'])) + raise EasyBuildError('Requested version %s not available, only %s', spec_version, cfg['version']) - tc_name = cfg['toolchain']['name'] - tc_version = cfg['toolchain']['version'] + tc_name = cfg.get('toolchain', {}).get('name', None) + tc_version = cfg.get('toolchain', {}).get('version', None) if spec_tc_name is not None and not spec_tc_name == tc_name: - self.log.error('Requested toolchain name %s not available, only %s' % (spec_tc_name, tc_name)) + raise EasyBuildError('Requested toolchain name %s not available, only %s', spec_tc_name, tc_name) if spec_tc_version is not None and not spec_tc_version == tc_version: - self.log.error('Requested toolchain version %s not available, only %s' % (spec_tc_version, tc_version)) + raise EasyBuildError('Requested toolchain version %s not available, only %s', spec_tc_version, tc_version) return cfg @@ -88,6 +122,227 @@ def parse(self, txt): """ super(FormatOneZero, self).parse(txt, strict_section_markers=True) + def _reformat_line(self, param_name, param_val, outer=False, addlen=0): + """ + Construct formatted string representation of iterable parameter (list/tuple/dict), including comments. + + @param param_name: parameter name + @param param_val: parameter value + @param outer: reformat for top-level parameter, or not + @param addlen: # characters to add to line length + """ + param_strval = str(param_val) + res = param_strval + + # determine whether line would be too long + # note: this does not take into account the parameter name + '=', only the value + line_too_long = len(param_strval) + addlen > REFORMAT_THRESHOLD_LENGTH + forced = param_name in REFORMAT_FORCED_PARAMS + + if param_name in REFORMAT_SKIPPED_PARAMS: + self.log.info("Skipping reformatting value for parameter '%s'", param_name) + + elif outer: + # only reformat outer (iterable) values for (too) long lines (or for select parameters) + if isinstance(param_val, (list, tuple, dict)) and ((len(param_val) > 1 and line_too_long) or forced): + + item_tmpl = INDENT_4SPACES + '%(item)s,%(comment)s\n' + + # start with opening character: [, (, { + res = '%s\n' % param_strval[0] + + # add items one-by-one, special care for dict values (order of keys, different format for elements) + if isinstance(param_val, dict): + ordered_item_keys = REFORMAT_ORDERED_ITEM_KEYS.get(param_name, sorted(param_val.keys())) + for item_key in ordered_item_keys: + item_val = param_val[item_key] + comment = self._get_item_comments(param_name, item_val).get(str(item_val), '') + key_pref = quote_py_str(item_key) + ': ' + addlen = addlen + len(INDENT_4SPACES) + len(key_pref) + len(comment) + formatted_item_val = self._reformat_line(param_name, item_val, addlen=addlen) + res += item_tmpl % { + 'comment': comment, + 'item': key_pref + formatted_item_val, + } + else: # list, tuple + for item in param_val: + comment = self._get_item_comments(param_name, item).get(str(item), '') + addlen = addlen + len(INDENT_4SPACES) + len(comment) + res += item_tmpl % { + 'comment': comment, + 'item': self._reformat_line(param_name, item, addlen=addlen) + } + + # end with closing character: ], ), } + res += param_strval[-1] + + else: + # dependencies are already dumped as strings, so they do not need to be quoted again + if isinstance(param_val, basestring) and param_name not in DEPENDENCY_PARAMETERS: + res = quote_py_str(param_val) + + return res + + def _get_item_comments(self, key, val): + """Get per-item comments for specified parameter name/value.""" + item_comments = {} + for comment_key, comment_val in self.comments['iter'].get(key, {}).items(): + if str(val) in comment_key: + item_comments[str(val)] = comment_val + + return item_comments + + def _find_param_with_comments(self, key, val, templ_const, templ_val): + """Find parameter definition and accompanying comments, to include in dumped easyconfig file.""" + res = [] + + val = self._reformat_line(key, val, outer=True) + + # templates + if key not in EXCLUDED_KEYS_REPLACE_TEMPLATES: + new_val = to_template_str(val, templ_const, templ_val) + # avoid self-referencing templated parameter definitions + if not r'%(' + key in new_val: + val = new_val + + if key in self.comments['inline']: + res.append("%s = %s%s" % (key, val, self.comments['inline'][key])) + else: + if key in self.comments['above']: + res.extend(self.comments['above'][key]) + res.append("%s = %s" % (key, val)) + + return res + + def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_val): + """ + Determine parameters in the dumped easyconfig file which have a non-default value. + """ + eclines = [] + printed_keys = [] + for group in keyset: + printed = False + for key in group: + # the value for 'dependencies' may have been modified after parsing via filter_hidden_deps + if key == 'dependencies': + val = ecfg[key] + ecfg['hiddendependencies'] + else: + val = ecfg[key] + + if val != default_values[key]: + # dependency easyconfig parameters were parsed, so these need special care to 'unparse' them + if key in DEPENDENCY_PARAMETERS: + valstr = [dump_dependency(d, ecfg['toolchain']) for d in val] + else: + valstr = quote_py_str(ecfg[key]) + + eclines.extend(self._find_param_with_comments(key, valstr, templ_const, templ_val)) + + printed_keys.append(key) + printed = True + if printed: + eclines.append('') + + return eclines, printed_keys + + def dump(self, ecfg, default_values, templ_const, templ_val): + """ + Dump easyconfig in format v1. + + @param ecfg: EasyConfig instance + @param default_values: default values for easyconfig parameters + @param templ_const: known template constants + @param templ_val: known template values + """ + # include header comments first + dump = self.comments['header'][:] + + # print easyconfig parameters ordered and in groups specified above + params, printed_keys = self._find_defined_params(ecfg, GROUPED_PARAMS, default_values, templ_const, templ_val) + dump.extend(params) + + # print other easyconfig parameters at the end + keys_to_ignore = printed_keys + LAST_PARAMS + for key in default_values: + if key not in keys_to_ignore and ecfg[key] != default_values[key]: + dump.extend(self._find_param_with_comments(key, quote_py_str(ecfg[key]), templ_const, templ_val)) + dump.append('') + + # print last parameters + params, _ = self._find_defined_params(ecfg, [[k] for k in LAST_PARAMS], default_values, templ_const, templ_val) + dump.extend(params) + + dump.extend(self.comments['tail']) + + return '\n'.join(dump) + + def extract_comments(self, rawtxt): + """ + Extract comments from raw content. + + Discriminates between comment header, comments above a line (parameter definition), and inline comments. + Inline comments on items of iterable values are also extracted. + """ + self.comments = { + 'above' : {}, # comments for a particular parameter definition + 'header' : [], # header comment lines + 'inline' : {}, # inline comments + 'iter': {}, # (inline) comments on elements of iterable values + 'tail': [], + } + + rawlines = rawtxt.split('\n') + + # extract header first + while rawlines[0].startswith('#'): + self.comments['header'].append(rawlines.pop(0)) + + parsed_ec = self.get_config_dict() + + while rawlines: + rawline = rawlines.pop(0) + if rawline.startswith('#'): + comment = [] + # comment could be multi-line + while rawline is not None and (rawline.startswith('#') or not rawline): + # drop empty lines (that don't even include a #) + if rawline: + comment.append(rawline) + # grab next line (if more lines are left) + if rawlines: + rawline = rawlines.pop(0) + else: + rawline = None + + if rawline is None: + self.comments['tail'] = comment + else: + key = rawline.split('=', 1)[0].strip() + self.comments['above'][key] = comment + + elif '#' in rawline: # inline comment + comment_key, comment_val = None, None + comment = rawline.rsplit('#', 1)[1].strip() + # check whether this line is parameter definition; + # if not, assume it's a continuation of a multi-line value + if re.match(r'^[a-z_]+\s*=', rawline): + comment_key = rawline.split('=', 1)[0].strip() + else: + # determine parameter value where the item value on this line is a part of + for key, val in parsed_ec.items(): + item_val = re.sub(r',$', r'', rawline.rsplit('#', 1)[0].strip()) + if not isinstance(val, basestring) and item_val in str(val): + comment_key, comment_val = key, item_val + break + + # check if hash actually indicated a comment; or is part of the value + if comment_key in parsed_ec: + if comment.replace("'", '').replace('"', '') not in str(parsed_ec[comment_key]): + if comment_val: + self.comments['iter'].setdefault(comment_key, {})[comment_val] = ' # ' + comment + else: + self.comments['inline'][comment_key] = ' # ' + comment + def retrieve_blocks_in_spec(spec, only_blocks, silent=False): """ @@ -102,7 +357,7 @@ def retrieve_blocks_in_spec(spec, only_blocks, silent=False): try: txt = open(spec).read() except IOError, err: - _log.error("Failed to read file %s: %s" % (spec, err)) + raise EasyBuildError("Failed to read file %s: %s", spec, err) # split into blocks using regex pieces = reg_block.split(txt) @@ -124,8 +379,7 @@ def retrieve_blocks_in_spec(spec, only_blocks, silent=False): block_contents = pieces.pop(0) if block_name in [b['name'] for b in blocks]: - msg = "Found block %s twice in %s." % (block_name, spec) - _log.error(msg) + raise EasyBuildError("Found block %s twice in %s.", block_name, spec) block = {'name': block_name, 'contents': block_contents} @@ -157,7 +411,7 @@ def retrieve_blocks_in_spec(spec, only_blocks, silent=False): if 'dependencies' in block: for dep in block['dependencies']: if not dep in [b['name'] for b in blocks]: - _log.error("Block %s depends on %s, but block was not found." % (name, dep)) + raise EasyBuildError("Block %s depends on %s, but block was not found.", name, dep) dep = [b for b in blocks if b['name'] == dep][0] txt += "\n# Dependency block %s" % (dep['name']) diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index 995931f1e1..0dc54d7da6 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -36,6 +36,7 @@ from easybuild.framework.easyconfig.format.format import get_format_version, EasyConfigFormat from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.configobj import ConfigObj from easybuild.tools.systemtools import get_shared_lib_ext @@ -45,11 +46,10 @@ def build_easyconfig_constants_dict(): """Make a dictionary with all constants that can be used""" - # sanity check all_consts = [ ('TEMPLATE_CONSTANTS', dict([(x[0], x[1]) for x in TEMPLATE_CONSTANTS])), ('EASYCONFIG_CONSTANTS', dict([(key, val[0]) for key, val in EASYCONFIG_CONSTANTS.items()])), - ('EASYCONFIG_LICENSES', EASYCONFIG_LICENSES_DICT), + ('EASYCONFIG_LICENSES', dict([(klass().name, name) for name, klass in EASYCONFIG_LICENSES_DICT.items()])), ] err = [] const_dict = {} @@ -68,14 +68,13 @@ def build_easyconfig_constants_dict(): const_dict[cst_key] = cst_val if len(err) > 0: - _log.error("EasyConfig constants sanity check failed: %s" % ("\n".join(err))) + raise EasyBuildError("EasyConfig constants sanity check failed: %s", '\n'.join(err)) else: return const_dict def build_easyconfig_variables_dict(): """Make a dictionary with all variables that can be used""" - _log.deprecated("Magic 'global' easyconfigs variables like shared_lib_ext should no longer be used", '2.0') vars_dict = { "shared_lib_ext": get_shared_lib_ext(), } @@ -129,8 +128,8 @@ def parse(self, txt, strict_section_markers=False): last_n = 100 pre_section_tail = txt[start_section-last_n:start_section] sections_head = txt[start_section:start_section+last_n] - tup = (start_section, last_n, pre_section_tail, sections_head) - self.log.debug('Sections start at index %s, %d-chars context:\n"""%s""""\n\n"""%s..."""' % tup) + self.log.debug('Sections start at index %s, %d-chars context:\n"""%s""""\n\n"""%s..."""', + start_section, last_n, pre_section_tail, sections_head) self.parse_pre_section(txt[:start_section]) if start_section is not None: @@ -149,7 +148,7 @@ def parse_pre_section(self, txt): format_version = get_format_version(line) if format_version is not None: if not format_version == self.VERSION: - self.log.error("Invalid format version %s for current format class" % format_version) + raise EasyBuildError("Invalid format version %s for current format class", format_version) else: self.log.info("Valid format version %s found" % format_version) # version is not part of header @@ -178,10 +177,15 @@ def parse_pyheader(self, pyheader): self.log.debug("pyheader initial local_vars %s" % local_vars) self.log.debug("pyheader text being exec'ed: %s" % pyheader) + # check for use of deprecated magic easyconfigs variables + for magic_var in build_easyconfig_variables_dict(): + if re.search(magic_var, pyheader, re.M): + _log.nosupport("Magic 'global' easyconfigs variable %s should no longer be used" % magic_var, '2.0') + try: exec(pyheader, global_vars, local_vars) except SyntaxError, err: - self.log.error("SyntaxError in easyconfig pyheader %s: %s" % (pyheader, err)) + raise EasyBuildError("SyntaxError in easyconfig pyheader %s: %s", pyheader, err) self.log.debug("pyheader final global_vars %s" % global_vars) self.log.debug("pyheader final local_vars %s" % local_vars) @@ -222,32 +226,33 @@ def pyheader_env(self): def _validate_pyheader(self): """ Basic validation of pyheader localvars. - This takes variable names from the PYHEADER_BLACKLIST and PYHEADER_MANDATORY; - blacklisted variables are not allowed, mandatory variables are - mandatory unless blacklisted + This takes parameter names from the PYHEADER_BLACKLIST and PYHEADER_MANDATORY; + blacklisted parameters are not allowed, mandatory parameters are mandatory unless blacklisted """ if self.pyheader_localvars is None: - self.log.error("self.pyheader_localvars must be initialized") + raise EasyBuildError("self.pyheader_localvars must be initialized") if self.PYHEADER_BLACKLIST is None or self.PYHEADER_MANDATORY is None: - self.log.error('Both PYHEADER_BLACKLIST and PYHEADER_MANDATORY must be set') + raise EasyBuildError('Both PYHEADER_BLACKLIST and PYHEADER_MANDATORY must be set') - for variable in self.PYHEADER_BLACKLIST: - if variable in self.pyheader_localvars: + for param in self.PYHEADER_BLACKLIST: + if param in self.pyheader_localvars: # TODO add to easyconfig unittest (similar to mandatory) - self.log.error('blacklisted variable %s not allowed in pyheader' % variable) + raise EasyBuildError('blacklisted param %s not allowed in pyheader', param) - for variable in self.PYHEADER_MANDATORY: - if variable in self.PYHEADER_BLACKLIST: + missing = [] + for param in self.PYHEADER_MANDATORY: + if param in self.PYHEADER_BLACKLIST: continue - if not variable in self.pyheader_localvars: - # message format in sync with easyconfig mandatory unittest! - self.log.error('mandatory variable %s not provided in pyheader' % variable) + if not param in self.pyheader_localvars: + missing.append(param) + if missing: + raise EasyBuildError('mandatory parameters not provided in pyheader: %s', ', '.join(missing)) def parse_section_block(self, section): """Parse the section block by trying to convert it into a ConfigObj instance""" try: self.configobj = ConfigObj(section.split('\n')) except SyntaxError, err: - self.log.error('Failed to convert section text %s: %s' % (section, err)) + raise EasyBuildError('Failed to convert section text %s: %s', section, err) self.log.debug("Found ConfigObj instance %s" % self.configobj) diff --git a/easybuild/framework/easyconfig/format/two.py b/easybuild/framework/easyconfig/format/two.py index ea48707c99..7e67ba0da2 100644 --- a/easybuild/framework/easyconfig/format/two.py +++ b/easybuild/framework/easyconfig/format/two.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,6 +37,7 @@ from easybuild.framework.easyconfig.format.pyheaderconfigobj import EasyConfigFormatConfigObj from easybuild.framework.easyconfig.format.format import EBConfigObj from easybuild.framework.easyconfig.format.version import EasyVersion, ToolchainVersionOperator, VersionOperator +from easybuild.tools.build_log import EasyBuildError class FormatTwoZero(EasyConfigFormatConfigObj): @@ -89,10 +90,10 @@ def _check_docstring(self): maintainers.append(res['name']) if self.AUTHOR_REQUIRED and not authors: - self.log.error("No author in docstring (regex: '%s')" % self.AUTHOR_DOCSTRING_REGEX.pattern) + raise EasyBuildError("No author in docstring (regex: '%s')", self.AUTHOR_DOCSTRING_REGEX.pattern) if self.MAINTAINER_REQUIRED and not maintainers: - self.log.error("No maintainer in docstring (regex: '%s')" % self.MAINTAINER_DOCSTRING_REGEX.pattern) + raise EasyBuildError("No maintainer in docstring (regex: '%s')", self.MAINTAINER_DOCSTRING_REGEX.pattern) def get_config_dict(self): """Return the best matching easyconfig dict""" @@ -131,3 +132,8 @@ def get_config_dict(self): self.log.debug("Final config dict (including correct version/toolchain): %s" % cfg) return cfg + + def extract_comments(self, rawtxt): + """Extract comments from raw content.""" + # this is fine-ish, it only implies that comments will be lost for format v2 easyconfig files that are dumped + self.log.warning("Extraction of comments not supported yet for easyconfig format v2") diff --git a/easybuild/framework/easyconfig/format/version.py b/easybuild/framework/easyconfig/format/version.py index cb9efe28ce..e4252ce460 100644 --- a/easybuild/framework/easyconfig/format/version.py +++ b/easybuild/framework/easyconfig/format/version.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,6 +34,7 @@ from distutils.version import LooseVersion from vsc.utils import fancylogger +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.utilities import search_toolchain @@ -80,7 +81,7 @@ def __init__(self, versop_str=None, error_on_parse_failure=False): """ Initialise VersionOperator instance. @param versop_str: intialise with version operator string - @param error_on_parse_failure: log.error in case of parse error + @param error_on_parse_failure: raise EasyBuildError in case of parse error """ self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) @@ -100,7 +101,7 @@ def parse_error(self, msg): """Special function to deal with parse errors""" # TODO major issue what to do in case of misparse. error or not? if self.error_on_parse_failure: - self.log.error(msg) + raise EasyBuildError(msg) else: self.log.debug(msg) @@ -122,7 +123,7 @@ def set(self, versop_str): """ versop_dict = self.parse_versop_str(versop_str) if versop_dict is None: - self.log.error("Failed to parse '%s' as a version operator string" % versop_str) + raise EasyBuildError("Failed to parse '%s' as a version operator string", versop_str) else: for k, v in versop_dict.items(): setattr(self, k, v) @@ -136,16 +137,16 @@ def test(self, test_version): """ # checks whether this VersionOperator instance is valid using __bool__ function if not self: - self.log.error('Not a valid %s. Not initialised yet?' % self.__class__.__name__) + raise EasyBuildError('Not a valid %s. Not initialised yet?', self.__class__.__name__) if isinstance(test_version, basestring): test_version = self._convert(test_version) elif not isinstance(test_version, EasyVersion): - self.log.error("test: argument should be a basestring or EasyVersion (type %s)" % (type(test_version))) + raise EasyBuildError("test: argument should be a basestring or EasyVersion (type %s)", type(test_version)) res = self.operator(test_version, self.version) - tup = (test_version, self.REVERSE_OPERATOR_MAP[self.operator], self.version, res) - self.log.debug("result of testing expression '%s %s %s': %s" % tup) + self.log.debug("result of testing expression '%s %s %s': %s", + test_version, self.REVERSE_OPERATOR_MAP[self.operator], self.version, res) return res @@ -178,7 +179,7 @@ def __eq__(self, versop): if versop is None: return False elif not isinstance(versop, self.__class__): - self.log.error("Types don't match in comparison: %s, expected %s" % (type(versop), self.__class__)) + raise EasyBuildError("Types don't match in comparison: %s, expected %s", type(versop), self.__class__) return self.version == versop.version and self.operator == versop.operator and self.suffix == versop.suffix def __ne__(self, versop): @@ -270,7 +271,7 @@ def parse_versop_str(self, versop_str, versop_dict=None): versop_dict['versop_str'] = versop_str if not 'versop_str' in versop_dict: - self.log.error('Missing versop_str in versop_dict %s' % versop_dict) + raise EasyBuildError('Missing versop_str in versop_dict %s', versop_dict) version = self._convert(versop_dict['version_str']) operator = self._convert_operator(versop_dict['operator_str'], version=version) @@ -311,8 +312,8 @@ def test_overlap_and_conflict(self, versop_other): versop_msg = "this versop %s and versop_other %s" % (self, versop_other) if not isinstance(versop_other, self.__class__): - self.log.error('overlap/conflict check needs instance of self %s (got type %s)' % - (self.__class__.__name__, type(versop_other))) + raise EasyBuildError("overlap/conflict check needs instance of self %s (got type %s)", + self.__class__.__name__, type(versop_other)) if self == versop_other: self.log.debug("%s are equal. Return overlap True, conflict False." % versop_msg) @@ -424,12 +425,12 @@ def _gt_safe(self, version_gt_op, versop_other): Suffix are not considered. """ if len(self.ORDERED_OPERATORS) != len(self.OPERATOR_MAP): - self.log.error('Inconsistency between ORDERED_OPERATORS and OPERATORS (lists are not of same length)') + raise EasyBuildError("Inconsistency between ORDERED_OPERATORS and OPERATORS (lists are not of same length)") # ensure this function is only used for non-conflicting version operators _, conflict = self.test_overlap_and_conflict(versop_other) if conflict: - self.log.error("Conflicting version operator expressions should not be compared with _gt_safe") + raise EasyBuildError("Conflicting version operator expressions should not be compared with _gt_safe") ordered_operators = [self.OPERATOR_MAP[x] for x in self.ORDERED_OPERATORS] if self.version == versop_other.version: @@ -530,7 +531,7 @@ def parse_versop_str(self, tcversop_str): tcversop_dict = super(ToolchainVersionOperator, self).parse_versop_str(None, versop_dict=tcversop_dict) if tcversop_dict.get('version_str', None) is not None and tcversop_dict.get('operator_str', None) is None: - self.log.error("Toolchain version found, but no operator (use ' == '?).") + raise EasyBuildError("Toolchain version found, but no operator (use ' == '?).") self.log.debug("toolchain versop expression '%s' parsed to '%s'" % (tcversop_str, tcversop_dict)) return tcversop_dict @@ -552,15 +553,14 @@ def test(self, name, version): """ # checks whether this ToolchainVersionOperator instance is valid using __bool__ function if not self: - self.log.error('Not a valid %s. Not initialised yet?' % self.__class__.__name__) + raise EasyBuildError('Not a valid %s. Not initialised yet?', self.__class__.__name__) tc_name_res = name == self.tc_name if not tc_name_res: self.log.debug('Toolchain name %s different from test toolchain name %s' % (self.tc_name, name)) version_res = super(ToolchainVersionOperator, self).test(version) res = tc_name_res and version_res - tup = (tc_name_res, version_res, res) - self.log.debug("result of testing expression tc_name_res %s version_res %s: %s" % tup) + self.log.debug("result of testing expression tc_name_res %s version_res %s: %s", tc_name_res, version_res, res) return res @@ -622,8 +622,8 @@ def add(self, versop_new, data=None, update=None): if isinstance(versop_new, basestring): versop_new = VersionOperator(versop_new) elif not isinstance(versop_new, VersionOperator): - tup = (versop_new, type(versop_new)) - self.log.error(("add: argument must be a VersionOperator instance or basestring: %s; type %s") % tup) + raise EasyBuildError("add: argument must be a VersionOperator instance or basestring: %s; type %s", + versop_new, type(versop_new)) if versop_new in self.versops: self.log.debug("Versop %s already added." % versop_new) @@ -632,9 +632,9 @@ def add(self, versop_new, data=None, update=None): gt_test = [versop_new > versop for versop in self.versops] if None in gt_test: # conflict - msg = 'add: conflict(s) between versop_new %s and existing versions %s' conflict_versops = [(idx, self.versops[idx]) for idx, gt_val in enumerate(gt_test) if gt_val is None] - self.log.error(msg % (versop_new, conflict_versops)) + raise EasyBuildError("add: conflict(s) between versop_new %s and existing versions %s", + versop_new, conflict_versops) else: if True in gt_test: # determine first element for which comparison is True @@ -655,8 +655,8 @@ def _add_data(self, versop_new, data, update): if update and versop_new_str in self.datamap: self.log.debug("Keeping track of data for %s UPDATE: %s" % (versop_new_str, data)) if not hasattr(self.datamap[versop_new_str], 'update'): - tup = (versop_new_str, type(self.datamap[versop_new_str])) - self.log.error("Can't update on datamap key %s type %s" % tup) + raise EasyBuildError("Can't update on datamap key %s type %s", + versop_new_str, type(self.datamap[versop_new_str])) self.datamap[versop_new_str].update(data) else: self.log.debug("Keeping track of data for %s SET: %s" % (versop_new_str, data)) @@ -665,11 +665,11 @@ def _add_data(self, versop_new, data, update): def get_data(self, versop): """Return the data for versop from datamap""" if not isinstance(versop, VersionOperator): - tup = (versop, type(versop)) - self.log.error(("get_data: argument must be a VersionOperator instance: %s; type %s") % tup) + raise EasyBuildError("get_data: argument must be a VersionOperator instance: %s; type %s", + versop, type(versop)) versop_str = str(versop) if versop_str in self.datamap: return self.datamap[versop_str] else: - self.log.error('No data in datamap for versop %s' % versop) + raise EasyBuildError("No data in datamap for versop %s", versop) diff --git a/easybuild/framework/easyconfig/licenses.py b/easybuild/framework/easyconfig/licenses.py index ce49c3d5dc..5ef1959b17 100644 --- a/easybuild/framework/easyconfig/licenses.py +++ b/easybuild/framework/easyconfig/licenses.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,11 +28,13 @@ be used within an Easyconfig file. @author: Stijn De Weirdt (Ghent University) +@author: Kenneth Hoste (Ghent University) """ from vsc.utils import fancylogger from vsc.utils.missing import get_subclasses + _log = fancylogger.getLogger('easyconfig.licenses', fname=False) @@ -51,14 +53,20 @@ class License(object): CLASSNAME_PREFIX = 'License' - def __init__(self): + @property + def name(self): + """Return license name.""" if self.NAME is None: name = self.__class__.__name__ if name.startswith(self.CLASSNAME_PREFIX): name = name[len(self.CLASSNAME_PREFIX):] else: name = self.NAME - self.name = name + + return name + + def __init__(self): + """License constructor.""" self.version = self.VERSION self.description = self.DESCRIPTION self.distribute_source = self.DISTRIBUTE_SOURCE @@ -66,12 +74,12 @@ def __init__(self): self.group_binary = self.GROUP_BINARY -class VeryRestrictive(License): +class LicenseVeryRestrictive(License): """Default license should be very restrictive, so nothing to do here, just a placeholder""" pass -class LicenseUnknown(VeryRestrictive): +class LicenseUnknown(LicenseVeryRestrictive): """A (temporary) license, could be used as default in case nothing was specified""" pass @@ -153,24 +161,26 @@ def what_licenses(): for lic in get_subclasses(License): if lic.HIDDEN: continue - lic_instance = lic() - res[lic_instance.name] = lic_instance + res[lic.__name__] = lic return res EASYCONFIG_LICENSES_DICT = what_licenses() -EASYCONFIG_LICENSES = EASYCONFIG_LICENSES_DICT.keys() def license_documentation(): """Generate the easyconfig licenses documentation""" - indent_l0 = " " * 2 - indent_l1 = indent_l0 + " " * 2 + indent_l0 = ' ' * 2 + indent_l1 = indent_l0 + ' ' * 2 doc = [] doc.append("Constants that can be used in easyconfigs") for lic_name, lic in EASYCONFIG_LICENSES_DICT.items(): - doc.append('%s%s: %s (version %s)' % (indent_l1, lic_name, lic.description, lic.version)) + lic_inst = lic() + strver = '' + if lic_inst.version: + strver = " (version: %s)" % '.'.join([str(d) for d in lic_inst.version]) + doc.append("%s%s: %s%s" % (indent_l1, lic_inst.name, lic_inst.description, strver)) - return "\n".join(doc) + return '\n'.join(doc) diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py index f3ab56996b..e5a6d1a341 100644 --- a/easybuild/framework/easyconfig/parser.py +++ b/easybuild/framework/easyconfig/parser.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,42 +30,113 @@ @author: Stijn De Weirdt (Ghent University) """ import os +import re from vsc.utils import fancylogger from easybuild.framework.easyconfig.format.format import FORMAT_DEFAULT_VERSION from easybuild.framework.easyconfig.format.format import get_format_version, get_format_version_classes +from easybuild.framework.easyconfig.types import TYPES, check_type_of_param_value +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file, write_file +# deprecated easyconfig parameters, and their replacements +DEPRECATED_PARAMETERS = { + # : (, ), +} + +# replaced easyconfig parameters, and their replacements +REPLACED_PARAMETERS = { + 'license': 'license_file', + 'makeopts': 'buildopts', + 'premakeopts': 'prebuildopts', +} + + _log = fancylogger.getLogger('easyconfig.parser', fname=False) +def fetch_parameters_from_easyconfig(rawtxt, params): + """ + Fetch (initial) parameter definition from the given easyconfig file contents. + @param rawtxt: contents of the easyconfig file + @param params: list of parameter names to fetch values for + """ + param_values = [] + for param in params: + regex = re.compile(r"^\s*%s\s*=\s*(?P\S.*?)\s*$" % param, re.M) + res = regex.search(rawtxt) + if res: + param_values.append(res.group('param').strip("'\"")) + else: + param_values.append(None) + _log.debug("Obtained parameters value for %s: %s" % (params, param_values)) + return param_values + + class EasyConfigParser(object): """Read the easyconfig file, return a parsed config object Can contain references to multiple version and toolchain/toolchain versions """ - def __init__(self, filename=None, format_version=None): - """Initialise the EasyConfigParser class""" + def __init__(self, filename=None, format_version=None, rawcontent=None, + auto_convert_value_types=True): + """ + Initialise the EasyConfigParser class + @param filename: path to easyconfig file to parse (superseded by rawcontent, if specified) + @param format_version: version of easyconfig file format, used to determine how to parse supplied easyconfig + @param rawcontent: raw content of easyconfig file to parse (preferred over easyconfig file supplied via filename) + @param auto_convert_value_types: indicates whether types of easyconfig values should be automatically converted + in case they are wrong + """ self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) self.rawcontent = None # the actual unparsed content + self.auto_convert = auto_convert_value_types + self.get_fn = None # read method and args self.set_fn = None # write method and args self.format_version = format_version self._formatter = None - - if filename is not None: + if rawcontent is not None: + self.rawcontent = rawcontent + self._set_formatter() + elif filename is not None: self._check_filename(filename) self.process() + else: + raise EasyBuildError("Neither filename nor rawcontent provided to EasyConfigParser") + + self._formatter.extract_comments(self.rawcontent) def process(self, filename=None): """Create an instance""" self._read(filename=filename) self._set_formatter() + def check_values_types(self, cfg): + """ + Check types of easyconfig parameter values. + + @param cfg: dictionary with easyconfig parameter values (result of get_config_dict()) + """ + wrong_type_msgs = [] + for key in cfg: + type_ok, newval = check_type_of_param_value(key, cfg[key], self.auto_convert) + if not type_ok: + wrong_type_msgs.append("value for '%s' should be of type '%s'" % (key, TYPES[key].__name__)) + elif newval != cfg[key]: + self.log.warning("Value for '%s' easyconfig parameter was converted from %s (type: %s) to %s (type: %s)", + key, cfg[key], type(cfg[key]), newval, type(newval)) + cfg[key] = newval + + if wrong_type_msgs: + raise EasyBuildError("Type checking of easyconfig parameter values failed: %s", ', '.join(wrong_type_msgs)) + else: + self.log.info("Type checking of easyconfig parameter values passed!") + def _check_filename(self, fn): """Perform sanity check on the filename, and set mechanism to set the content of the file""" if os.path.isfile(fn): @@ -75,9 +146,9 @@ def _check_filename(self, fn): self.log.debug("Process filename %s with get function %s, set function %s" % (fn, self.get_fn, self.set_fn)) if self.get_fn is None: - self.log.error('Failed to determine get function for filename %s' % fn) + raise EasyBuildError('Failed to determine get function for filename %s', fn) if self.set_fn is None: - self.log.error('Failed to determine set function for filename %s' % fn) + raise EasyBuildError('Failed to determine set function for filename %s', fn) def _read(self, filename=None): """Read the easyconfig, dump content in self.rawcontent""" @@ -87,11 +158,11 @@ def _read(self, filename=None): try: self.rawcontent = self.get_fn[0](*self.get_fn[1]) except IOError, err: - self.log.error('Failed to obtain content with %s: %s' % (self.get_fn, err)) + raise EasyBuildError('Failed to obtain content with %s: %s', self.get_fn, err) if not isinstance(self.rawcontent, basestring): msg = 'rawcontent is not basestring: type %s, content %s' % (type(self.rawcontent), self.rawcontent) - self.log.error("Unexpected result for raw content: %s" % msg) + raise EasyBuildError("Unexpected result for raw content: %s", msg) def _det_format_version(self): """Extract the format version from the raw content""" @@ -109,10 +180,10 @@ def _get_format_version_class(self): if len(found_classes) == 1: return found_classes[0] elif not found_classes: - self.log.error('No format classes found matching version %s' % self.format_version) + raise EasyBuildError('No format classes found matching version %s', self.format_version) else: - msg = 'More than one format class found matching version %s in %s' % (self.format_version, found_classes) - self.log.error(msg) + raise EasyBuildError("More than one format class found matching version %s in %s", + self.format_version, found_classes) def _set_formatter(self): """Obtain instance of the formatter""" @@ -134,7 +205,7 @@ def write(self, filename=None): try: self.set_fn[0](*self.set_fn[1]) except IOError, err: - self.log.error('Failed to process content with %s: %s' % (self.set_fn, err)) + raise EasyBuildError("Failed to process content with %s: %s", self.set_fn, err) def set_specifications(self, specs): """Set specifications.""" @@ -145,4 +216,12 @@ def get_config_dict(self, validate=True): # allows to bypass the validation step, typically for testing if validate: self._formatter.validate() - return self._formatter.get_config_dict() + + cfg = self._formatter.get_config_dict() + self.check_values_types(cfg) + + return cfg + + def dump(self, ecfg, default_values, templ_const, templ_val): + """Dump easyconfig in format it was parsed from.""" + return self._formatter.dump(ecfg, default_values, templ_const, templ_val) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 42ec1038f8..7cdfcfc9b4 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,12 +28,13 @@ be used within an Easyconfig file. @author: Stijn De Weirdt (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ - +import re from vsc.utils import fancylogger from distutils.version import LooseVersion +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.systemtools import get_shared_lib_ext @@ -87,9 +88,9 @@ 'googlecode.com source url'), ('LAUNCHPAD_SOURCE', 'https://launchpad.net/%(namelower)s/%(version_major_minor)s.x/%(version)s/+download/', 'launchpad.net source url'), - ('PYPI_SOURCE', 'http://pypi.python.org/packages/source/%(nameletter)s/%(name)s', + ('PYPI_SOURCE', 'https://pypi.python.org/packages/source/%(nameletter)s/%(name)s', 'pypi source url'), # e.g., Cython, Sphinx - ('PYPI_LOWER_SOURCE', 'http://pypi.python.org/packages/source/%(nameletterlower)s/%(namelower)s', + ('PYPI_LOWER_SOURCE', 'https://pypi.python.org/packages/source/%(nameletterlower)s/%(namelower)s', 'pypi source url (lowercase name)'), # e.g., Greenlet, PyZMQ ('R_SOURCE', 'http://cran.r-project.org/src/base/R-%(version_major)s', 'cran.r-project.org (base) source url'), @@ -113,7 +114,7 @@ ('SHLIB_EXT', get_shared_lib_ext(), 'extension for shared libraries'), ] -extensions = ['tar.gz', 'tar.xz', 'tar.bz2', 'tgz', 'txz', 'tbz2', 'tb2', 'gtgz', 'zip', 'tar', 'xz'] +extensions = ['tar.gz', 'tar.xz', 'tar.bz2', 'tgz', 'txz', 'tbz2', 'tb2', 'gtgz', 'zip', 'tar', 'xz', 'tar.Z'] for ext in extensions: suffix = ext.replace('.', '_').upper() TEMPLATE_CONSTANTS += [ @@ -124,6 +125,7 @@ # TODO derived config templates # versionmajor, versionminor, versionmajorminor (eg '.'.join(version.split('.')[:2])) ) + def template_constant_dict(config, ignore=None, skip_lower=True): """Create a dict for templating the values in the easyconfigs. - config is a dict with the structure of EasyConfig._config @@ -175,7 +177,7 @@ def template_constant_dict(config, ignore=None, skip_lower=True): if softname is not None: template_values['nameletter'] = softname[0] else: - _log.error("Undefined name %s from TEMPLATE_NAMES_EASYCONFIG" % name) + raise EasyBuildError("Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name) # step 2: add remaining from config for name in TEMPLATE_NAMES_CONFIG: @@ -194,7 +196,7 @@ def template_constant_dict(config, ignore=None, skip_lower=True): if t_v is None: continue try: - template_values[TEMPLATE_NAMES_LOWER_TEMPLATE % {'name':name}] = t_v.lower() + template_values[TEMPLATE_NAMES_LOWER_TEMPLATE % {'name': name}] = t_v.lower() except: _log.debug("_getitem_string: can't get .lower() for name %s value %s (type %s)" % (name, t_v, type(t_v))) @@ -202,6 +204,29 @@ def template_constant_dict(config, ignore=None, skip_lower=True): return template_values +def to_template_str(value, templ_const, templ_val): + """ + Insert template values where possible + - value is a string + - templ_const is a dictionary of template strings (constants) + - templ_val is an ordered dictionary of template strings specific for this easyconfig file + """ + old_value = None + while value != old_value: + old_value = value + # check for constant values + for tval, tname in templ_const.items(): + value = re.sub(r'(^|\W)' + re.escape(tval) + r'(\W|$)', r'\1' + tname + r'\2', value) + + for tval, tname in templ_val.items(): + # only replace full words with templates: word to replace should be at the beginning of a line + # or be preceded by a non-alphanumeric (\W). It should end at the end of a line or be succeeded + # by another non-alphanumeric. + value = re.sub(r'(^|\W)' + re.escape(tval) + r'(\W|$)', r'\1%(' + tname + r')s\2', value) + + return value + + def template_documentation(): """Generate the templating documentation""" # This has to reflect the methods/steps used in easyconfig _generate_template_values diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index d834141ab8..abfd9c22d0 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -33,81 +33,101 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) @author: Toon Willems (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) +@author: Fotis Georgatos (Uni.Lu, NTUA) +@author: Ward Poelmans (Ghent University) """ - +import glob import os +import re import sys +import tempfile +from distutils.version import LooseVersion from vsc.utils import fancylogger +from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR +from easybuild.framework.easyconfig.easyconfig import ActiveMNS, create_paths, process_easyconfig +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option +from easybuild.tools.filetools import find_easyconfigs, which, write_file +from easybuild.tools.github import fetch_easyconfigs_from_pr, download_repo +from easybuild.tools.modules import modules_tool +from easybuild.tools.multidiff import multidiff +from easybuild.tools.ordereddict import OrderedDict +from easybuild.tools.run import run_cmd +from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME +from easybuild.tools.utilities import only_if_module_is_available, quote_str + # optional Python packages, these might be missing # failing imports are just ignored # a NameError should be catched where these are used -# PyGraph (used for generating dependency graphs) -graph_errors = [] try: + # PyGraph (used for generating dependency graphs) + # https://pypi.python.org/pypi/python-graph-core from pygraph.classes.digraph import digraph -except ImportError, err: - graph_errors.append("Failed to import pygraph-core: try easy_install python-graph-core") - -try: + # https://pypi.python.org/pypi/python-graph-dot import pygraph.readwrite.dot as dot -except ImportError, err: - graph_errors.append("Failed to import pygraph-dot: try easy_install python-graph-dot") - -# graphviz (used for creating dependency graph images) -try: + # graphviz (used for creating dependency graph images) sys.path.append('..') sys.path.append('/usr/lib/graphviz/python/') sys.path.append('/usr/lib64/graphviz/python/') + # https://pypi.python.org/pypi/pygraphviz + # graphviz-python (yum) or python-pygraphviz (apt-get) + # or brew install graphviz --with-bindings (OS X) import gv -except ImportError, err: - graph_errors.append("Failed to import graphviz: try yum install graphviz-python, or apt-get install python-pygraphviz") +except ImportError: + pass -from easybuild.framework.easyconfig.easyconfig import ActiveMNS -from easybuild.framework.easyconfig.easyconfig import process_easyconfig, robot_find_easyconfig -from easybuild.tools.build_log import EasyBuildError, print_msg -from easybuild.tools.config import build_option -from easybuild.tools.filetools import det_common_path_prefix, run_cmd, write_file -from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS -from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version -from easybuild.tools.modules import modules_tool -from easybuild.tools.ordereddict import OrderedDict _log = fancylogger.getLogger('easyconfig.tools', fname=False) -def skip_available(easyconfigs, testing=False): - """Skip building easyconfigs for which a module is already available.""" - avail_modules = modules_tool().available() - easyconfigs, check_easyconfigs = [], easyconfigs - for ec in check_easyconfigs: - module = ec['full_mod_name'] - if module in avail_modules: - msg = "%s is already installed (module found), skipping" % module - print_msg(msg, log=_log, silent=testing) - _log.info(msg) +def skip_available(easyconfigs): + """Skip building easyconfigs for existing modules.""" + modtool = modules_tool() + module_names = [ec['full_mod_name'] for ec in easyconfigs] + modules_exist = modtool.exist(module_names) + retained_easyconfigs = [] + for ec, mod_name, mod_exists in zip(easyconfigs, module_names, modules_exist): + if mod_exists: + _log.info("%s is already installed (module found), skipping" % mod_name) else: - _log.debug("%s is not installed yet, so retaining it" % module) - easyconfigs.append(ec) - return easyconfigs + _log.debug("%s is not installed yet, so retaining it" % mod_name) + retained_easyconfigs.append(ec) + return retained_easyconfigs -def find_resolved_modules(unprocessed, avail_modules): +def find_resolved_modules(unprocessed, avail_modules, retain_all_deps=False): """ Find easyconfigs in 1st argument which can be fully resolved using modules specified in 2nd argument """ ordered_ecs = [] new_avail_modules = avail_modules[:] new_unprocessed = [] + modtool = modules_tool() for ec in unprocessed: new_ec = ec.copy() deps = [] for dep in new_ec['dependencies']: - if not ActiveMNS().det_full_module_name(dep) in new_avail_modules: - deps.append(dep) + full_mod_name = dep.get('full_mod_name', None) + if full_mod_name is None: + full_mod_name = ActiveMNS().det_full_module_name(dep) + + dep_resolved = full_mod_name in new_avail_modules + if not retain_all_deps: + # hidden modules need special care, since they may not be included in list of available modules + dep_resolved |= dep['hidden'] and modtool.exist([full_mod_name])[0] + + if not dep_resolved: + # treat external modules as resolved when retain_all_deps is enabled (e.g., under --dry-run), + # since no corresponding easyconfig can be found for them + if retain_all_deps and dep.get('external_module', False): + _log.debug("Treating dependency marked as external dependency as resolved: %s", dep) + else: + # no module available (yet) => retain dependency as one to be resolved + deps.append(dep) + new_ec['dependencies'] = deps if len(new_ec['dependencies']) == 0: @@ -121,174 +141,11 @@ def find_resolved_modules(unprocessed, avail_modules): return ordered_ecs, new_unprocessed, new_avail_modules -def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False): - """ - Work through the list of easyconfigs to determine an optimal order - @param unprocessed: list of easyconfigs - @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) - """ - - robot = build_option('robot_path') - - retain_all_deps = build_option('retain_all_deps') or retain_all_deps - if retain_all_deps: - # assume that no modules are available when forced, to retain all dependencies - avail_modules = [] - _log.info("Forcing all dependencies to be retained.") - else: - # Get a list of all available modules (format: [(name, installversion), ...]) - avail_modules = modules_tool().available() - - if len(avail_modules) == 0: - _log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH')) - - ordered_ecs = [] - # all available modules can be used for resolving dependencies except those that will be installed - being_installed = [p['full_mod_name'] for p in unprocessed] - avail_modules = [m for m in avail_modules if not m in being_installed] - - _log.debug('unprocessed before resolving deps: %s' % unprocessed) - - # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though) - irresolvable = [] - loopcnt = 0 - maxloopcnt = 10000 - while unprocessed: - # make sure this stops, we really don't want to get stuck in an infinite loop - loopcnt += 1 - if loopcnt > maxloopcnt: - tup = (maxloopcnt, unprocessed, irresolvable) - msg = "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)" % tup - _log.error(msg) - - # first try resolving dependencies without using external dependencies - last_processed_count = -1 - while len(avail_modules) > last_processed_count: - last_processed_count = len(avail_modules) - more_ecs, unprocessed, avail_modules = find_resolved_modules(unprocessed, avail_modules) - for ec in more_ecs: - if not ec['full_mod_name'] in [x['full_mod_name'] for x in ordered_ecs]: - ordered_ecs.append(ec) - - # robot: look for existing dependencies, add them - if robot and unprocessed: - - # rely on EasyBuild module naming scheme when resolving dependencies, since we know that will - # generate sensible module names that include the necessary information for the resolution to work - # (name, version, toolchain, versionsuffix) - being_installed = [EasyBuildMNS().det_full_module_name(p['ec']) for p in unprocessed] - - additional = [] - for i, entry in enumerate(unprocessed): - # do not choose an entry that is being installed in the current run - # if they depend, you probably want to rebuild them using the new dependency - deps = entry['dependencies'] - candidates = [d for d in deps if not EasyBuildMNS().det_full_module_name(d) in being_installed] - if len(candidates) > 0: - cand_dep = candidates[0] - # find easyconfig, might not find any - _log.debug("Looking for easyconfig for %s" % str(cand_dep)) - # note: robot_find_easyconfig may return None - path = robot_find_easyconfig(cand_dep['name'], det_full_ec_version(cand_dep)) - - if path is None: - # no easyconfig found for dependency, add to list of irresolvable dependencies - if cand_dep not in irresolvable: - _log.debug("Irresolvable dependency found: %s" % cand_dep) - irresolvable.append(cand_dep) - # remove irresolvable dependency from list of dependencies so we can continue - entry['dependencies'].remove(cand_dep) - else: - _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) - # build specs should not be passed down to resolved dependencies, - # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself - processed_ecs = process_easyconfig(path, validate=not retain_all_deps) - - # ensure that selected easyconfig provides required dependency - mods = [spec['ec'].full_mod_name for spec in processed_ecs] - dep_mod_name = ActiveMNS().det_full_module_name(cand_dep) - if not dep_mod_name in mods: - tup = (path, dep_mod_name, mods) - _log.error("easyconfig file %s does not contain module %s (mods: %s)" % tup) - - for ec in processed_ecs: - if not ec in unprocessed + additional: - additional.append(ec) - _log.debug("Added %s as dependency of %s" % (ec, entry)) - else: - mod_name = EasyBuildMNS().det_full_module_name(entry['ec']) - _log.debug("No more candidate dependencies to resolve for %s" % mod_name) - - # add additional (new) easyconfigs to list of stuff to process - unprocessed.extend(additional) - - elif not robot: - # no use in continuing if robot is not enabled, dependencies won't be resolved anyway - irresolvable = [dep for x in unprocessed for dep in x['dependencies']] - break - - if irresolvable: - _log.warning("Irresolvable dependencies (details): %s" % irresolvable) - irresolvable_mods_eb = [EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable] - _log.warning("Irresolvable dependencies (EasyBuild module names): %s" % ', '.join(irresolvable_mods_eb)) - irresolvable_mods = [ActiveMNS().det_full_module_name(dep) for dep in irresolvable] - _log.error('Irresolvable dependencies encountered: %s' % ', '.join(irresolvable_mods)) - - _log.info("Dependency resolution complete, building as follows:\n%s" % ordered_ecs) - return ordered_ecs - - -def print_dry_run(easyconfigs, short=False, build_specs=None): - """ - Print dry run information - @param easyconfigs: list of easyconfig files - @param short: print short output (use a variable for the common prefix) - @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) - """ - lines = [] - if build_option('robot_path') is None: - lines.append("Dry run: printing build status of easyconfigs") - all_specs = easyconfigs - else: - lines.append("Dry run: printing build status of easyconfigs and dependencies") - all_specs = resolve_dependencies(easyconfigs, build_specs=build_specs, retain_all_deps=True) - - unbuilt_specs = skip_available(all_specs, testing=True) - dry_run_fmt = " * [%1s] %s (module: %s)" # markdown compatible (list of items with checkboxes in front) - - var_name = 'CFGS' - common_prefix = det_common_path_prefix([spec['spec'] for spec in all_specs]) - # only allow short if common prefix is long enough - short = short and common_prefix is not None and len(common_prefix) > len(var_name) * 2 - for spec in all_specs: - if spec in unbuilt_specs: - ans = ' ' - else: - ans = 'x' - - if spec['ec'].short_mod_name != spec['ec'].full_mod_name: - mod = "%s | %s" % (spec['ec'].mod_subdir, spec['ec'].short_mod_name) - else: - mod = spec['ec'].full_mod_name - - if short: - item = os.path.join('$%s' % var_name, spec['spec'][len(common_prefix) + 1:]) - else: - item = spec['spec'] - lines.append(dry_run_fmt % (ans, item, mod)) - - if short: - # insert after 'Dry run:' message - lines.insert(1, "%s=%s" % (var_name, common_prefix)) - silent = build_option('silent') - print_msg('\n'.join(lines), log=_log, silent=silent, prefix=False) - - -def _dep_graph(fn, specs, silent=False): +@only_if_module_is_available('pygraph.classes.digraph', pkgname='python-graph-core') +def dep_graph(filename, specs): """ Create a dependency graph for the given easyconfigs. """ - # check whether module names are unique # if so, we can omit versions in the graph names = set() @@ -297,48 +154,58 @@ def _dep_graph(fn, specs, silent=False): omit_versions = len(names) == len(specs) def mk_node_name(spec): - if omit_versions: - return spec['name'] + if spec.get('external_module', False): + node_name = "%s (EXT)" % spec['full_mod_name'] + elif omit_versions: + node_name = spec['name'] else: - return ActiveMNS().det_full_module_name(spec) + node_name = ActiveMNS().det_full_module_name(spec) + + return node_name # enhance list of specs + all_nodes = set() for spec in specs: spec['module'] = mk_node_name(spec['ec']) - spec['unresolved_deps'] = [mk_node_name(s) for s in spec['unresolved_deps']] + all_nodes.add(spec['module']) + spec['ec'].all_dependencies = [mk_node_name(s) for s in spec['ec'].all_dependencies] + all_nodes.update(spec['ec'].all_dependencies) # build directed graph dgr = digraph() - dgr.add_nodes([spec['module'] for spec in specs]) + dgr.add_nodes(all_nodes) for spec in specs: - for dep in spec['unresolved_deps']: + for dep in spec['ec'].all_dependencies: dgr.add_edge((spec['module'], dep)) + _dep_graph_dump(dgr, filename) + + if not build_option('silent'): + print "Wrote dependency graph for %d easyconfigs to %s" % (len(specs), filename) + + +@only_if_module_is_available('pygraph.readwrite.dot', pkgname='python-graph-dot') +def _dep_graph_dump(dgr, filename): + """Dump dependency graph to file, in specified format.""" # write to file dottxt = dot.write(dgr) - if fn.endswith(".dot"): + if os.path.splitext(filename)[-1] == '.dot': # create .dot file - write_file(fn, dottxt) + write_file(filename, dottxt) else: - # try and render graph in specified file format - gvv = gv.readstring(dottxt) - gv.layout(gvv, 'dot') - gv.render(gvv, fn.split('.')[-1], fn) + _dep_graph_gv(dottxt, filename) - if not silent: - print "Wrote dependency graph for %d easyconfigs to %s" % (len(specs), fn) +@only_if_module_is_available('gv', pkgname='graphviz') +def _dep_graph_gv(dottxt, filename): + """Render dependency graph to file using graphviz.""" + # try and render graph in specified file format + gvv = gv.readstring(dottxt) + gv.layout(gvv, 'dot') + gv.render(gvv, os.path.splitext(filename)[-1], filename) -def dep_graph(*args, **kwargs): - try: - _dep_graph(*args, **kwargs) - except NameError, err: - errors = "\n".join(graph_errors) - msg = "An optional Python packages required to generate dependency graphs is missing: %s" % errors - _log.error("%s\nerr: %s" % (msg, err)) - -def get_paths_for(subdir="easyconfigs", robot_path=None): +def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None): """ Return a list of absolute paths where the specified subdir can be found, determined by the PYTHONPATH """ @@ -355,19 +222,19 @@ def get_paths_for(subdir="easyconfigs", robot_path=None): path_list.extend(sys.path) # figure out installation prefix, e.g. distutils install path for easyconfigs - (out, ec) = run_cmd("which eb", simple=False, log_all=False, log_ok=False) - if ec: - _log.warning("eb not found (%s), failed to determine installation prefix" % out) + eb_path = which('eb') + if eb_path is None: + _log.warning("'eb' not found in $PATH, failed to determine installation prefix") else: # eb should reside in /bin/eb - install_prefix = os.path.dirname(os.path.dirname(out)) + install_prefix = os.path.dirname(os.path.dirname(eb_path)) path_list.append(install_prefix) _log.debug("Also considering installation prefix %s..." % install_prefix) # look for desired subdirs for path in path_list: path = os.path.join(path, "easybuild", subdir) - _log.debug("Looking for easybuild/%s in path %s" % (subdir, path)) + _log.debug("Checking for easybuild/%s at %s" % (subdir, path)) try: if os.path.exists(path): paths.append(os.path.abspath(path)) @@ -378,25 +245,211 @@ def get_paths_for(subdir="easyconfigs", robot_path=None): return paths +def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_pr=False): + """Obtain alternative paths for easyconfig files.""" + # path where tweaked easyconfigs will be placed + tweaked_ecs_path = None + if tweaked_ecs: + tweaked_ecs_path = os.path.join(tmpdir, 'tweaked_easyconfigs') + + # path where files touched in PR will be downloaded to + pr_path = None + if from_pr: + pr_path = os.path.join(tmpdir, "files_pr%s" % from_pr) + + return tweaked_ecs_path, pr_path + + +def det_easyconfig_paths(orig_paths): + """ + Determine paths to easyconfig files. + @param orig_paths: list of original easyconfig paths + @return: list of paths to easyconfig files + """ + from_pr = build_option('from_pr') + robot_path = build_option('robot_path') + + # list of specified easyconfig files + ec_files = orig_paths[:] + + if from_pr is not None: + pr_files = fetch_easyconfigs_from_pr(from_pr) + + if ec_files: + # replace paths for specified easyconfigs that are touched in PR + for i, ec_file in enumerate(ec_files): + for pr_file in pr_files: + if ec_file == os.path.basename(pr_file): + ec_files[i] = pr_file + else: + # if no easyconfigs are specified, use all the ones touched in the PR + ec_files = [path for path in pr_files if path.endswith('.eb')] + + if ec_files and robot_path: + # look for easyconfigs with relative paths in robot search path, + # unless they were found at the given relative paths + + # determine which easyconfigs files need to be found, if any + ecs_to_find = [] + for idx, ec_file in enumerate(ec_files): + if ec_file == os.path.basename(ec_file) and not os.path.exists(ec_file): + ecs_to_find.append((idx, ec_file)) + _log.debug("List of easyconfig files to find: %s" % ecs_to_find) + + # find missing easyconfigs by walking paths in robot search path + for path in robot_path: + _log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path)) + for (subpath, dirnames, filenames) in os.walk(path, topdown=True): + for idx, orig_path in ecs_to_find[:]: + if orig_path in filenames: + full_path = os.path.join(subpath, orig_path) + _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) + ec_files[idx] = full_path + # if file was found, stop looking for it (first hit wins) + ecs_to_find.remove((idx, orig_path)) + + # stop os.walk insanity as soon as we have all we need (os.walk loop) + if not ecs_to_find: + break + + # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk + dirnames[:] = [d for d in dirnames if d not in build_option('ignore_dirs')] + + # stop os.walk insanity as soon as we have all we need (outer loop) + if not ecs_to_find: + break + + return ec_files + + +def parse_easyconfigs(paths, validate=True): + """ + Parse easyconfig files + @params paths: paths to easyconfigs + """ + easyconfigs = [] + generated_ecs = False + for (path, generated) in paths: + path = os.path.abspath(path) + # keep track of whether any files were generated + generated_ecs |= generated + if not os.path.exists(path): + raise EasyBuildError("Can't find path %s", path) + try: + ec_files = find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs')) + for ec_file in ec_files: + # only pass build specs when not generating easyconfig files + kwargs = {'validate': validate} + if not build_option('try_to_generate'): + kwargs['build_specs'] = build_option('build_specs') + ecs = process_easyconfig(ec_file, **kwargs) + easyconfigs.extend(ecs) + except IOError, err: + raise EasyBuildError("Processing easyconfigs in path %s failed: %s", path, err) + + return easyconfigs, generated_ecs + + def stats_to_str(stats): """ Pretty print build statistics to string. """ if not isinstance(stats, (OrderedDict, dict)): - _log.error("Can only pretty print build stats in dictionary form, not of type %s" % type(stats)) + raise EasyBuildError("Can only pretty print build stats in dictionary form, not of type %s", type(stats)) txt = "{\n" - pref = " " + for (k, v) in stats.items(): + txt += "%s%s: %s,\n" % (pref, quote_str(k), quote_str(v)) + txt += "}" + return txt + + +def find_related_easyconfigs(path, ec): + """ + Find related easyconfigs for provided parsed easyconfig in specified path. + + A list of easyconfigs for the same software (name) is returned, + matching the 1st criterion that yields a non-empty list. - def tostr(x): - if isinstance(x, basestring): - return "'%s'" % x + The following criteria are considered (in this order) next to common software version criterion, i.e. + exact version match, a major/minor version match, a major version match, or no version match (in that order). + + (i) matching versionsuffix and toolchain name/version + (ii) matching versionsuffix and toolchain name (any toolchain version) + (iii) matching versionsuffix (any toolchain name/version) + (iv) matching toolchain name/version (any versionsuffix) + (v) matching toolchain name (any versionsuffix, toolchain version) + (vi) no extra requirements (any versionsuffix, toolchain name/version) + + If no related easyconfigs with a matching software name are found, an empty list is returned. + """ + name = ec.name + version = ec.version + versionsuffix = ec['versionsuffix'] + toolchain_name = ec['toolchain']['name'] + toolchain_name_pattern = r'-%s-\S+' % toolchain_name + toolchain_pattern = '-%s-%s' % (toolchain_name, ec['toolchain']['version']) + if toolchain_name == DUMMY_TOOLCHAIN_NAME: + toolchain_name_pattern = '' + toolchain_pattern = '' + + potential_paths = [glob.glob(ec_path) for ec_path in create_paths(path, name, '*')] + potential_paths = sum(potential_paths, []) # flatten + _log.debug("found these potential paths: %s" % potential_paths) + + parsed_version = LooseVersion(version).version + version_patterns = [version] # exact version match + if len(parsed_version) >= 2: + version_patterns.append(r'%s\.%s\.\w+' % tuple(parsed_version[:2])) # major/minor version match + if parsed_version != parsed_version[0]: + version_patterns.append(r'%s\.[\d-]+\.\w+' % parsed_version[0]) # major version match + version_patterns.append(r'[\w.]+') # any version + + regexes = [] + for version_pattern in version_patterns: + common_pattern = r'^\S+/%s-%s%%s\.eb$' % (name, version_pattern) + regexes.extend([ + common_pattern % (toolchain_pattern + versionsuffix), + common_pattern % (toolchain_name_pattern + versionsuffix), + common_pattern % (r'\S*%s' % versionsuffix), + common_pattern % toolchain_pattern, + common_pattern % toolchain_name_pattern, + common_pattern % r'\S*', + ]) + + for regex in regexes: + res = [p for p in potential_paths if re.match(regex, p)] + if res: + _log.debug("Related easyconfigs found using '%s': %s" % (regex, res)) + break else: - return str(x) + _log.debug("No related easyconfigs in potential paths using '%s'" % regex) - for (k, v) in stats.items(): - txt += "%s%s: %s,\n" % (pref, tostr(k), tostr(v)) + return sorted(res) - txt += "}" - return txt + +def review_pr(pr, colored=True, branch='develop'): + """ + Print multi-diff overview between easyconfigs in specified PR and specified branch. + @param pr: pull request number in easybuild-easyconfigs repo to review + @param colored: boolean indicating whether a colored multi-diff should be generated + @param branch: easybuild-easyconfigs branch to compare with + """ + tmpdir = tempfile.mkdtemp() + + download_repo_path = download_repo(branch=branch, path=tmpdir) + repo_path = os.path.join(download_repo_path, 'easybuild', 'easyconfigs') + pr_files = [path for path in fetch_easyconfigs_from_pr(pr) if path.endswith('.eb')] + + lines = [] + ecs, _ = parse_easyconfigs([(fp, False) for fp in pr_files], validate=False) + for ec in ecs: + files = find_related_easyconfigs(repo_path, ec['ec']) + _log.debug("File in PR#%s %s has these related easyconfigs: %s" % (pr, ec['spec'], files)) + if files: + lines.append(multidiff(ec['spec'], files, colored=colored)) + else: + lines.extend(['', "(no related easyconfigs found for %s)\n" % os.path.basename(ec['spec'])]) + + return '\n'.join(lines) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index e5b346d13c..339e349d71 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -31,7 +31,7 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) @author: Toon Willems (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ import copy import glob @@ -42,11 +42,12 @@ from vsc.utils import fancylogger from vsc.utils.missing import nub -from easybuild.tools.build_log import print_error, print_msg, print_warning +from easybuild.framework.easyconfig.default import get_easyconfig_parameter_default from easybuild.framework.easyconfig.easyconfig import EasyConfig, create_paths, process_easyconfig -from easybuild.framework.easyconfig.tools import resolve_dependencies +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file, write_file from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version +from easybuild.tools.robot import resolve_dependencies from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME from easybuild.tools.utilities import quote_str @@ -69,13 +70,29 @@ def ec_filename_for(path): return fn -def tweak(easyconfigs, build_specs): +def tweak(easyconfigs, build_specs, targetdir=None): """Tweak list of easyconfigs according to provided build specifications.""" # make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble) toolchains = nub(['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs]) if len(toolchains) > 1: - _log.error("Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s" % toolchains) + raise EasyBuildError("Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s", + toolchains) + + if 'name' in build_specs or 'version' in build_specs: + # no recursion if software name/version build specification are included + # in that case, do not construct full dependency graph + orig_ecs = easyconfigs + _log.debug("Software name/version found, so not applying build specifications recursively: %s" % build_specs) + else: + # build specifications should be applied to the whole dependency graph + # obtain full dependency graph for specified easyconfigs + # easyconfigs will be ordered 'top-to-bottom': toolchain dependencies and toolchain first + _log.debug("Applying build specifications recursively (no software name/version found): %s" % build_specs) + orig_ecs = resolve_dependencies(easyconfigs, retain_all_deps=True) + + # keep track of originally listed easyconfigs (via their path) + listed_ec_paths = [ec['spec'] for ec in easyconfigs] # obtain full dependency graph for specified easyconfigs # easyconfigs will be ordered 'top-to-bottom': toolchain dependencies and toolchain first @@ -91,16 +108,19 @@ def tweak(easyconfigs, build_specs): orig_ecs = orig_ecs[1:] # generate tweaked easyconfigs, and continue with those instead - easyconfigs = [] + tweaked_easyconfigs = [] for orig_ec in orig_ecs: - new_ec_file = tweak_one(orig_ec['spec'], None, build_specs) - new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs) - easyconfigs.extend(new_ecs) + new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=targetdir) + # only return tweaked easyconfigs for easyconfigs which were listed originally + # easyconfig files for dependencies are also generated but not included, and will be resolved via --robot + if orig_ec['spec'] in listed_ec_paths: + new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs) + tweaked_easyconfigs.extend(new_ecs) - return easyconfigs + return tweaked_easyconfigs -def tweak_one(src_fn, target_fn, tweaks): +def tweak_one(src_fn, target_fn, tweaks, targetdir=None): """ Tweak an easyconfig file with the given list of tweaks, using replacement via regular expressions. Note: this will only work 'well-written' easyconfig files, i.e. ones that e.g. set the version @@ -123,10 +143,11 @@ def tweak_one(src_fn, target_fn, tweaks): # determine new toolchain if it's being changed keys = tweaks.keys() if 'toolchain_name' in keys or 'toolchain_version' in keys: + # note: this assumes that the toolchain spec is single-line tc_regexp = re.compile(r"^\s*toolchain\s*=\s*(.*)$", re.M) res = tc_regexp.search(ectxt) if not res: - _log.error("No toolchain found in easyconfig file %s?" % src_fn) + raise EasyBuildError("No toolchain found in easyconfig file %s: %s", src_fn, ectxt) toolchain = eval(res.group(1)) for key in ['name', 'version']: @@ -145,11 +166,16 @@ def __repr__(self): additions = [] + # automagically clear out list of checksums if software version is being tweaked + if 'version' in tweaks and 'checksums' not in tweaks: + tweaks['checksums'] = [] + _log.warning("Tweaking version: checksums cleared, verification disabled.") + # we need to treat list values seperately, i.e. we prepend to the current value (if any) for (key, val) in tweaks.items(): if isinstance(val, list): - regexp = re.compile(r"^\s*%s\s*=\s*(.*)$" % key, re.M) + regexp = re.compile(r"^(?P\s*%s)\s*=\s*(?P\[(.|\n)*\])\s*$" % key, re.M) res = regexp.search(ectxt) if res: fval = [x for x in val if x != ''] # filter out empty strings @@ -157,54 +183,57 @@ def __repr__(self): # - input ending with comma (empty tail list element) => prepend # - input starting with comma (empty head list element) => append # - no empty head/tail list element => overwrite - if val[0] == '': - newval = "%s + %s" % (res.group(1), fval) + if not val: + newval = '[]' + _log.debug("Clearing %s to empty list (was: %s)" % (key, res.group('val'))) + elif val[0] == '': + newval = "%s + %s" % (res.group('val'), fval) _log.debug("Appending %s to %s" % (fval, key)) elif val[-1] == '': - newval = "%s + %s" % (fval, res.group(1)) + newval = "%s + %s" % (fval, res.group('val')) _log.debug("Prepending %s to %s" % (fval, key)) else: newval = "%s" % fval _log.debug("Overwriting %s with %s" % (key, fval)) - ectxt = regexp.sub("%s = %s # tweaked by EasyBuild (was: %s)" % (key, newval, res.group(1)), ectxt) + ectxt = regexp.sub("%s = %s" % (res.group('key'), newval), ectxt) _log.info("Tweaked %s list to '%s'" % (key, newval)) - else: - additions.append("%s = %s # added by EasyBuild" % (key, val)) + elif get_easyconfig_parameter_default(key) != val: + additions.append("%s = %s" % (key, val)) tweaks.pop(key) # add parameters or replace existing ones for (key, val) in tweaks.items(): - regexp = re.compile(r"^\s*%s\s*=\s*(.*)$" % key, re.M) + regexp = re.compile(r"^(?P\s*%s)\s*=\s*(?P.*)$" % key, re.M) _log.debug("Regexp pattern for replacing '%s': %s" % (key, regexp.pattern)) res = regexp.search(ectxt) if res: # only tweak if the value is different diff = True try: - _log.debug("eval(%s): %s" % (res.group(1), eval(res.group(1)))) - diff = not eval(res.group(1)) == val + _log.debug("eval(%s): %s" % (res.group('val'), eval(res.group('val')))) + diff = eval(res.group('val')) != val except (NameError, SyntaxError): # if eval fails, just fall back to string comparison - _log.debug("eval failed for \"%s\", falling back to string comparison against \"%s\"..." % (res.group(1), val)) - diff = not res.group(1) == val + _log.debug("eval failed for \"%s\", falling back to string comparison against \"%s\"...", + res.group('val'), val) + diff = res.group('val') != val if diff: - ectxt = regexp.sub("%s = %s # tweaked by EasyBuild (was: %s)" % (key, quote_str(val), res.group(1)), ectxt) + ectxt = regexp.sub("%s = %s" % (res.group('key'), quote_str(val)), ectxt) _log.info("Tweaked '%s' to '%s'" % (key, quote_str(val))) - else: + elif get_easyconfig_parameter_default(key) != val: additions.append("%s = %s" % (key, quote_str(val))) if additions: - _log.info("Adding additional parameters to tweaked easyconfig file: %s") - ectxt += "\n\n# added by EasyBuild as dictated by command line options\n" - ectxt += '\n'.join(additions) + '\n' + _log.info("Adding additional parameters to tweaked easyconfig file: %s" % additions) + ectxt = '\n'.join([ectxt] + additions) _log.debug("Contents of tweaked easyconfig file:\n%s" % ectxt) # come up with suiting file name for tweaked easyconfig file if none was specified - if not target_fn: + if target_fn is None: fn = None try: # obtain temporary filename @@ -220,9 +249,11 @@ def __repr__(self): # get rid of temporary file os.remove(tmpfn) except OSError, err: - _log.error("Failed to determine suiting filename for tweaked easyconfig file: %s" % err) + raise EasyBuildError("Failed to determine suiting filename for tweaked easyconfig file: %s", err) - target_fn = os.path.join(tempfile.gettempdir(), fn) + if targetdir is None: + targetdir = tempfile.gettempdir() + target_fn = os.path.join(targetdir, fn) _log.debug("Generated file name for tweaked easyconfig file: %s" % target_fn) # write out tweaked easyconfig file @@ -235,33 +266,34 @@ def __repr__(self): def pick_version(req_ver, avail_vers): """Pick version based on an optionally desired version and available versions. - If a desired version is specifed, the most recent version that is less recent - than the desired version will be picked; else, the most recent version will be picked. + If a desired version is specifed, the most recent version that is less recent than or equal to + the desired version will be picked; else, the most recent version will be picked. - This function returns both the version to be used, which is equal to the desired version + This function returns both the version to be used, which is equal to the required version if it was specified, and the version picked that matches that closest. + + @param req_ver: required version + @param avail_vers: list of available versions """ if not avail_vers: - _log.error("Empty list of available versions passed.") + raise EasyBuildError("Empty list of available versions passed.") selected_ver = None if req_ver: # if a desired version is specified, - # retain the most recent version that's less recent than the desired version - + # retain the most recent version that's less recent or equal than the desired version ver = req_ver if len(avail_vers) == 1: selected_ver = avail_vers[0] else: - retained_vers = [v for v in avail_vers if v < LooseVersion(ver)] + retained_vers = [v for v in avail_vers if v <= LooseVersion(ver)] if retained_vers: selected_ver = retained_vers[-1] else: # if no versions are available that are less recent, take the least recent version selected_ver = sorted([LooseVersion(v) for v in avail_vers])[0] - else: # if no desired version is specified, just use last version ver = avail_vers[-1] @@ -270,6 +302,26 @@ def pick_version(req_ver, avail_vers): return (ver, selected_ver) +def find_matching_easyconfigs(name, installver, paths): + """ + Find easyconfigs that match specified name/installversion in specified list of paths. + + @param name: software name + @param installver: software install version (which includes version, toolchain, versionprefix/suffix, ...) + @param paths: list of paths to search easyconfigs in + """ + ec_files = [] + for path in paths: + patterns = create_paths(path, name, installver) + for pattern in patterns: + more_ec_files = filter(os.path.isfile, sorted(glob.glob(pattern))) + _log.debug("Including files that match glob pattern '%s': %s" % (pattern, more_ec_files)) + ec_files.extend(more_ec_files) + + # only retain unique easyconfig paths + return nub(ec_files) + + def select_or_generate_ec(fp, paths, specs): """ Select or generate an easyconfig file with the given requirements, from existing easyconfig files. @@ -294,12 +346,11 @@ def select_or_generate_ec(fp, paths, specs): # ensure that at least name is specified if not specs.get('name'): - _log.error("Supplied 'specs' dictionary doesn't even contain a name of a software package?") + raise EasyBuildError("Supplied 'specs' dictionary doesn't even contain a name of a software package?") name = specs['name'] handled_params = ['name'] # find ALL available easyconfig files for specified software - ec_files = [] cfg = { 'version': '*', 'toolchain': {'name': DUMMY_TOOLCHAIN_NAME, 'version': '*'}, @@ -307,10 +358,8 @@ def select_or_generate_ec(fp, paths, specs): 'versionsuffix': '*', } installver = det_full_ec_version(cfg) - for path in paths: - patterns = create_paths(path, name, installver) - for pattern in patterns: - ec_files.extend(glob.glob(pattern)) + ec_files = find_matching_easyconfigs(name, installver, paths) + _log.debug("Unique ec_files: %s" % ec_files) # we need at least one config file to start from if len(ec_files) == 0: @@ -325,11 +374,8 @@ def select_or_generate_ec(fp, paths, specs): _log.debug("No template found at %s." % templ_file) if len(ec_files) == 0: - _log.error("No easyconfig files found for software %s, and no templates available. I'm all out of ideas." % name) - - # only retain unique easyconfig files - ec_files = nub(ec_files) - _log.debug("Unique ec_files: %s" % ec_files) + raise EasyBuildError("No easyconfig files found for software %s, and no templates available. " + "I'm all out of ideas.", name) ecs_and_files = [(EasyConfig(f, validate=False), f) for f in ec_files] @@ -357,8 +403,8 @@ def unique(l): if EASYCONFIG_TEMPLATE in tcnames: _log.info("No easyconfig file for specified toolchain, but template is available.") else: - _log.error("No easyconfig file for %s with toolchain %s, " \ - "and no template available." % (name, specs['toolchain_name'])) + raise EasyBuildError("No easyconfig file for %s with toolchain %s, and no template available.", + name, specs['toolchain_name']) tcname = specs.pop('toolchain_name', None) handled_params.append('toolchain_name') @@ -381,7 +427,7 @@ def unique(l): else: # if multiple toolchains are available, and none is specified, we quit # we can't just pick one, how would we prefer one over the other? - _log.error("No toolchain name specified, and more than one available: %s." % tcnames) + raise EasyBuildError("No toolchain name specified, and more than one available: %s.", tcnames) _log.debug("Filtering easyconfigs based on toolchain name '%s'..." % selected_tcname) ecs_and_files = [x for x in ecs_and_files if x[0]['toolchain']['name'] == selected_tcname] @@ -464,9 +510,7 @@ def unique(l): filter_ecs = True else: # otherwise, we fail, because we don't know how to pick between different fixes - _log.error("No %s specified, and can't pick from available %ses %s" % (param, - param, - vals)) + raise EasyBuildError("No %s specified, and can't pick from available ones: %s", param, vals) if filter_ecs: _log.debug("Filtering easyconfigs based on %s '%s'..." % (param, selected_val)) @@ -482,7 +526,7 @@ def unique(l): cnt = len(ecs_and_files) if not cnt == 1: fs = [x[1] for x in ecs_and_files] - _log.error("Failed to select a single easyconfig from available ones, %s left: %s" % (cnt, fs)) + raise EasyBuildError("Failed to select a single easyconfig from available ones, %s left: %s", cnt, fs) else: (selected_ec, selected_ec_file) = ecs_and_files[0] @@ -491,7 +535,7 @@ def unique(l): for (key, val) in specs.items(): if key in selected_ec._config: # values must be equal to have a full match - if not selected_ec[key] == val: + if selected_ec[key] != val: match = False else: # if we encounter a key that is not set in the selected easyconfig, we don't have a full match @@ -505,7 +549,7 @@ def unique(l): # GENERATE # if no file path was specified, generate a file name - if not fp: + if fp is None: cfg = { 'version': ver, 'toolchain': {'name': tcname, 'version': tcver}, @@ -523,7 +567,7 @@ def unique(l): return (True, fp) -def obtain_ec_for(specs, paths, fp): +def obtain_ec_for(specs, paths, fp=None): """ Obtain an easyconfig file to the given specifications. @@ -537,36 +581,11 @@ def obtain_ec_for(specs, paths, fp): # ensure that at least name is specified if not specs.get('name'): - _log.error("Supplied 'specs' dictionary doesn't even contain a name of a software package?") + raise EasyBuildError("Supplied 'specs' dictionary doesn't even contain a name of a software package?") # collect paths to search in if not paths: - _log.error("No paths to look for easyconfig files, specify a path with --robot.") - - # create glob patterns based on supplied info - - # figure out the install version - cfg = { - 'version': specs.get('version', '*'), - 'toolchain': { - 'name': specs.get('toolchain_name', '*'), - 'version': specs.get('toolchain_version', '*'), - }, - 'versionprefix': specs.get('versionprefix', '*'), - 'versionsuffix': specs.get('versionsuffix', '*'), - } - installver = det_full_ec_version(cfg) - - # find easyconfigs that match a pattern - easyconfig_files = [] - for path in paths: - patterns = create_paths(path, specs['name'], installver) - for pattern in patterns: - easyconfig_files.extend(glob.glob(pattern)) - - cnt = len(easyconfig_files) - - _log.debug("List of obtained easyconfig files (%d): %s" % (cnt, easyconfig_files)) + raise EasyBuildError("No paths to look for easyconfig files, specify a path with --robot.") # select best easyconfig, or try to generate one that fits the requirements res = select_or_generate_ec(fp, paths, specs) @@ -574,27 +593,4 @@ def obtain_ec_for(specs, paths, fp): if res: return res else: - _log.error("No easyconfig found for requested software, and also failed to generate one.") - - -def obtain_path(specs, paths, try_to_generate=False, exit_on_error=True, silent=False): - """Obtain a path for an easyconfig that matches the given specifications.""" - - # if no easyconfig files/paths were provided, but we did get a software name, - # we can try and find a suitable easyconfig ourselves, or generate one if we can - (generated, fn) = obtain_ec_for(specs, paths, None) - if not generated: - return (fn, generated) - else: - # if an easyconfig was generated, make sure we're allowed to use it - if try_to_generate: - print_msg("Generated an easyconfig file %s, going to use it now..." % fn, silent=silent) - return (fn, generated) - else: - try: - os.remove(fn) - except OSError, err: - print_warning("Failed to remove generated easyconfig file %s: %s" % (fn, err)) - print_error(("Unable to find an easyconfig for the given specifications: %s; " - "to make EasyBuild try to generate a matching easyconfig, " - "use the --try-X options ") % specs, log=_log, exit_on_error=exit_on_error) + raise EasyBuildError("No easyconfig found for requested software, and also failed to generate one.") diff --git a/easybuild/framework/easyconfig/types.py b/easybuild/framework/easyconfig/types.py new file mode 100644 index 0000000000..88ad008744 --- /dev/null +++ b/easybuild/framework/easyconfig/types.py @@ -0,0 +1,118 @@ +# # +# Copyright 2015-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # + +""" +Support for checking types of easyconfig parameter values. + +@author: Kenneth Hoste (Ghent University) +""" +from vsc.utils import fancylogger + +from easybuild.tools.build_log import EasyBuildError + + +# easy types, that can be verified with isinstance +EASY_TYPES = [basestring, int] +# type checking is skipped for easyconfig parameters names not listed in TYPES +TYPES = { + 'name': basestring, + 'version': basestring, +} +TYPE_CONVERSION_FUNCTIONS = { + basestring: str, + float: float, + int: int, + str: str, +} + + +_log = fancylogger.getLogger('easyconfig.types', fname=False) + + +def check_type_of_param_value(key, val, auto_convert=False): + """ + Check value type of specified easyconfig parameter. + + @param key: name of easyconfig parameter + @param val: easyconfig parameter value, of which type should be checked + @param auto_convert: try to automatically convert to expected value type if required + """ + type_ok, newval = False, None + expected_type = TYPES.get(key) + + if expected_type is None: + _log.debug("No type specified for easyconfig parameter '%s', so skipping type check.", key) + type_ok, newval = True, val + + elif expected_type in EASY_TYPES: + # easy types can be checked using isinstance + if isinstance(val, expected_type): + type_ok, newval = True, val + _log.debug("Value type checking of easyconfig parameter '%s' passed: expected '%s', got '%s'", + key, expected_type.__name__, type(val).__name__) + + else: + _log.warning("Value type checking of easyconfig parameter '%s' FAILED: expected '%s', got '%s'", + key, expected_type.__name__, type(val).__name__) + else: + raise EasyBuildError("Don't know how to check whether specified value is of type %s", expected_type) + + if not type_ok and auto_convert: + _log.debug("Value type check failed, going to try to automatically convert to %s", expected_type) + newval = convert_value_type(val, expected_type) + type_ok = True + + return type_ok, newval + + +def convert_value_type(val, typ): + """ + Try to convert type of provided value to specific type. + + @param val: value to convert type of + @param typ: target type + """ + res = None + + if isinstance(val, typ): + _log.debug("Value %s is already of specified target type %s, no conversion needed", val, typ) + res = val + + elif typ in TYPE_CONVERSION_FUNCTIONS: + func = TYPE_CONVERSION_FUNCTIONS[typ] + _log.debug("Trying to convert value %s (type: %s) to %s using %s", val, type(val), typ, func) + try: + res = func(val) + _log.debug("Type conversion seems to have worked, new type: %s", type(res)) + except Exception as err: + raise EasyBuildError("Converting type of %s (%s) to %s using %s failed: %s", val, type(val), typ, func, err) + + if not isinstance(res, typ): + raise EasyBuildError("Converting value %s to type %s didn't work as expected: got %s", val, typ, type(res)) + + else: + raise EasyBuildError("No conversion function available (yet) for target type %s", typ) + + return res diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index af5b17672b..1713130057 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -36,8 +36,9 @@ import copy import os +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_path -from easybuild.tools.filetools import run_cmd +from easybuild.tools.run import run_cmd class Extension(object): @@ -54,10 +55,11 @@ def __init__(self, mself, ext): self.ext = copy.deepcopy(ext) if not 'name' in self.ext: - self.log.error("'name' is missing in supplied class instance 'ext'.") + raise EasyBuildError("'name' is missing in supplied class instance 'ext'.") - self.src = self.ext.get('src', None) - self.patches = self.ext.get('patches', None) + # list of source/patch files: we use an empty list as default value like in EasyBlock + self.src = self.ext.get('src', []) + self.patches = self.ext.get('patches', []) self.options = copy.deepcopy(self.ext.get('options', {})) self.toolchain.prepare(self.cfg['onlytcmod']) @@ -111,7 +113,7 @@ def sanity_check_step(self): try: os.chdir(build_path()) except OSError, err: - self.log.error("Failed to change directory: %s" % err) + raise EasyBuildError("Failed to change directory: %s", err) # disabling templating is required here to support legacy string templates like name/version self.cfg.enable_templating = False diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index 1308561fb9..52e3e11783 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of the University of Ghent (http://ugent.be/hpc). @@ -31,6 +31,7 @@ from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extension import Extension +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import apply_patch, extract_file from easybuild.tools.utilities import remove_unwanted_chars @@ -58,8 +59,7 @@ def extra_options(extra_vars=None): extra_vars = {} if not isinstance(extra_vars, dict): - _log.deprecated("Obtained value of type '%s' for extra_vars, should be 'dict'" % type(extra_vars), '2.0') - extra_vars = dict(extra_vars) + _log.nosupport("Obtained value of type '%s' for extra_vars, should be 'dict'" % type(extra_vars), '2.0') extra_vars.update({ 'options': [{}, "Dictionary with extension options.", CUSTOM], @@ -98,7 +98,7 @@ def run(self, unpack_src=False): if self.patches: for patchfile in self.patches: if not apply_patch(patchfile, self.ext_dir): - self.log.error("Applying patch %s failed" % patchfile) + raise EasyBuildError("Applying patch %s failed", patchfile) def sanity_check_step(self, exts_filter=None, custom_paths=None, custom_commands=None): """ @@ -119,9 +119,10 @@ def sanity_check_step(self, exts_filter=None, custom_paths=None, custom_commands # unload fake module and clean up self.clean_up_fake_module(fake_mod_data) - if custom_paths or self.cfg['sanity_check_paths'] or custom_commands or self.cfg['sanity_check_commands']: - EasyBlock.sanity_check_step(self, custom_paths=custom_paths, custom_commands=custom_commands, - extension=self.is_extension) + if custom_paths or custom_commands or not self.is_extension: + super(ExtensionEasyBlock, self).sanity_check_step(custom_paths=custom_paths, + custom_commands=custom_commands, + extension=self.is_extension) # pass or fail sanity check if not sanity_check_ok: @@ -129,7 +130,7 @@ def sanity_check_step(self, exts_filter=None, custom_paths=None, custom_commands if self.is_extension: self.log.warning(msg) else: - self.log.error(msg) + raise EasyBuildError(msg) return False else: self.log.info("Sanity check for %s successful!" % self.name) diff --git a/easybuild/main.py b/easybuild/main.py old mode 100644 new mode 100755 index 89d001e809..68ccea97a4 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -33,55 +33,83 @@ @author: Jens Timmerman (Ghent University) @author: Toon Willems (Ghent University) @author: Ward Poelmans (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ import copy import os -import subprocess +import stat import sys import tempfile import traceback -from vsc.utils import fancylogger -from vsc.utils.missing import any # IMPORTANT this has to be the first easybuild import as it customises the logging # expect missing log output when this not the case! -from easybuild.tools.build_log import EasyBuildError, print_msg, print_error +from easybuild.tools.build_log import EasyBuildError, init_logging, print_msg, print_error, stop_logging import easybuild.tools.config as config import easybuild.tools.options as eboptions from easybuild.framework.easyblock import EasyBlock, build_and_install_one -from easybuild.framework.easyconfig.easyconfig import process_easyconfig -from easybuild.framework.easyconfig.tools import dep_graph, get_paths_for, print_dry_run -from easybuild.framework.easyconfig.tools import resolve_dependencies, skip_available -from easybuild.framework.easyconfig.tweak import obtain_path, tweak -from easybuild.tools.config import get_repository, module_classes, get_repositorypath, set_tmpdir -from easybuild.tools.filetools import cleanup, find_easyconfigs, search_file, write_file -from easybuild.tools.github import fetch_easyconfigs_from_pr +from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR +from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, dep_graph, det_easyconfig_paths +from easybuild.framework.easyconfig.tools import get_paths_for, parse_easyconfigs, review_pr, skip_available +from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak +from easybuild.tools.config import get_repository, get_repositorypath +from easybuild.tools.filetools import adjust_permissions, cleanup, write_file from easybuild.tools.options import process_software_build_specs -from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel +from easybuild.tools.robot import det_robot_path, dry_run, resolve_dependencies, search_easyconfigs +from easybuild.tools.package.utilities import check_pkg_support +from easybuild.tools.parallelbuild import submit_jobs from easybuild.tools.repository.repository import init_repository -from easybuild.tools.testing import create_test_report, post_easyconfigs_pr_test_report, upload_test_report_as_gist -from easybuild.tools.testing import regtest, session_module_list, session_state -from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME -from easybuild.tools.version import this_is_easybuild # from a single location +from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_module_list, session_state +from easybuild.tools.version import this_is_easybuild _log = None +def log_start(eb_command_line, eb_tmpdir): + """Log startup info.""" + _log.info(this_is_easybuild()) + + # log used command line + _log.info("Command line: %s" % (' '.join(eb_command_line))) + + _log.info("Using %s as temporary directory" % eb_tmpdir) + + +def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=False): + """Find easyconfigs by build specifications.""" + generated, ec_file = obtain_ec_for(build_specs, robot_path, None) + if generated: + if try_to_generate: + print_msg("Generated an easyconfig file %s, going to use it now..." % ec_file, silent=testing) + else: + # (try to) cleanup + try: + os.remove(ec_file) + except OSError, err: + _log.warning("Failed to remove generated easyconfig file %s: %s" % (ec_file, err)) + + # don't use a generated easyconfig unless generation was requested (using a --try-X option) + raise EasyBuildError("Unable to find an easyconfig for the given specifications: %s; " + "to make EasyBuild try to generate a matching easyconfig, " + "use the --try-X options ", build_specs) + + return [(ec_file, generated)] + + def build_and_install_software(ecs, init_session_state, exit_on_failure=True): """Build and install software for all provided parsed easyconfig files.""" # obtain a copy of the starting environment so each build can start afresh # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since # e.g. via easyconfig.handle_allowed_system_deps - orig_environ = copy.deepcopy(os.environ) + init_env = copy.deepcopy(os.environ) res = [] for ec in ecs: ec_res = {} try: - (ec_res['success'], app_log, err) = build_and_install_one(ec, orig_environ) + (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env) ec_res['log_file'] = app_log if not ec_res['success']: ec_res['err'] = EasyBuildError(err) @@ -103,293 +131,181 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): test_report_txt = create_test_report(test_msg, [(ec, ec_res)], init_session_state) if 'log_file' in ec_res: test_report_fp = "%s_test_report.md" % '.'.join(ec_res['log_file'].split('.')[:-1]) - write_file(test_report_fp, test_report_txt) + parent_dir = os.path.dirname(test_report_fp) + # parent dir for test report may not be writable at this time, e.g. when --read-only-installdir is used + if os.stat(parent_dir).st_mode & 0200: + write_file(test_report_fp, test_report_txt) + else: + adjust_permissions(parent_dir, stat.S_IWUSR, add=True, recursive=False) + write_file(test_report_fp, test_report_txt) + adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False) if not ec_res['success'] and exit_on_failure: if 'traceback' in ec_res: - _log.error(ec_res['traceback']) + raise EasyBuildError(ec_res['traceback']) else: - _log.error(test_msg) + raise EasyBuildError(test_msg) res.append((ec, ec_res)) return res -def main(testing_data=(None, None, None)): +def main(args=None, logfile=None, do_build=None, testing=False): """ - Main function: - @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options - This function will: - - read easyconfig - - build software + Main function: parse command line options, and act accordingly. + @param args: command line arguments to use + @param logfile: log file to use + @param do_build: whether or not to actually perform the build + @param testing: enable testing mode """ - # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() - # disallow running EasyBuild as root - if os.getuid() == 0: - sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n" - "That's not wise, so let's end this here.\n" - "Exiting.\n") - sys.exit(1) - - # steer behavior when testing main - testing = testing_data[0] is not None - args, logfile, do_build = testing_data - # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args - eb_config = eb_go.generate_cmd_line(add_default=True) - init_session_state.update({'easybuild_configuration': eb_config}) # set umask (as early as possible) if options.umask is not None: new_umask = int(options.umask, 8) old_umask = os.umask(new_umask) - # set temporary directory to use - eb_tmpdir = set_tmpdir(options.tmpdir) + # set by option parsers via set_tmpdir + eb_tmpdir = tempfile.gettempdir() # initialise logging for main - if options.logtostdout: - fancylogger.logToScreen(enable=True, stdout=True) - else: - if logfile is None: - # mkstemp returns (fd,filename), fd is from os.open, not regular open! - fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-') - os.close(fd) + global _log + _log, logfile = init_logging(logfile, logtostdout=options.logtostdout, testing=testing) - fancylogger.logToFile(logfile) - print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing) + # disallow running EasyBuild as root + if os.getuid() == 0: + raise EasyBuildError("You seem to be running EasyBuild with root privileges which is not wise, " + "so let's end this here.") - global _log - _log = fancylogger.getLogger(fname=False) + # log startup info + eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args + log_start(eb_cmd_line, eb_tmpdir) if options.umask is not None: _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask))) - # hello world! - _log.info(this_is_easybuild()) - - # how was EB called? - eb_command_line = eb_go.generate_cmd_line() + eb_go.args - _log.info("Command line: %s" % (" ".join(eb_command_line))) - - _log.info("Using %s as temporary directory" % eb_tmpdir) - - if not options.robot is None: - if options.robot: - _log.info("Using robot path(s): %s" % options.robot) - else: - _log.error("No robot paths specified, and unable to determine easybuild-easyconfigs install path.") - - # do not pass options.robot, it's not a list instance (and it shouldn't be modified) - robot_path = None - if options.robot: - robot_path = list(options.robot) - - # determine easybuild-easyconfigs package install path - easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path) - # keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs - easyconfigs_pkg_full_paths = easyconfigs_paths[:] - if not easyconfigs_paths: - _log.warning("Failed to determine install path for easybuild-easyconfigs package.") - # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) - # specified robot paths are preferred over installed easyconfig files - # --try-X and --dep-graph both require --robot, so enable it with path of installed easyconfigs - if robot_path or try_to_generate or options.dep_graph: - if robot_path is None: - robot_path = [] - robot_path.extend(easyconfigs_paths) - easyconfigs_paths = robot_path[:] - _log.info("Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path) - - # initialise the easybuild configuration - config.init(options, eb_go.get_options_by_section('config')) - - # building a dependency graph implies force, so that all dependencies are retained - # and also skips validation of easyconfigs (e.g. checking os dependencies) - retain_all_deps = False - if options.dep_graph: - _log.info("Enabling force to generate dependency graph.") - options.force = True - retain_all_deps = True - - if options.dep_graph or options.dry_run or options.dry_run_short: - options.ignore_osdeps = True - - config.init_build_options({ - 'aggregate_regtest': options.aggregate_regtest, - 'allow_modules_tool_mismatch': options.allow_modules_tool_mismatch, - 'check_osdeps': not options.ignore_osdeps, - 'filter_deps': options.filter_deps, - 'cleanup_builddir': options.cleanup_builddir, - 'command_line': eb_command_line, - 'debug': options.debug, - 'dry_run': options.dry_run or options.dry_run_short, - 'easyblock': options.easyblock, - 'experimental': options.experimental, - 'force': options.force, - 'github_user': options.github_user, - 'group': options.group, - 'ignore_dirs': options.ignore_dirs, - 'modules_footer': options.modules_footer, - 'only_blocks': options.only_blocks, - 'optarch': options.optarch, - 'recursive_mod_unload': options.recursive_module_unload, - 'regtest_output_dir': options.regtest_output_dir, - 'retain_all_deps': retain_all_deps, + # determine robot path + # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs + tweaked_ecs = try_to_generate and build_specs + tweaked_ecs_path, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr) + auto_robot = try_to_generate or options.dep_graph or options.search or options.search_short + robot_path = det_robot_path(options.robot_paths, tweaked_ecs_path, pr_path, auto_robot=auto_robot) + _log.debug("Full robot path: %s" % robot_path) + + # configure & initialize build options + config_options_dict = eb_go.get_options_by_section('config') + build_options = { + 'build_specs': build_specs, + 'command_line': eb_cmd_line, + 'pr_path': pr_path, 'robot_path': robot_path, - 'sequential': options.sequential, 'silent': testing, - 'set_gid_bit': options.set_gid_bit, - 'skip': options.skip, - 'skip_test_cases': options.skip_test_cases, - 'sticky_bit': options.sticky_bit, - 'stop': options.stop, - 'suffix_modules_path': options.suffix_modules_path, - 'test_report_env_filter': options.test_report_env_filter, - 'umask': options.umask, - 'valid_module_classes': module_classes(), + 'try_to_generate': try_to_generate, 'valid_stops': [x[0] for x in EasyBlock.get_steps()], - 'validate': not options.force, - }) + } + # initialise the EasyBuild configuration & build options + config.init(options, config_options_dict) + config.init_build_options(build_options=build_options, cmdline_options=options) + + # check whether packaging is supported when it's being used + if options.package: + check_pkg_support() + else: + _log.debug("Packaging not enabled, so not checking for packaging support.") - # obtain list of loaded modules, build options must be initialized first - modlist = session_module_list() + # update session state + eb_config = eb_go.generate_cmd_line(add_default=True) + modlist = session_module_list(testing=testing) # build options must be initialized first before 'module list' works + init_session_state.update({'easybuild_configuration': eb_config}) init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) - # search for easyconfigs - if options.search or options.search_short: - search_path = [os.getcwd()] - if easyconfigs_paths: - search_path = easyconfigs_paths - query = options.search or options.search_short - ignore_dirs = config.build_option('ignore_dirs') - silent = config.build_option('silent') - search_file(search_path, query, short=not options.search, ignore_dirs=ignore_dirs, silent=silent) - - paths = [] - if len(orig_paths) == 0: - if options.from_pr: - pr_path = os.path.join(eb_tmpdir, "files_pr%s" % options.from_pr) - pr_files = fetch_easyconfigs_from_pr(options.from_pr, path=pr_path, github_user=options.github_user) - paths = [(path, False) for path in pr_files if path.endswith('.eb')] - elif 'name' in build_specs: - paths = [obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate, - exit_on_error=not testing)] - elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]): + # review specified PR + if options.review_pr: + print review_pr(options.review_pr, colored=options.color) + + # search for easyconfigs, if a query is specified + query = options.search or options.search_short + if query: + search_easyconfigs(query, short=not options.search) + + # determine easybuild-easyconfigs package install path + easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR) + if not easyconfigs_pkg_paths: + _log.warning("Failed to determine install path for easybuild-easyconfigs package.") + + # command line options that do not require any easyconfigs to be specified + no_ec_opts = [options.aggregate_regtest, options.review_pr, options.search, options.search_short, options.regtest] + + # determine paths to easyconfigs + paths = det_easyconfig_paths(orig_paths) + if paths: + # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated + paths = [(p, False) for p in paths] + else: + if 'name' in build_specs: + # try to obtain or generate an easyconfig file via build specifications if a software name is provided + paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing) + elif not any(no_ec_opts): print_error(("Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), - log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) - else: - # look for easyconfigs with relative paths in easybuild-easyconfigs package, - # unless they were found at the given relative paths - if easyconfigs_pkg_full_paths: - # determine which easyconfigs files need to be found, if any - ecs_to_find = [] - for idx, orig_path in enumerate(orig_paths): - if orig_path == os.path.basename(orig_path) and not os.path.exists(orig_path): - ecs_to_find.append((idx, orig_path)) - _log.debug("List of easyconfig files to find: %s" % ecs_to_find) - - # find missing easyconfigs by walking paths with installed easyconfig files - for path in easyconfigs_pkg_full_paths: - _log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path)) - for (subpath, dirnames, filenames) in os.walk(path, topdown=True): - for idx, orig_path in ecs_to_find[:]: - if orig_path in filenames: - full_path = os.path.join(subpath, orig_path) - _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) - orig_paths[idx] = full_path - # if file was found, stop looking for it (first hit wins) - ecs_to_find.remove((idx, orig_path)) - - # stop os.walk insanity as soon as we have all we need (os.walk loop) - if len(ecs_to_find) == 0: - break - - # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk - dirnames[:] = [d for d in dirnames if not d in options.ignore_dirs] - - # stop os.walk insanity as soon as we have all we need (paths loop) - if len(ecs_to_find) == 0: - break - - # indicate that specified paths do not contain generated easyconfig files - paths = [(path, False) for path in orig_paths] - + log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") - if paths: - ec_paths = [path[0] for path in paths] - else: # fallback: easybuild-easyconfigs install path - ec_paths = easyconfigs_pkg_full_paths - regtest_ok = regtest(ec_paths) - + # fallback: easybuild-easyconfigs install path + regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files - easyconfigs = [] - generated_ecs = False - for (path, generated) in paths: - path = os.path.abspath(path) - # keep track of whether any files were generated - generated_ecs |= generated - if not os.path.exists(path): - print_error("Can't find path %s" % path) - - try: - ec_files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs) - for ec_file in ec_files: - # only pass build specs when not generating easyconfig files - if try_to_generate: - ecs = process_easyconfig(ec_file) - else: - ecs = process_easyconfig(ec_file, build_specs=build_specs) - easyconfigs.extend(ecs) - except IOError, err: - _log.error("Processing easyconfigs in path %s failed: %s" % (path, err)) + easyconfigs, generated_ecs = parse_easyconfigs(paths) # tweak obtained easyconfig files, if requested # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail # if easyconfig files for the dependencies are not available if try_to_generate and build_specs and not generated_ecs: - easyconfigs = tweak(easyconfigs, build_specs) - - # before building starts, take snapshot of environment (watch out -t option!) - os.chdir(os.environ['PWD']) + easyconfigs = tweak(easyconfigs, build_specs, targetdir=tweaked_ecs_path) # dry_run: print all easyconfigs and dependencies, and whether they are already built if options.dry_run or options.dry_run_short: - print_dry_run(easyconfigs, short=not options.dry_run, build_specs=build_specs) + txt = dry_run(easyconfigs, short=not options.dry_run, build_specs=build_specs) + print_msg(txt, log=_log, silent=testing, prefix=False) - if any([options.dry_run, options.dry_run_short, options.regtest, options.search, options.search_short]): + # cleanup and exit after dry run, searching easyconfigs or submitting regression test + if any(no_ec_opts + [options.dry_run, options.dry_run_short]): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # skip modules that are already installed unless forced if not options.force: - easyconfigs = skip_available(easyconfigs, testing=testing) + retained_ecs = skip_available(easyconfigs) + if not testing: + for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]: + print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name']) + easyconfigs = retained_ecs # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: - print_msg("resolving dependencies ...", log=_log, silent=testing) - ordered_ecs = resolve_dependencies(easyconfigs, build_specs=build_specs) + if options.robot: + print_msg("resolving dependencies ...", log=_log, silent=testing) + ordered_ecs = resolve_dependencies(easyconfigs, build_specs=build_specs) + else: + ordered_ecs = easyconfigs else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] @@ -400,27 +316,11 @@ def main(testing_data=(None, None, None)): dep_graph(options.dep_graph, ordered_ecs) sys.exit(0) - # submit build as job(s) and exit + # submit build as job(s), clean up and exit if options.job: - curdir = os.getcwd() - - # the options to ignore (help options can't reach here) - ignore_opts = ['robot', 'job'] - - # generate_cmd_line returns the options in form --longopt=value - opts = [x for x in eb_go.generate_cmd_line() if not x.split('=')[0] in ['--%s' % y for y in ignore_opts]] - - quoted_opts = subprocess.list2cmdline(opts) - - command = "unset TMPDIR && cd %s && eb %%(spec)s %s" % (curdir, quoted_opts) - _log.info("Command template for jobs: %s" % command) + submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing) if not testing: - jobs = build_easyconfigs_in_parallel(command, ordered_ecs) - txt = ["List of submitted jobs:"] - txt.extend(["%s (%s): %s" % (job.name, job.module, job.jobid) for job in jobs]) - txt.append("(%d jobs submitted)" % len(jobs)) - - print_msg("Submitted parallel build jobs, exiting now: %s" % '\n'.join(txt), log=_log) + print_msg("Submitted parallel build jobs, exiting now") cleanup(logfile, eb_tmpdir, testing) sys.exit(0) @@ -438,24 +338,10 @@ def main(testing_data=(None, None, None)): repo = init_repository(get_repository(), get_repositorypath()) repo.cleanup() - # report back in PR in case of testing - if options.upload_test_report: - msg = success_msg + " (%d easyconfigs in this PR)" % len(paths) - test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nr=options.from_pr, gist_log=True) - if options.from_pr: - # upload test report to gist and issue a comment in the PR to notify - msg = post_easyconfigs_pr_test_report(options.from_pr, test_report, success_msg, init_session_state, overall_success) - print_msg(msg) - else: - # only upload test report as a gist - gist_url = upload_test_report_as_gist(test_report) - print_msg("Test report uploaded to %s" % gist_url) - else: - test_report = create_test_report(success_msg, ecs_with_res, init_session_state) - _log.debug("Test report: %s" % test_report) - if options.dump_test_report is not None: - write_file(options.dump_test_report, test_report) - _log.info("Test report dumped to %s" % options.dump_test_report) + # dump/upload overall test report + test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state) + if test_report_msg is not None: + print_msg(test_report_msg) print_msg(success_msg, log=_log, silent=testing) @@ -464,17 +350,14 @@ def main(testing_data=(None, None, None)): if 'original_spec' in ec and os.path.isfile(ec['spec']): os.remove(ec['spec']) - # cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir path) - if options.logtostdout: - fancylogger.logToScreen(enable=False, stdout=True) - else: - fancylogger.logToFile(logfile, enable=False) + # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir) + stop_logging(logfile, logtostdout=options.logtostdout) if overall_success: cleanup(logfile, eb_tmpdir, testing) + if __name__ == "__main__": try: main() except EasyBuildError, e: - sys.stderr.write('ERROR: %s\n' % e.msg) - sys.exit(1) + print_error(e.msg) diff --git a/easybuild/scripts/add_header.py b/easybuild/scripts/add_header.py index a7d48f6214..090350e297 100644 --- a/easybuild/scripts/add_header.py +++ b/easybuild/scripts/add_header.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC-UGent team. diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 91deb59323..fa7bec2385 100755 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -42,20 +42,33 @@ """ import copy +import glob import os import re import shutil +import site import sys import tempfile from distutils.version import LooseVersion +PYPI_SOURCE_URL = 'https://pypi.python.org/packages/source' + +VSC_BASE = 'vsc-base' +EASYBUILD_PACKAGES = [VSC_BASE, 'easybuild-framework', 'easybuild-easyblocks', 'easybuild-easyconfigs'] + # set print_debug to True for detailed progress info -print_debug = False -#print_debug = True +print_debug = os.environ.pop('EASYBUILD_BOOTSTRAP_DEBUG', False) + +# don't add user site directory to sys.path (equivalent to python -s), see https://www.python.org/dev/peps/pep-0370/ +os.environ['PYTHONNOUSERSITE'] = '1' +site.ENABLE_USER_SITE = False # clean PYTHONPATH to avoid finding readily installed stuff os.environ['PYTHONPATH'] = '' +EASYBUILD_BOOTSTRAP_SOURCEPATH = os.environ.pop('EASYBUILD_BOOTSTRAP_SOURCEPATH', None) +EASYBUILD_BOOTSTRAP_SKIP_STAGE0 = os.environ.pop('EASYBUILD_BOOTSTRAP_SKIP_STAGE0', False) + # keep track of original environment (after clearing PYTHONPATH) orig_os_environ = copy.deepcopy(os.environ) @@ -93,16 +106,17 @@ def find_egg_dir_for(path, pkg): for libdir in ['lib', 'lib64']: full_libpath = os.path.join(path, det_lib_path(libdir)) - eggdir_regex = re.compile('%s-[0-9a-z.]+-py[0-9.]+.egg' % pkg.replace('-', '_')) - subdirs = os.listdir(full_libpath) + subdirs = (os.path.exists(full_libpath) and os.listdir(full_libpath)) or [] for subdir in subdirs: if eggdir_regex.match(subdir): eggdir = os.path.join(full_libpath, subdir) debug("Found egg dir for %s at %s" % (pkg, eggdir)) return eggdir - error("Failed to determine egg dir path for %s in %s (subdirs: %s)" % (pkg, path, subdirs)) + # no egg dir found + debug("Failed to determine egg dir path for %s in %s (subdirs: %s)" % (pkg, path, subdirs)) + return None def prep(path): """Prepare for installing a Python package in the specified path.""" @@ -212,7 +226,11 @@ def stage0(tmpdir): error("Installing distribute which should deliver easy_install failed?") # prepend distribute egg dir to sys.path, so we know which setuptools we're using - sys.path.insert(0, find_egg_dir_for(tmpdir, 'distribute')) + distribute_egg_dir = find_egg_dir_for(tmpdir, 'distribute') + if distribute_egg_dir is None: + error("Failed to determine egg dir path for distribute_egg_dir in %s" % tmpdir) + else: + sys.path.insert(0, distribute_egg_dir) # make sure we're getting the setuptools we expect import setuptools @@ -221,62 +239,100 @@ def stage0(tmpdir): else: debug("Found setuptools in expected path, good!") -def stage1(tmpdir): + return distribute_egg_dir + + +def stage1(tmpdir, sourcepath): """STAGE 1: temporary install EasyBuild using distribute's easy_install.""" info("\n\n+++ STAGE 1: installing EasyBuild in temporary dir with easy_install...\n\n") + # determine locations of source tarballs, if sources path is specified + source_tarballs = {} + if sourcepath is not None: + info("Fetching sources from %s..." % sourcepath) + for pkg in EASYBUILD_PACKAGES: + pkg_tarball_glob = os.path.join(sourcepath, '%s*.tar.gz' % pkg) + pkg_tarball_paths = glob.glob(pkg_tarball_glob) + if len(pkg_tarball_paths) > 1: + error("Multiple tarballs found for %s: %s" % (pkg, pkg_tarball_paths)) + elif len(pkg_tarball_paths) == 0: + if pkg != VSC_BASE: + # vsc-base package is not strictly required + # it's only a dependency since EasyBuild v2.0; + # with EasyBuild v2.0, it will be pulled in from PyPI when installing easybuild-framework + error("Missing source tarball: %s" % pkg_tarball_glob) + else: + info("Found %s for %s package" % (pkg_tarball_paths[0], pkg)) + source_tarballs.update({pkg: pkg_tarball_paths[0]}) + from setuptools.command import easy_install # prepare install dir targetdir_stage1 = os.path.join(tmpdir, 'eb_stage1') prep(targetdir_stage1) # set PATH, Python search path - # install latest EasyBuild with easy_install from PyPi cmd = [] cmd.append('--upgrade') # make sure the latest version is pulled from PyPi cmd.append('--prefix=%s' % targetdir_stage1) - cmd.append('easybuild') + + if source_tarballs: + # install provided source tarballs (order matters) + cmd.extend([source_tarballs[pkg] for pkg in EASYBUILD_PACKAGES if pkg in source_tarballs]) + else: + # install meta-package easybuild from PyPI + cmd.append('easybuild') + if not print_debug: cmd.insert(0, '--quiet') - debug("installing EasyBuild with 'easy_install %s'" % (" ".join(cmd))) + info("installing EasyBuild with 'easy_install %s'" % (' '.join(cmd))) easy_install.main(cmd) # clear the Python search path, we only want the individual eggs dirs to be in the PYTHONPATH (see below) # this is needed to avoid easy-install.pth controlling what Python packages are actually used os.environ['PYTHONPATH'] = '' - versions = {} + # template string to inject in template easyconfig + templates = {} + + for pkg in EASYBUILD_PACKAGES: + templates.update({pkg: ''}) - pkg_egg_dir_framework = None - for pkg in ['easyconfigs', 'easyblocks', 'framework']: - pkg_egg_dir = find_egg_dir_for(targetdir_stage1, 'easybuild-%s' % pkg) + pkg_egg_dir = find_egg_dir_for(targetdir_stage1, pkg) + if pkg_egg_dir is None: + if pkg == VSC_BASE: + # vsc-base is optional in older EasyBuild versions + continue # prepend EasyBuild egg dirs to Python search path, so we know which EasyBuild we're using sys.path.insert(0, pkg_egg_dir) pythonpaths = [x for x in os.environ.get('PYTHONPATH', '').split(os.pathsep) if len(x) > 0] os.environ['PYTHONPATH'] = os.pathsep.join([pkg_egg_dir] + pythonpaths) - # determine per-package versions based on egg dirs - version_regex = re.compile('easybuild_%s-([0-9a-z.-]*)-py[0-9.]*.egg' % pkg) - pkg_egg_dirname = os.path.basename(pkg_egg_dir) - res = version_regex.search(pkg_egg_dirname) - if res is not None: - pkg_version = res.group(1) - versions.update({'%s_version' % pkg: pkg_version}) - debug("Found version for easybuild-%s: %s" % (pkg, pkg_version)) + if source_tarballs: + if pkg in source_tarballs: + templates.update({pkg: "'%s'," % os.path.basename(source_tarballs[pkg])}) else: - error("Failed to determine version for easybuild-%s package from %s with %s" % (pkg, pkg_egg_dirname, version_regex.pattern)) - - if pkg == 'framework': - pkg_egg_dir_framework = pkg_egg_dir + # determine per-package versions based on egg dirs, to use them in easyconfig template + version_regex = re.compile('%s-([0-9a-z.-]*)-py[0-9.]*.egg' % pkg.replace('-', '_')) + pkg_egg_dirname = os.path.basename(pkg_egg_dir) + res = version_regex.search(pkg_egg_dirname) + if res is not None: + pkg_version = res.group(1) + debug("Found version for easybuild-%s: %s" % (pkg, pkg_version)) + templates.update({pkg: "'%s-%s.tar.gz'," % (pkg, pkg_version)}) + else: + tup = (pkg, pkg_egg_dirname, version_regex.pattern) + error("Failed to determine version for easybuild-%s package from %s with %s" % tup) # figure out EasyBuild version via eb command line - # NOTE: EasyBuild uses some magic to determine the EasyBuild version based on the versions of the individual packages - version_re = re.compile("This is EasyBuild (?P[0-9.]*[a-z0-9]*) \(framework: [0-9.]*[a-z0-9]*, easyblocks: [0-9.]*[a-z0-9]*\)") + # note: EasyBuild uses some magic to determine the EasyBuild version based on the versions of the individual packages + pattern = "This is EasyBuild (?P%(v)s) \(framework: %(v)s, easyblocks: %(v)s\)" % {'v': '[0-9.]*[a-z0-9]*'} + version_re = re.compile(pattern) version_out_file = os.path.join(tmpdir, 'eb_version.out') - cmd = "python -c 'from easybuild.tools.version import this_is_easybuild; print this_is_easybuild()' > %s 2>&1" % version_out_file + eb_version_cmd = 'from easybuild.tools.version import this_is_easybuild; print this_is_easybuild()' + cmd = "python -c '%s' > %s 2>&1" % (eb_version_cmd, version_out_file) debug("Determining EasyBuild version using command '%s'" % cmd) os.system(cmd) txt = open(version_out_file, "r").read() @@ -287,7 +343,7 @@ def stage1(tmpdir): else: error("Stage 1 failed, could not determine EasyBuild version (txt: %s)." % txt) - versions.update({'version': eb_version}) + templates.update({'version': eb_version}) # clear PYTHONPATH before we go to stage2 # PYTHONPATH doesn't need to (and shouldn't) include the stage1 egg dirs @@ -306,21 +362,33 @@ def stage1(tmpdir): else: debug("Found easybuild-easyblocks in expected path, good!") - return versions + debug("templates: %s" % templates) + return templates -def stage2(tmpdir, versions, install_path): +def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): """STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1.""" info("\n\n+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n\n" % install_path) - # make sure we still have distribute in PYTHONPATH, so we have control over which 'setup' is used - pythonpaths = [x for x in os.environ.get('PYTHONPATH', '').split(os.pathsep) if len(x) > 0] - os.environ['PYTHONPATH'] = os.pathsep.join([find_egg_dir_for(tmpdir, 'distribute')] + pythonpaths) + # inject path to distribute installed in stage 1 into $PYTHONPATH via preinstallopts + # other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH + if distribute_egg_dir is None: + preinstallopts = '' + else: + preinstallopts = 'PYTHONPATH=%s:$PYTHONPATH' % distribute_egg_dir + templates.update({ + 'preinstallopts': preinstallopts, + }) # create easyconfig file - ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % versions['version']) + ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version']) f = open(ebfile, "w") - f.write(EB_EC_FILE % versions) + templates.update({ + 'source_urls': '\n'.join(["'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg) for pkg in EASYBUILD_PACKAGES]), + 'sources': "%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates, + 'pythonpath': distribute_egg_dir, + }) + f.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates) f.close() # unset $MODULEPATH, we don't care about already installed modules @@ -333,7 +401,7 @@ def stage2(tmpdir, versions, install_path): # make sure we don't leave any stuff behind in default path $HOME/.local/easybuild # and set build and install path explicitely - if LooseVersion(versions['version']) < LooseVersion("1.3.0"): + if LooseVersion(templates['version']) < LooseVersion('1.3.0'): os.environ['EASYBUILD_PREFIX'] = tmpdir os.environ['EASYBUILD_BUILDPATH'] = tmpdir if install_path is not None: @@ -344,6 +412,8 @@ def stage2(tmpdir, versions, install_path): eb_args.append('--buildpath=%s' % tmpdir) if install_path is not None: eb_args.append('--installpath=%s' % install_path) + if sourcepath is not None: + eb_args.append('--sourcepath=%s' % sourcepath) # make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH) if install_path is not None: @@ -372,6 +442,10 @@ def main(): error("Usage: %s " % sys.argv[0]) install_path = os.path.abspath(sys.argv[1]) + sourcepath = EASYBUILD_BOOTSTRAP_SOURCEPATH + if sourcepath is not None: + info("Fetching sources from %s..." % sourcepath) + # create temporary dir for temporary installations tmpdir = tempfile.mkdtemp() debug("Going to use %s as temporary directory" % tmpdir) @@ -385,10 +459,13 @@ def main(): sys.path = [] for path in orig_sys_path: include_path = True - # exclude path if it's potentially an EasyBuild package - if 'easybuild' in path: + # exclude path if it's potentially an EasyBuild/VSC package, providing the 'easybuild'/'vsc' namespace, resp. + if any([os.path.exists(os.path.join(path, pkg, '__init__.py')) for pkg in ['easyblocks', 'easybuild', 'vsc']]): include_path = False - # exclude path if it contain an easy-install.pth file + # exclude any .egg paths + if path.endswith('.egg'): + include_path = False + # exclude any path that contains an easy-install.pth file if os.path.exists(os.path.join(path, 'easy-install.pth')): include_path = False @@ -402,13 +479,17 @@ def main(): # install EasyBuild in stages # STAGE 0: install distribute, which delivers easy_install - stage0(tmpdir) + if EASYBUILD_BOOTSTRAP_SKIP_STAGE0: + distribute_egg_dir = None + info("Skipping stage0, using local distribute/setuptools providing easy_install") + else: + distribute_egg_dir = stage0(tmpdir) # STAGE 1: install EasyBuild using easy_install to tmp dir - versions = stage1(tmpdir) + templates = stage1(tmpdir, sourcepath) # STAGE 2: install EasyBuild using EasyBuild (to final target installation dir) - stage2(tmpdir, versions, install_path) + stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath) # clean up the mess debug("Cleaning up %s..." % tmpdir) @@ -419,10 +500,10 @@ def main(): info('') if install_path is not None: info('EasyBuild v%s was installed to %s, so make sure your $MODULEPATH includes %s' % \ - (versions['version'], install_path, os.path.join(install_path, 'modules', 'all'))) + (templates['version'], install_path, os.path.join(install_path, 'modules', 'all'))) else: info('EasyBuild v%s was installed to configured install path, make sure your $MODULEPATH is set correctly.' % \ - versions['version']) + templates['version']) info('(default config => add "$HOME/.local/easybuild/modules/all" in $MODULEPATH)') info('') @@ -431,10 +512,10 @@ def main(): info('') info("By default, EasyBuild will install software to $HOME/.local/easybuild.") info("To install software with EasyBuild to %s, make sure $EASYBUILD_INSTALLPATH is set accordingly." % install_path) - info("See https://github.com/hpcugent/easybuild/wiki/Configuration for details on configuring EasyBuild.") + info("See http://easybuild.readthedocs.org/en/latest/Configuration.html for details on configuring EasyBuild.") # template easyconfig file for EasyBuild -EB_EC_FILE = """ +EASYBUILD_EASYCONFIG_TEMPLATE = """ easyblock = 'EB_EasyBuildMeta' name = 'EasyBuild' @@ -447,21 +528,22 @@ def main(): toolchain = {'name': 'dummy', 'version': 'dummy'} -source_urls = [ - 'http://pypi.python.org/packages/source/e/easybuild-framework/', - 'http://pypi.python.org/packages/source/e/easybuild-easyblocks/', - 'http://pypi.python.org/packages/source/e/easybuild-easyconfigs/', - ] -# order matters a lot, to avoid having dependencies auto-resolved (--no-deps easy_install option doesn't work?) -sources = [ - 'easybuild-framework-%(framework_version)s.tar.gz', - 'easybuild-easyblocks-%(easyblocks_version)s.tar.gz', - 'easybuild-easyconfigs-%(easyconfigs_version)s.tar.gz', - ] +source_urls = [%(source_urls)s] +sources = [%(sources)s] # EasyBuild is a (set of) Python packages, so it depends on Python # usually, we want to use the system Python, so no actual Python dependency is listed allow_system_deps = [('Python', SYS_PYTHON_VERSION)] + +preinstallopts = '%(preinstallopts)s' + +pyshortver = '.'.join(SYS_PYTHON_VERSION.split('.')[:2]) +sanity_check_paths = { + 'files': ['bin/eb'], + 'dirs': ['lib/python%%s/site-packages' %% pyshortver], +} + +moduleclass = 'tools' """ # distribute_setup.py script (https://pypi.python.org/pypi/distribute) diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py new file mode 100755 index 0000000000..ff903b4265 --- /dev/null +++ b/easybuild/scripts/clean_gists.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python +## +# Copyright 2014 Ward Poelmans +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +This script cleans up old gists created by easybuild. It checks if the gists was +created from a pull-request and if that PR is closed/merged, it will delete the gist. +You need a github token for this. The script uses the same username and token +as easybuild. Optionally, you can specify a different github username. + +@author: Ward Poelmans +""" + + +import re + +from vsc.utils import fancylogger +from vsc.utils.generaloption import simple_option +from vsc.utils.rest import RestClient +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.github import GITHUB_API_URL, HTTP_STATUS_OK, GITHUB_EASYCONFIGS_REPO +from easybuild.tools.github import GITHUB_EB_MAIN, fetch_github_token +from easybuild.tools.options import EasyBuildOptions + +HTTP_DELETE_OK = 204 + + +def main(): + """the main function""" + fancylogger.logToScreen(enable=True, stdout=True) + fancylogger.setLogLevelInfo() + + options = { + 'github-user': ('Your github username to use', None, 'store', None, 'g'), + 'closed-pr': ('Delete all gists from closed pull-requests', None, 'store_true', True, 'p'), + 'all': ('Delete all gists from Easybuild ', None, 'store_true', False, 'a'), + 'orphans': ('Delete all gists without a pull-request', None, 'store_true', False, 'o'), + } + + go = simple_option(options) + log = go.log + + if not (go.options.all or go.options.closed_pr or go.options.orphans): + raise EasyBuildError("Please tell me what to do?") + + if go.options.github_user is None: + eb_go = EasyBuildOptions(envvar_prefix='EASYBUILD', go_args=[]) + username = eb_go.options.github_user + log.debug("Fetch github username from easybuild, found: %s", username) + else: + username = go.options.github_user + + if username is None: + raise EasyBuildError("Could not find a github username") + else: + log.info("Using username = %s", username) + + token = fetch_github_token(username) + + gh = RestClient(GITHUB_API_URL, username=username, token=token) + # ToDo: add support for pagination + status, gists = gh.gists.get(per_page=100) + + if status != HTTP_STATUS_OK: + raise EasyBuildError("Failed to get a lists of gists for user %s: error code %s, message = %s", + username, status, gists) + else: + log.info("Found %s gists", len(gists)) + + regex = re.compile(r"(EasyBuild test report|EasyBuild log for failed build).*?(?:PR #(?P[0-9]+))?\)?$") + + pr_cache = {} + num_deleted = 0 + + for gist in gists: + if not gist["description"]: + continue + re_pr_num = regex.search(gist["description"]) + delete_gist = False + + if re_pr_num: + log.debug("Found a Easybuild gist (id=%s)", gist["id"]) + pr_num = re_pr_num.group("PR") + if go.options.all: + delete_gist = True + elif pr_num and go.options.closed_pr: + log.debug("Found Easybuild test report for PR #%s", pr_num) + + if pr_num not in pr_cache: + status, pr = gh.repos[GITHUB_EB_MAIN][GITHUB_EASYCONFIGS_REPO].pulls[pr_num].get() + if status != HTTP_STATUS_OK: + raise EasyBuildError("Failed to get pull-request #%s: error code %s, message = %s", + pr_num, status, pr) + pr_cache[pr_num] = pr["state"] + + if pr_cache[pr_num] == "closed": + log.debug("Found report from closed PR #%s (id=%s)", pr_num, gist["id"]) + delete_gist = True + + elif not pr_num and go.options.orphans: + log.debug("Found Easybuild test report without PR (id=%s)", gist["id"]) + delete_gist = True + + if delete_gist: + status, del_gist = gh.gists[gist["id"]].delete() + + if status != HTTP_DELETE_OK: + raise EasyBuildError("Unable to remove gist (id=%s): error code %s, message = %s", + gist["id"], status, del_gist) + else: + log.info("Delete gist with id=%s", gist["id"]) + num_deleted += 1 + + log.info("Deleted %s gists", num_deleted) + + +if __name__ == '__main__': + main() diff --git a/easybuild/scripts/fix_broken_easyconfigs.py b/easybuild/scripts/fix_broken_easyconfigs.py new file mode 100755 index 0000000000..7f1cc626a0 --- /dev/null +++ b/easybuild/scripts/fix_broken_easyconfigs.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python +# Copyright 2015-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Script to fix easyconfigs that broke due to support for deprecated functionality being dropped in EasyBuild 2.0 + +@author: Kenneth Hoste (Ghent University) +""" +import os +import re +import sys +from vsc.utils import fancylogger +from vsc.utils.generaloption import SimpleOption + +from easybuild.framework.easyconfig.easyconfig import get_easyblock_class +from easybuild.framework.easyconfig.parser import REPLACED_PARAMETERS, fetch_parameters_from_easyconfig +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import init_build_options +from easybuild.tools.filetools import find_easyconfigs, read_file, write_file + + +class FixBrokenEasyconfigsOption(SimpleOption): + """Custom option parser for this script.""" + ALLOPTSMANDATORY = False + + +def fix_broken_easyconfig(ectxt, easyblock_class): + """ + Fix provided easyconfig file, that may be broken due to non-backwards-compatible changes. + @param ectxt: raw contents of easyconfig to fix + @param easyblock_class: easyblock class, as derived from software name/specified easyblock + """ + log.debug("Raw contents of potentially broken easyconfig file to fix: %s" % ectxt) + + subs = { + # replace former 'magic' variable shared_lib_ext with SHLIB_EXT constant + 'shared_lib_ext': 'SHLIB_EXT', + } + # include replaced easyconfig parameters + subs.update(REPLACED_PARAMETERS) + + # check whether any substitions need to be made + for old, new in subs.items(): + regex = re.compile(r'(\W)%s(\W)' % old) + if regex.search(ectxt): + tup = (regex.pattern, old, new) + log.debug("Broken stuff detected using regex pattern '%s', replacing '%s' with '%s'" % tup) + ectxt = regex.sub(r'\1%s\2' % new, ectxt) + + # check whether missing "easyblock = 'ConfigureMake'" needs to be inserted + if easyblock_class is None: + # prepend "easyblock = 'ConfigureMake'" to line containing "name =..." + easyblock_spec = "easyblock = 'ConfigureMake'" + log.debug("Inserting \"%s\", since no easyblock class was derived from easyconfig parameters" % easyblock_spec) + ectxt = re.sub(r'(\s*)(name\s*=)', r"\1%s\n\n\2" % easyblock_spec, ectxt, re.M) + + return ectxt + + +def process_easyconfig_file(ec_file): + """Process an easyconfig file: fix if it's broken, back it up before fixing it inline (if requested).""" + ectxt = read_file(ec_file) + name, easyblock = fetch_parameters_from_easyconfig(ectxt, ['name', 'easyblock']) + derived_easyblock_class = get_easyblock_class(easyblock, name=name, default_fallback=False) + + fixed_ectxt = fix_broken_easyconfig(ectxt, derived_easyblock_class) + + if ectxt != fixed_ectxt: + if go.options.backup: + try: + backup_ec_file = '%s.bk' % ec_file + i = 1 + while os.path.exists(backup_ec_file): + backup_ec_file = '%s.bk%d' % (ec_file, i) + i += 1 + os.rename(ec_file, backup_ec_file) + log.info("Backed up %s to %s" % (ec_file, backup_ec_file)) + except OSError, err: + raise EasyBuildError("Failed to backup %s before rewriting it: %s", ec_file, err) + + write_file(ec_file, fixed_ectxt) + log.debug("Contents of fixed easyconfig file: %s" % fixed_ectxt) + + log.info("%s: fixed" % ec_file) + else: + log.info("%s: nothing to fix" % ec_file) + +# MAIN + +try: + init_build_options() + + options = { + 'backup': ("Backup up easyconfigs before modifying them", None, 'store_true', True, 'b'), + } + go = FixBrokenEasyconfigsOption(options) + log = go.log + + fancylogger.logToScreen(enable=True, stdout=True) + fancylogger.setLogLevel('WARNING') + + try: + import easybuild.easyblocks.generic.configuremake + except ImportError, err: + raise EasyBuildError("easyblocks are not available in Python search path: %s", err) + + for path in go.args: + if not os.path.exists(path): + raise EasyBuildError("Non-existing path %s specified", path) + + ec_files = [ec for p in go.args for ec in find_easyconfigs(p)] + if not ec_files: + raise EasyBuildError("No easyconfig files specified") + + log.info("Processing %d easyconfigs" % len(ec_files)) + for ec_file in ec_files: + try: + process_easyconfig_file(ec_file) + except EasyBuildError, err: + log.warning("Ignoring issue when processing %s: %s", ec_file, err) + +except EasyBuildError, err: + sys.stderr.write("ERROR: %s\n" % err) + sys.exit(1) diff --git a/easybuild/scripts/generate_software_list.py b/easybuild/scripts/generate_software_list.py index 0183ff2c5e..26feaab3b2 100644 --- a/easybuild/scripts/generate_software_list.py +++ b/easybuild/scripts/generate_software_list.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of the University of Ghent (http://ugent.be/hpc). @@ -33,10 +33,10 @@ from datetime import date from optparse import OptionParser -import easybuild.tools.build_log # ensure use of EasyBuildLog import easybuild.tools.config as config import easybuild.tools.options as eboptions from easybuild.framework.easyconfig.easyconfig import EasyConfig, get_easyblock_class +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.github import Githubfs from vsc.utils import fancylogger @@ -101,7 +101,7 @@ # configure EasyBuild, by parsing options eb_go = eboptions.parse_options(args=args) config.init(eb_go.options, eb_go.get_options_by_section('config')) -config.init_build_options({'validate': False}) +config.init_build_options({'validate': False, 'external_modules_metadata': {}}) configs = [] @@ -125,17 +125,18 @@ ec = EasyConfig(ec_file) log.info("found valid easyconfig %s" % ec) if not ec.name in names: - log.info("found new software package %s" % ec) + log.info("found new software package %s" % ec.name) + ec.easyblock = None # check if an easyblock exists - module = get_easyblock_class(None, name=ec.name).__module__.split('.')[-1] - if module != "configuremake": - ec.easyblock = module - else: - ec.easyblock = None + ebclass = get_easyblock_class(None, name=ec.name, default_fallback=False) + if ebclass is not None: + module = ebclass.__module__.split('.')[-1] + if module != "configuremake": + ec.easyblock = module configs.append(ec) names.append(ec.name) except Exception, err: - log.error("faulty easyconfig %s: %s" % (ec_file, err)) + raise EasyBuildError("faulty easyconfig %s: %s", ec_file, err) log.info("Found easyconfigs: %s" % [x.name for x in configs]) # sort by name diff --git a/easybuild/scripts/install-EasyBuild-develop.sh b/easybuild/scripts/install-EasyBuild-develop.sh index 5474adb48b..83e93031ed 100644 --- a/easybuild/scripts/install-EasyBuild-develop.sh +++ b/easybuild/scripts/install-EasyBuild-develop.sh @@ -10,11 +10,11 @@ set -e # Print script help print_usage() { - echo "Usage: $0 github_username install_directory" + echo "Usage: $0 " echo - echo " github_username: username on GitHub for which the easybuild repositories should be cloned" + echo " github_username: username on GitHub for which the EasyBuild repositories should be cloned" echo - echo " install_directory: directory were all the EasyBuild files will be installed" + echo " install_dir: directory were all the EasyBuild files will be installed" echo } @@ -25,17 +25,17 @@ github_clone_branch() BRANCH="$2" cd "${INSTALL_DIR}" - echo "=== Cloning ${REPO} ..." - git clone git@github.com:${GITHUB_USERNAME}/${REPO}.git + echo "=== Cloning ${GITHUB_USERNAME}/${REPO} ..." + git clone --branch master git@github.com:${GITHUB_USERNAME}/${REPO}.git - echo "=== Add and fetch HPC UGent GitHub repository" + echo "=== Adding and fetching HPC-UGent GitHub repository @ hpcugent/{$REPO} ..." cd "${REPO}" git remote add "github_hpcugent" "git@github.com:hpcugent/${REPO}.git" git fetch github_hpcugent # If branch is not 'master', track and checkout it if [ "$BRANCH" != "master" ] ; then - echo "=== Checking out the '${BRANCH}' branch" + echo "=== Checking out the '${BRANCH}' branch ..." git branch --track "${BRANCH}" "github_hpcugent/${BRANCH}" git checkout "${BRANCH}" fi @@ -69,6 +69,7 @@ conflict EasyBuild prepend-path PATH "\$root/easybuild-framework" +prepend-path PYTHONPATH "\$root/vsc-base/lib" prepend-path PYTHONPATH "\$root/easybuild-framework" prepend-path PYTHONPATH "\$root/easybuild-easyblocks" prepend-path PYTHONPATH "\$root/easybuild-easyconfigs" @@ -107,6 +108,9 @@ mkdir -p "${INSTALL_DIR}" cd "${INSTALL_DIR}" INSTALL_DIR="${PWD}" # get the full path +# Clone repository for vsc-base dependency with 'master' branch +github_clone_branch "vsc-base" "master" + # Clone code repositories with the 'develop' branch github_clone_branch "easybuild-framework" "develop" github_clone_branch "easybuild-easyblocks" "develop" @@ -119,11 +123,14 @@ github_clone_branch "easybuild" "master" github_clone_branch "easybuild-wiki" "master" # Create the module file -EB_DEVEL_MODULE="${INSTALL_DIR}/module-EasyBuild-develop" +EB_DEVEL_MODULE_NAME="EasyBuild-develop" +EB_DEVEL_MODULE="${INSTALL_DIR}/${EB_DEVEL_MODULE_NAME}" print_devel_module > "${EB_DEVEL_MODULE}" echo -echo "=== Run 'module load ${EB_DEVEL_MODULE}' to use your development version of EasyBuild." -echo "=== (you can add a symlink in your MODULEPATH to make this module appear together with the others)" +echo "=== Run 'module use ${INSTALL_DIR}' and 'module load ${EB_DEVEL_MODULE_NAME}' to use your development version of EasyBuild." +echo "=== (you can append ${INSTALL_DIR} to your MODULEPATH to make this module always available for loading)" +echo +echo "=== To update each repository, run 'git pull origin' in each subdirectory of ${INSTALL_DIR}" echo exit 0 diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py index 8c25f14135..e3b1b3e220 100755 --- a/easybuild/scripts/mk_tmpl_easyblock_for.py +++ b/easybuild/scripts/mk_tmpl_easyblock_for.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -121,7 +121,7 @@ import easybuild.tools.toolchain as toolchain %(parent_import)s from easybuild.framework.easyconfig import CUSTOM, MANDATORY -from easybuild.tools.filetools import run_cmd +from easybuild.tools.run import run_cmd class %(class_name)s(%(parent)s): @@ -136,11 +136,10 @@ def __init__(self, *args, **kwargs): @staticmethod def extra_options(): \"\"\"Custom easyconfig parameters for %(name)s.\"\"\" - - extra_vars = [ - ('mandatory_extra_param', ['default value', "short description", MANDATORY]), - ('optional_extra_param', ['default value', "short description", CUSTOM]), - ] + extra_vars = { + 'mandatory_extra_param': ['default value', "short description", MANDATORY], + 'optional_extra_param': ['default value', "short description", CUSTOM], + } return %(parent)s.extra_options(extra_vars) def configure_step(self): @@ -208,8 +207,8 @@ def make_module_extra(self): txt = super(%(class_name)s, self).make_module_extra() - txt += self.moduleGenerator.set_environment("VARIABLE", 'value') - txt += self.moduleGenerator.prepend_paths("PATH_VAR", ['path1', 'path2']) + txt += self.module_generator.set_environment("VARIABLE", 'value') + txt += self.module_generator.prepend_paths("PATH_VAR", ['path1', 'path2']) return txt """ diff --git a/easybuild/scripts/port_easyblock.py b/easybuild/scripts/port_easyblock.py index 7fbe90a2d9..515e29edcc 100644 --- a/easybuild/scripts/port_easyblock.py +++ b/easybuild/scripts/port_easyblock.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/prep_for_release.py b/easybuild/scripts/prep_for_release.py index ffc4045807..60b7651e60 100644 --- a/easybuild/scripts/prep_for_release.py +++ b/easybuild/scripts/prep_for_release.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/repo_setup.py b/easybuild/scripts/repo_setup.py index 23686cdd73..2cf4686821 100644 --- a/easybuild/scripts/repo_setup.py +++ b/easybuild/scripts/repo_setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/__init__.py b/easybuild/toolchains/__init__.py index 9c05df7c2b..b0a226866d 100644 --- a/easybuild/toolchains/__init__.py +++ b/easybuild/toolchains/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/cgmpich.py b/easybuild/toolchains/cgmpich.py index a85f500257..dd7f96b3aa 100644 --- a/easybuild/toolchains/cgmpich.py +++ b/easybuild/toolchains/cgmpich.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. @@ -38,4 +38,3 @@ class Cgmpich(ClangGcc, Mpich): """Compiler toolchain with Clang, GFortran and MPICH.""" NAME = 'cgmpich' - COMPILER_MODULE_NAME = ['ClangGCC'] diff --git a/easybuild/toolchains/cgmpolf.py b/easybuild/toolchains/cgmpolf.py index 86668d28bd..fa6a958ab7 100644 --- a/easybuild/toolchains/cgmpolf.py +++ b/easybuild/toolchains/cgmpolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgmvapich2.py b/easybuild/toolchains/cgmvapich2.py index d3a4b596c2..3ef8902633 100644 --- a/easybuild/toolchains/cgmvapich2.py +++ b/easybuild/toolchains/cgmvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. @@ -38,4 +38,3 @@ class Cgmvapich2(ClangGcc, Mvapich2): """Compiler toolchain with Clang, GFortran and MVAPICH2.""" NAME = 'cgmvapich2' - COMPILER_MODULE_NAME = ['ClangGCC'] diff --git a/easybuild/toolchains/cgmvolf.py b/easybuild/toolchains/cgmvolf.py index 682853b2be..dcfc7740f2 100644 --- a/easybuild/toolchains/cgmvolf.py +++ b/easybuild/toolchains/cgmvolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgompi.py b/easybuild/toolchains/cgompi.py index 9fe8105313..c988a62db3 100644 --- a/easybuild/toolchains/cgompi.py +++ b/easybuild/toolchains/cgompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. @@ -38,4 +38,3 @@ class Cgompi(ClangGcc, OpenMPI): """Compiler toolchain with Clang, GFortran and OpenMPI.""" NAME = 'cgompi' - COMPILER_MODULE_NAME = ['ClangGCC'] diff --git a/easybuild/toolchains/cgoolf.py b/easybuild/toolchains/cgoolf.py index a02cbb7b94..abc073c41b 100644 --- a/easybuild/toolchains/cgoolf.py +++ b/easybuild/toolchains/cgoolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/clanggcc.py b/easybuild/toolchains/clanggcc.py index 17208be047..1fe2beb801 100644 --- a/easybuild/toolchains/clanggcc.py +++ b/easybuild/toolchains/clanggcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/compiler/__init__.py b/easybuild/toolchains/compiler/__init__.py index 79d806fe7e..95a05537f5 100644 --- a/easybuild/toolchains/compiler/__init__.py +++ b/easybuild/toolchains/compiler/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/clang.py b/easybuild/toolchains/compiler/clang.py index b78919bba1..4192561fff 100644 --- a/easybuild/toolchains/compiler/clang.py +++ b/easybuild/toolchains/compiler/clang.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. @@ -32,6 +32,7 @@ """ import easybuild.tools.systemtools as systemtools +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.compiler import Compiler @@ -53,38 +54,38 @@ class Clang(Compiler): 'loop-vectorize': ['fvectorize'], 'basic-block-vectorize': ['fslp-vectorize'], 'optarch':'march=native', - - # Clang's options do not map well onto these precision modes. The flags enable and disable certain classes of - # optimizations. - # - # -fassociative-math: allow re-association of operands in series of floating-point operations, violates the - # ISO C and C++ language standard by possibly changing computation result. - # -freciprocal-math: allow optimizations to use the reciprocal of an argument rather than perform division. - # -fsigned-zeros: do not allow optimizations to treat the sign of a zero argument or result as insignificant. - # -fhonor-infinities: disallow optimizations to assume that arguments and results are not +/- Infs. - # -fhonor-nans: disallow optimizations to assume that arguments and results are not +/- NaNs. - # -ffinite-math-only: allow optimizations for floating-point arithmetic that assume that arguments and results - # are not NaNs or +-Infs (equivalent to -fno-honor-nans -fno-honor-infinities) - # -funsafe-math-optimizations: allow unsafe math optimizations (implies -fassociative-math, -fno-signed-zeros, - # -freciprocal-math). - # -ffast-math: an umbrella flag that enables all optimizations listed above, provides preprocessor macro - # __FAST_MATH__. - # - # Using -fno-fast-math is equivalent to disabling all individual optimizations, see - # http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Driver/Tools.cpp?view=markup (lines 2100 and following) - # - # 'strict', 'precise' and 'defaultprec' are all ISO C++ and IEEE complaint, but we explicitly specify details - # flags for strict and precise for robustness against future changes. - 'strict': ['fno-fast-math'], - 'precise': ['fno-unsafe-math-optimizations'], - 'defaultprec': [], - 'loose': ['ffast-math', 'fno-unsafe-math-optimizations'], - 'veryloose': ['ffast-math'], + # Clang's options do not map well onto these precision modes. The flags enable and disable certain classes of + # optimizations. + # + # -fassociative-math: allow re-association of operands in series of floating-point operations, violates the + # ISO C and C++ language standard by possibly changing computation result. + # -freciprocal-math: allow optimizations to use the reciprocal of an argument rather than perform division. + # -fsigned-zeros: do not allow optimizations to treat the sign of a zero argument or result as insignificant. + # -fhonor-infinities: disallow optimizations to assume that arguments and results are not +/- Infs. + # -fhonor-nans: disallow optimizations to assume that arguments and results are not +/- NaNs. + # -ffinite-math-only: allow optimizations for floating-point arithmetic that assume that arguments and results + # are not NaNs or +-Infs (equivalent to -fno-honor-nans -fno-honor-infinities) + # -funsafe-math-optimizations: allow unsafe math optimizations (implies -fassociative-math, -fno-signed-zeros, + # -freciprocal-math). + # -ffast-math: an umbrella flag that enables all optimizations listed above, provides preprocessor macro + # __FAST_MATH__. + # + # Using -fno-fast-math is equivalent to disabling all individual optimizations, see + # http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Driver/Tools.cpp?view=markup (lines 2100 and following) + # + # 'strict', 'precise' and 'defaultprec' are all ISO C++ and IEEE complaint, but we explicitly specify details + # flags for strict and precise for robustness against future changes. + 'strict': ['fno-fast-math'], + 'precise': ['fno-unsafe-math-optimizations'], + 'defaultprec': [], + 'loose': ['ffast-math', 'fno-unsafe-math-optimizations'], + 'veryloose': ['ffast-math'], } COMPILER_OPTIMAL_ARCHITECTURE_OPTION = { systemtools.INTEL : 'march=native', - systemtools.AMD : 'march=native' + systemtools.AMD : 'march=native', + systemtools.POWER: 'mcpu=native', # no support for march=native on POWER } COMPILER_CC = 'clang' @@ -98,6 +99,5 @@ def _set_compiler_vars(self): super(Clang, self)._set_compiler_vars() if self.options.get('32bit', None): - self.log.raiseException("_set_compiler_vars: 32bit set, but no support yet for " \ - "32bit Clang in EasyBuild") + raise EasyBuildError("_set_compiler_vars: 32bit set, but no support yet for 32bit Clang in EasyBuild") diff --git a/easybuild/toolchains/compiler/craype.py b/easybuild/toolchains/compiler/craype.py new file mode 100644 index 0000000000..0383f13121 --- /dev/null +++ b/easybuild/toolchains/compiler/craype.py @@ -0,0 +1,159 @@ +## +# Copyright 2014-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for the Cray Programming Environment (craype) compiler drivers (aka cc, CC, ftn). + +The basic concept is that the compiler driver knows how to invoke the true underlying +compiler with the compiler's specific options tuned to Cray systems. + +That means that certain defaults are set that are specific to Cray's computers. + +The compiler drivers are quite similar to EB toolchains as they include +linker and compiler directives to use the Cray libraries for their MPI (and network drivers) +Cray's LibSci (BLAS/LAPACK et al), FFT library, etc. + +@author: Petar Forai (IMP/IMBA, Austria) +@author: Kenneth Hoste (Ghent University) +""" +import easybuild.tools.environment as env +from easybuild.toolchains.compiler.gcc import TC_CONSTANT_GCC, Gcc +from easybuild.toolchains.compiler.inteliccifort import TC_CONSTANT_INTELCOMP, IntelIccIfort +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option +from easybuild.tools.toolchain.compiler import Compiler + + +TC_CONSTANT_CRAYPE = "CrayPE" +TC_CONSTANT_CRAYCE = "CrayCE" + + +class CrayPECompiler(Compiler): + """Generic support for using Cray compiler drivers.""" + TOOLCHAIN_FAMILY = TC_CONSTANT_CRAYPE + + # compiler module name is PrgEnv, suffix name depends on CrayPE flavor (gnu, intel, cray) + COMPILER_MODULE_NAME = None + # compiler family depends on CrayPE flavor + COMPILER_FAMILY = None + + COMPILER_UNIQUE_OPTS = { + 'dynamic': (False, "Generate dynamically linked executable"), + 'mpich-mt': (False, "Directs the driver to link in an alternate version of the Cray-MPICH library which \ + provides fine-grained multi-threading support to applications that perform \ + MPI operations within threaded regions."), + 'optarch': (False, "Enable architecture optimizations"), + 'verbose': (True, "Verbose output"), + } + + COMPILER_UNIQUE_OPTION_MAP = { + # handle shared and dynamic always via $CRAYPE_LINK_TYPE environment variable, don't pass flags to wrapper + 'shared': '', + 'dynamic': '', + 'verbose': 'craype-verbose', + 'mpich-mt': 'craympich-mt', + } + + COMPILER_CC = 'cc' + COMPILER_CXX = 'CC' + + COMPILER_F77 = 'ftn' + COMPILER_F90 = 'ftn' + COMPILER_FC = 'ftn' + + # suffix for PrgEnv module that matches this toolchain + # e.g. 'gnu' => 'PrgEnv-gnu/' + PRGENV_MODULE_NAME_SUFFIX = None + + # template for craype module (determines code generator backend of Cray compiler wrappers) + CRAYPE_MODULE_NAME_TEMPLATE = 'craype-%(optarch)s' + + def __init__(self, *args, **kwargs): + """Constructor.""" + super(CrayPECompiler, self).__init__(*args, **kwargs) + # 'register' additional toolchain options that correspond to a compiler flag + self.COMPILER_FLAGS.extend(['dynamic', 'mpich-mt']) + + # use name of PrgEnv module as name of module that provides compiler + self.COMPILER_MODULE_NAME = ['PrgEnv-%s' % self.PRGENV_MODULE_NAME_SUFFIX] + + def _set_optimal_architecture(self): + """Load craype module specified via 'optarch' build option.""" + optarch = build_option('optarch') + if optarch is None: + raise EasyBuildError("Don't know which 'craype' module to load, 'optarch' build option is unspecified.") + else: + craype_mod_name = self.CRAYPE_MODULE_NAME_TEMPLATE % {'optarch': optarch} + if self.modules_tool.exist([craype_mod_name])[0]: + self.modules_tool.load([craype_mod_name]) + else: + raise EasyBuildError("Necessary craype module with name '%s' is not available (optarch: '%s')", + craype_mod_name, optarch) + + # no compiler flag when optarch toolchain option is enabled + self.options.options_map['optarch'] = '' + + def prepare(self, *args, **kwargs): + """Prepare to use this toolchain; define $CRAYPE_LINK_TYPE if 'dynamic' toolchain option is enabled.""" + super(CrayPECompiler, self).prepare(*args, **kwargs) + + if self.options['dynamic'] or self.options['shared']: + self.log.debug("Enabling building of shared libs/dynamically linked executables via $CRAYPE_LINK_TYPE") + env.setvar('CRAYPE_LINK_TYPE', 'dynamic') + + +class CrayPEGCC(CrayPECompiler): + """Support for using the Cray GNU compiler wrappers.""" + PRGENV_MODULE_NAME_SUFFIX = 'gnu' # PrgEnv-gnu + COMPILER_FAMILY = TC_CONSTANT_GCC + + def __init__(self, *args, **kwargs): + """CrayPEGCC constructor.""" + super(CrayPEGCC, self).__init__(*args, **kwargs) + for precflag in self.COMPILER_PREC_FLAGS: + self.COMPILER_UNIQUE_OPTION_MAP[precflag] = Gcc.COMPILER_UNIQUE_OPTION_MAP[precflag] + + +class CrayPEIntel(CrayPECompiler): + """Support for using the Cray Intel compiler wrappers.""" + PRGENV_MODULE_NAME_SUFFIX = 'intel' # PrgEnv-intel + COMPILER_FAMILY = TC_CONSTANT_INTELCOMP + + def __init__(self, *args, **kwargs): + """CrayPEIntel constructor.""" + super(CrayPEIntel, self).__init__(*args, **kwargs) + for precflag in self.COMPILER_PREC_FLAGS: + self.COMPILER_UNIQUE_OPTION_MAP[precflag] = IntelIccIfort.COMPILER_UNIQUE_OPTION_MAP[precflag] + + +class CrayPECray(CrayPECompiler): + """Support for using the Cray CCE compiler wrappers.""" + PRGENV_MODULE_NAME_SUFFIX = 'cray' # PrgEnv-cray + COMPILER_FAMILY = TC_CONSTANT_CRAYCE + + def __init__(self, *args, **kwargs): + """CrayPEIntel constructor.""" + super(CrayPECray, self).__init__(*args, **kwargs) + for precflag in self.COMPILER_PREC_FLAGS: + self.COMPILER_UNIQUE_OPTION_MAP[precflag] = [] diff --git a/easybuild/toolchains/compiler/cuda.py b/easybuild/toolchains/compiler/cuda.py index 7304b8b1f2..ffbe45099c 100644 --- a/easybuild/toolchains/compiler/cuda.py +++ b/easybuild/toolchains/compiler/cuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -96,7 +96,7 @@ def _set_compiler_flags(self): self.variables.nappend('CUDA_CXXFLAGS', cuda_flags) # add gencode compiler flags to list of flags for compiler variables - for gencode_val in self.options['cuda_gencode']: + for gencode_val in self.options.get('cuda_gencode', []): gencode_option = 'gencode %s' % gencode_val self.variables.nappend('CUDA_CFLAGS', gencode_option) self.variables.nappend('CUDA_CXXFLAGS', gencode_option) diff --git a/easybuild/toolchains/compiler/dummycompiler.py b/easybuild/toolchains/compiler/dummycompiler.py index f99b687c49..cc1dea94ea 100644 --- a/easybuild/toolchains/compiler/dummycompiler.py +++ b/easybuild/toolchains/compiler/dummycompiler.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -46,3 +46,4 @@ class DummyCompiler(Compiler): COMPILER_F77 = '%sF77' % TC_CONSTANT_DUMMY COMPILER_F90 = '%sF90' % TC_CONSTANT_DUMMY + COMPILER_FC = '%sFC' % TC_CONSTANT_DUMMY diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index e47a9c4778..cc1eb80e2a 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,6 +30,7 @@ """ import easybuild.tools.systemtools as systemtools +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.compiler import Compiler @@ -43,31 +44,30 @@ class Gcc(Compiler): COMPILER_FAMILY = TC_CONSTANT_GCC COMPILER_UNIQUE_OPTS = { - 'loop': (False, "Automatic loop parallellisation"), - 'f2c': (False, "Generate code compatible with f2c and f77"), - 'lto':(False, "Enable Link Time Optimization"), - } + 'loop': (False, "Automatic loop parallellisation"), + 'f2c': (False, "Generate code compatible with f2c and f77"), + 'lto':(False, "Enable Link Time Optimization"), + } COMPILER_UNIQUE_OPTION_MAP = { - 'i8': 'fdefault-integer-8', - 'r8': 'fdefault-real-8', - 'unroll': 'funroll-loops', - 'f2c': 'ff2c', - 'loop': ['ftree-switch-conversion', 'floop-interchange', - 'floop-strip-mine', 'floop-block'], - 'lto':'flto', - 'optarch':'march=native', - 'openmp':'fopenmp', - 'strict': ['mieee-fp', 'mno-recip'], - 'precise':['mno-recip'], - 'defaultprec':[], - 'loose': ['mrecip', 'mno-ieee-fp'], - 'veryloose': ['mrecip=all', 'mno-ieee-fp'], - } + 'i8': 'fdefault-integer-8', + 'r8': 'fdefault-real-8', + 'unroll': 'funroll-loops', + 'f2c': 'ff2c', + 'loop': ['ftree-switch-conversion', 'floop-interchange', 'floop-strip-mine', 'floop-block'], + 'lto': 'flto', + 'openmp': 'fopenmp', + 'strict': ['mieee-fp', 'mno-recip'], + 'precise':['mno-recip'], + 'defaultprec':[], + 'loose': ['mrecip', 'mno-ieee-fp'], + 'veryloose': ['mrecip=all', 'mno-ieee-fp'], + } COMPILER_OPTIMAL_ARCHITECTURE_OPTION = { - systemtools.INTEL : 'march=native', - systemtools.AMD : 'march=native' - } + systemtools.AMD : 'march=native', + systemtools.INTEL : 'march=native', + systemtools.POWER: 'mcpu=native', # no support for march=native on POWER + } COMPILER_CC = 'gcc' COMPILER_CXX = 'g++' @@ -75,6 +75,7 @@ class Gcc(Compiler): COMPILER_F77 = 'gfortran' COMPILER_F90 = 'gfortran' + COMPILER_FC = 'gfortran' COMPILER_F_UNIQUE_FLAGS = ['f2c'] LIB_MULTITHREAD = ['pthread'] @@ -84,8 +85,7 @@ def _set_compiler_vars(self): super(Gcc, self)._set_compiler_vars() if self.options.get('32bit', None): - self.log.raiseException("_set_compiler_vars: 32bit set, but no support yet for " \ - "32bit GCC in EasyBuild") + raise EasyBuildError("_set_compiler_vars: 32bit set, but no support yet for 32bit GCC in EasyBuild") # to get rid of lots of problems with libgfortranbegin # or remove the system gcc-gfortran diff --git a/easybuild/toolchains/compiler/inteliccifort.py b/easybuild/toolchains/compiler/inteliccifort.py index c224278eeb..58e33b9d41 100644 --- a/easybuild/toolchains/compiler/inteliccifort.py +++ b/easybuild/toolchains/compiler/inteliccifort.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -32,6 +32,7 @@ from distutils.version import LooseVersion import easybuild.tools.systemtools as systemtools +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.compiler import Compiler @@ -56,7 +57,7 @@ class IntelIccIfort(Compiler): COMPILER_UNIQUE_OPTION_MAP = { 'i8': 'i8', 'r8': 'r8', - 'optarch': 'xHOST', + 'optarch': 'xHost', 'openmp': 'openmp', # both -openmp/-fopenmp are valid for enabling OpenMP 'strict': ['fp-speculation=strict', 'fp-model strict'], 'precise': ['fp-model precise'], @@ -69,8 +70,8 @@ class IntelIccIfort(Compiler): } COMPILER_OPTIMAL_ARCHITECTURE_OPTION = { - systemtools.INTEL : 'xHOST', - systemtools.AMD : 'msse3', + systemtools.INTEL : 'xHost', + systemtools.AMD : 'xHost', } COMPILER_CC = 'icc' @@ -79,6 +80,7 @@ class IntelIccIfort(Compiler): COMPILER_F77 = 'ifort' COMPILER_F90 = 'ifort' + COMPILER_FC = 'ifort' COMPILER_F_UNIQUE_FLAGS = ['intel-static'] LINKER_TOGGLE_STATIC_DYNAMIC = { @@ -86,21 +88,28 @@ class IntelIccIfort(Compiler): 'dynamic':'-Bdynamic', } - LIB_MULTITHREAD = ['iomp5', 'pthread'] ## iomp5 is OpenMP related + LIB_MULTITHREAD = ['iomp5', 'pthread'] # iomp5 is OpenMP related + + def __init__(self, *args, **kwargs): + """Toolchain constructor.""" + class_constants = kwargs.setdefault('class_constants', []) + class_constants.append('LIB_MULTITHREAD') + super(IntelIccIfort, self).__init__(*args, **kwargs) def _set_compiler_vars(self): """Intel compilers-specific adjustments after setting compiler variables.""" super(IntelIccIfort, self)._set_compiler_vars() if not ('icc' in self.COMPILER_MODULE_NAME and 'ifort' in self.COMPILER_MODULE_NAME): - self.log.raiseException("_set_compiler_vars: missing icc and/or ifort from COMPILER_MODULE_NAME %s" % self.COMPILER_MODULE_NAME) + raise EasyBuildError("_set_compiler_vars: missing icc and/or ifort from COMPILER_MODULE_NAME %s", + self.COMPILER_MODULE_NAME) icc_root, _ = self.get_software_root(self.COMPILER_MODULE_NAME) icc_version, ifort_version = self.get_software_version(self.COMPILER_MODULE_NAME) if not ifort_version == icc_version: - msg = "_set_compiler_vars: mismatch between icc version %s and ifort version %s" - self.log.raiseException(msg % (icc_version, ifort_version)) + raise EasyBuildError("_set_compiler_vars: mismatch between icc version %s and ifort version %s", + icc_version, ifort_version) if LooseVersion(icc_version) < LooseVersion('2011'): self.LIB_MULTITHREAD.insert(1, "guide") diff --git a/easybuild/toolchains/craycce.py b/easybuild/toolchains/craycce.py new file mode 100644 index 0000000000..db86c0b266 --- /dev/null +++ b/easybuild/toolchains/craycce.py @@ -0,0 +1,44 @@ +## +# Copyright 2014-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +CrayCCE toolchain: Cray compilers (CCE) and MPI via Cray compiler drivers + LibSci (PrgEnv-cray) and Cray FFTW + +@author: Petar Forai (IMP/IMBA, Austria) +@author: Kenneth Hoste (Ghent University) +""" +from easybuild.toolchains.compiler.craype import CrayPECray +from easybuild.toolchains.fft.crayfftw import CrayFFTW +from easybuild.toolchains.linalg.libsci import LibSci +from easybuild.toolchains.mpi.craympich import CrayMPICH + + +class CrayCCE(CrayPECray, CrayMPICH, LibSci, CrayFFTW): + """Compiler toolchain for Cray Programming Environment for Cray Compiling Environment (CCE) (PrgEnv-cray).""" + NAME = 'CrayCCE' + + def prepare(self, *args, **kwargs): + """Prepare to use this toolchain; marked as experimental.""" + self.log.experimental("Using %s toolchain", self.NAME) + super(CrayCCE, self).prepare(*args, **kwargs) diff --git a/easybuild/toolchains/craygnu.py b/easybuild/toolchains/craygnu.py new file mode 100644 index 0000000000..9deef321ec --- /dev/null +++ b/easybuild/toolchains/craygnu.py @@ -0,0 +1,44 @@ +## +# Copyright 2014-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +CrayGNU toolchain: GCC and MPI via Cray compiler drivers + LibSci (PrgEnv-gnu) and Cray FFTW + +@author: Petar Forai (IMP/IMBA, Austria) +@author: Kenneth Hoste (Ghent University) +""" +from easybuild.toolchains.compiler.craype import CrayPEGCC +from easybuild.toolchains.fft.crayfftw import CrayFFTW +from easybuild.toolchains.linalg.libsci import LibSci +from easybuild.toolchains.mpi.craympich import CrayMPICH + + +class CrayGNU(CrayPEGCC, CrayMPICH, LibSci, CrayFFTW): + """Compiler toolchain for Cray Programming Environment for GCC compilers (PrgEnv-gnu).""" + NAME = 'CrayGNU' + + def prepare(self, *args, **kwargs): + """Prepare to use this toolchain; marked as experimental.""" + self.log.experimental("Using %s toolchain", self.NAME) + super(CrayGNU, self).prepare(*args, **kwargs) diff --git a/easybuild/toolchains/crayintel.py b/easybuild/toolchains/crayintel.py new file mode 100644 index 0000000000..92ee0e9b1a --- /dev/null +++ b/easybuild/toolchains/crayintel.py @@ -0,0 +1,44 @@ +## +# Copyright 2014-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +CrayIntel toolchain: Intel compilers and MPI via Cray compiler drivers + LibSci (PrgEnv-intel) and Cray FFTW + +@author: Petar Forai (IMP/IMBA, Austria) +@author: Kenneth Hoste (Ghent University) +""" +from easybuild.toolchains.compiler.craype import CrayPEIntel +from easybuild.toolchains.fft.crayfftw import CrayFFTW +from easybuild.toolchains.linalg.libsci import LibSci +from easybuild.toolchains.mpi.craympich import CrayMPICH + + +class CrayIntel(CrayPEIntel, CrayMPICH, LibSci, CrayFFTW): + """Compiler toolchain for Cray Programming Environment for Intel compilers (PrgEnv-intel).""" + NAME = 'CrayIntel' + + def prepare(self, *args, **kwargs): + """Prepare to use this toolchain; marked as experimental.""" + self.log.experimental("Using %s toolchain", self.NAME) + super(CrayIntel, self).prepare(*args, **kwargs) diff --git a/easybuild/toolchains/dummy.py b/easybuild/toolchains/dummy.py index c7791744fe..c641b89952 100644 --- a/easybuild/toolchains/dummy.py +++ b/easybuild/toolchains/dummy.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fft/__init__.py b/easybuild/toolchains/fft/__init__.py index 8490061d53..0e74f343d7 100644 --- a/easybuild/toolchains/fft/__init__.py +++ b/easybuild/toolchains/fft/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fft/crayfftw.py b/easybuild/toolchains/fft/crayfftw.py new file mode 100644 index 0000000000..b575cbc9d9 --- /dev/null +++ b/easybuild/toolchains/fft/crayfftw.py @@ -0,0 +1,71 @@ +## +# Copyright 2014-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for Cray FFTW. + +@author: Petar Forai (IMP/IMBA, Austria) +@author: Kenneth Hoste (Ghent University) +""" +import os + +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.tools.build_log import EasyBuildError + + +class CrayFFTW(Fftw): + """Support for Cray FFTW.""" + # FFT support, via Cray-provided fftw module + FFT_MODULE_NAME = ['fftw'] + + def _get_software_root(self, name): + """Get install prefix for specified software name; special treatment for Cray modules.""" + if name == 'fftw': + # Cray-provided fftw module + env_var = 'FFTW_INC' + incdir = os.getenv(env_var, None) + if incdir is None: + raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_var) + else: + root = os.path.dirname(incdir) + self.log.debug("Obtained install prefix for %s via $%s: %s", name, env_var, root) + else: + root = super(CrayFFTW, self)._get_software_root(name) + + return root + + def _get_software_version(self, name): + """Get version for specified software name; special treatment for Cray modules.""" + if name == 'fftw': + # Cray-provided fftw module + env_var = 'FFTW_VERSION' + ver = os.getenv(env_var, None) + if ver is None: + raise EasyBuildError("Failed to determine version for %s via $%s", name, env_var) + else: + self.log.debug("Obtained version for %s via $%s: %s", name, env_var, ver) + else: + ver = super(CrayFFTW, self)._get_software_version(name) + + return ver diff --git a/easybuild/toolchains/fft/fftw.py b/easybuild/toolchains/fft/fftw.py index 1d36693733..1e9a9c3825 100644 --- a/easybuild/toolchains/fft/fftw.py +++ b/easybuild/toolchains/fft/fftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -31,6 +31,7 @@ from distutils.version import LooseVersion +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.fft import Fft @@ -44,7 +45,7 @@ def _set_fftw_variables(self): suffix = '' version = self.get_software_version(self.FFT_MODULE_NAME)[0] if LooseVersion(version) < LooseVersion('2') or LooseVersion(version) >= LooseVersion('4'): - self.log.raiseException("_set_fft_variables: FFTW unsupported version %s (major should be 2 or 3)" % version) + raise EasyBuildError("_set_fft_variables: FFTW unsupported version %s (major should be 2 or 3)", version) elif LooseVersion(version) > LooseVersion('2'): suffix = '3' diff --git a/easybuild/toolchains/fft/intelfftw.py b/easybuild/toolchains/fft/intelfftw.py index 4dc4c12ca6..94b53c7372 100644 --- a/easybuild/toolchains/fft/intelfftw.py +++ b/easybuild/toolchains/fft/intelfftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -31,9 +31,10 @@ import os from distutils.version import LooseVersion +from easybuild.tools.build_log import EasyBuildError from easybuild.toolchains.fft.fftw import Fftw from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.utilities import all, any + class IntelFFTW(Fftw): """FFTW wrapper functionality of Intel MKL""" @@ -45,26 +46,35 @@ class IntelFFTW(Fftw): def _set_fftw_variables(self): if not hasattr(self, 'BLAS_LIB_DIR'): - self.log.raiseException("_set_fftw_variables: IntelFFT based on IntelMKL (no BLAS_LIB_DIR found)") + raise EasyBuildError("_set_fftw_variables: IntelFFT based on IntelMKL (no BLAS_LIB_DIR found)") imklver = get_software_version(self.FFT_MODULE_NAME[0]) - fftwsuff = "" + picsuff = '' if self.options.get('pic', None): - fftwsuff = "_pic" - fftw_libs = ["fftw3xc_intel%s" % fftwsuff] + picsuff = '_pic' + bitsuff = '_lp64' + if self.options.get('i8', None): + bitsuff = '_ilp64' + compsuff = '_intel' + if get_software_root('icc') is None: + if get_software_root('GCC'): + compsuff = '_gnu' + else: + raise EasyBuildError("Not using Intel compilers or GCC, don't know compiler suffix for FFTW libraries.") + + fftw_libs = ["fftw3xc%s%s" % (compsuff, picsuff)] if self.options['usempi']: - # add cluster interface - if LooseVersion(imklver) < LooseVersion("11.1"): - if LooseVersion(imklver) >= LooseVersion("11.0"): - fftw_libs.append("fftw3x_cdft_lp64%s" % fftwsuff) - elif LooseVersion(imklver) >= LooseVersion("10.3"): - fftw_libs.append("fftw3x_cdft%s" % fftwsuff) - fftw_libs.append("mkl_cdft_core") # add cluster dft - fftw_libs.extend(self.variables['LIBBLACS'].flatten()) ## add BLACS; use flatten because ListOfList + # add cluster interface for recent imkl versions + if LooseVersion(imklver) >= LooseVersion("11.0.2"): + fftw_libs.append("fftw3x_cdft%s%s" % (bitsuff, picsuff)) + elif LooseVersion(imklver) >= LooseVersion("10.3"): + fftw_libs.append("fftw3x_cdft%s" % picsuff) + fftw_libs.append("mkl_cdft_core") # add cluster dft + fftw_libs.extend(self.variables['LIBBLACS'].flatten()) # add BLACS; use flatten because ListOfList self.log.debug('fftw_libs %s' % fftw_libs.__repr__()) - fftw_libs.extend(self.variables['LIBBLAS'].flatten()) ## add core (contains dft) ; use flatten because ListOfList + fftw_libs.extend(self.variables['LIBBLAS'].flatten()) # add BLAS libs (contains dft) self.log.debug('fftw_libs %s' % fftw_libs.__repr__()) self.FFT_LIB_DIR = self.BLAS_LIB_DIR @@ -74,8 +84,11 @@ def _set_fftw_variables(self): # so make sure libraries are there before FFT_LIB is set imklroot = get_software_root(self.FFT_MODULE_NAME[0]) fft_lib_dirs = [os.path.join(imklroot, d) for d in self.FFT_LIB_DIR] - if all([any([os.path.exists(os.path.join(d, "lib%s.a" % lib)) for d in fft_lib_dirs]) for lib in fftw_libs]): + # filter out gfortran from list of FFTW libraries to check for, since it's not provided by imkl + check_fftw_libs = [lib for lib in fftw_libs if lib != 'gfortran'] + fftw_lib_exists = lambda x: any([os.path.exists(os.path.join(d, "lib%s.a" % x)) for d in fft_lib_dirs]) + if all([fftw_lib_exists(lib) for lib in check_fftw_libs]): self.FFT_LIB = fftw_libs else: - msg = "Not all FFTW interface libraries %s are found in %s, can't set FFT_LIB." % (fftw_libs, fft_lib_dirs) - self.log.error(msg) + raise EasyBuildError("Not all FFTW interface libraries %s are found in %s, can't set FFT_LIB.", + check_fftw_libs, fft_lib_dirs) diff --git a/easybuild/toolchains/foss.py b/easybuild/toolchains/foss.py index d67dd5267c..1a8b7c69c4 100755 --- a/easybuild/toolchains/foss.py +++ b/easybuild/toolchains/foss.py @@ -1,5 +1,5 @@ ## -# Copyright 2013 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gcc.py b/easybuild/toolchains/gcc.py index 749dd7cd9c..60115094d7 100644 --- a/easybuild/toolchains/gcc.py +++ b/easybuild/toolchains/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gcccuda.py b/easybuild/toolchains/gcccuda.py index 9bd72024bb..c76c53d200 100644 --- a/easybuild/toolchains/gcccuda.py +++ b/easybuild/toolchains/gcccuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gimkl.py b/easybuild/toolchains/gimkl.py index 894eda83af..a9ae8db870 100644 --- a/easybuild/toolchains/gimkl.py +++ b/easybuild/toolchains/gimkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gimpi.py b/easybuild/toolchains/gimpi.py new file mode 100644 index 0000000000..fb525d962b --- /dev/null +++ b/easybuild/toolchains/gimpi.py @@ -0,0 +1,38 @@ +## +# Copyright 2012-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gimpi compiler toolchain (includes GCC and Intel MPI). + +@author: Stijn De Weirdt (Ghent University) +@author: Kenneth Hoste (Ghent University) +""" + +from easybuild.toolchains.compiler.gcc import Gcc +from easybuild.toolchains.mpi.intelmpi import IntelMPI + + +class Gimpi(Gcc, IntelMPI): + """Compiler toolchain with GCC and Intel MPI.""" + NAME = 'gimpi' diff --git a/easybuild/toolchains/gmacml.py b/easybuild/toolchains/gmacml.py index b76b3e3838..d64d715c90 100644 --- a/easybuild/toolchains/gmacml.py +++ b/easybuild/toolchains/gmacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpich.py b/easybuild/toolchains/gmpich.py new file mode 100644 index 0000000000..8640dc68ca --- /dev/null +++ b/easybuild/toolchains/gmpich.py @@ -0,0 +1,37 @@ +## +# Copyright 2012-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gmpich compiler toolchain (includes GCC and MPICH). + +@author: Kenneth Hoste (Ghent University) +""" + +from easybuild.toolchains.compiler.gcc import Gcc +from easybuild.toolchains.mpi.mpich import Mpich + + +class Gmpich(Gcc, Mpich): + """Compiler toolchain with GCC and MPICH.""" + NAME = 'gmpich' diff --git a/easybuild/toolchains/gmpich2.py b/easybuild/toolchains/gmpich2.py index caf6e0af5b..b70848a354 100644 --- a/easybuild/toolchains/gmpich2.py +++ b/easybuild/toolchains/gmpich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpolf.py b/easybuild/toolchains/gmpolf.py index 077290259e..e849650be2 100644 --- a/easybuild/toolchains/gmpolf.py +++ b/easybuild/toolchains/gmpolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. @@ -36,9 +36,9 @@ from easybuild.toolchains.fft.fftw import Fftw from easybuild.toolchains.linalg.openblas import OpenBLAS from easybuild.toolchains.linalg.scalapack import ScaLAPACK -from easybuild.toolchains.mpi.mpich2 import Mpich2 +from easybuild.toolchains.mpi.mpich import Mpich -class Gmpolf(Gcc, Mpich2, OpenBLAS, ScaLAPACK, Fftw): - """Compiler toolchain with GCC, MPICH2, OpenBLAS, ScaLAPACK and FFTW.""" +class Gmpolf(Gcc, Mpich, OpenBLAS, ScaLAPACK, Fftw): + """Compiler toolchain with GCC, MPICH, OpenBLAS, ScaLAPACK and FFTW.""" NAME = 'gmpolf' diff --git a/easybuild/toolchains/gmvapich2.py b/easybuild/toolchains/gmvapich2.py index 03c4b94366..438d209d07 100644 --- a/easybuild/toolchains/gmvapich2.py +++ b/easybuild/toolchains/gmvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmvolf.py b/easybuild/toolchains/gmvolf.py index 9e8d9ee34d..90ba0e39e1 100644 --- a/easybuild/toolchains/gmvolf.py +++ b/easybuild/toolchains/gmvolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/gnu.py b/easybuild/toolchains/gnu.py new file mode 100644 index 0000000000..8b7f97ad70 --- /dev/null +++ b/easybuild/toolchains/gnu.py @@ -0,0 +1,36 @@ +## +# Copyright 2012-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for GCC compiler toolchain. + +@author: Kenneth Hoste (Ghent University) +""" + +from easybuild.toolchains.compiler.gcc import Gcc + + +class GNU(Gcc): + """Compiler-only toolchain, including only GCC and binutils.""" + NAME = 'GNU' diff --git a/easybuild/toolchains/goalf.py b/easybuild/toolchains/goalf.py index 8c71ee927b..28a087887b 100644 --- a/easybuild/toolchains/goalf.py +++ b/easybuild/toolchains/goalf.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gompi.py b/easybuild/toolchains/gompi.py index 18698f9cd7..cbea82b54c 100644 --- a/easybuild/toolchains/gompi.py +++ b/easybuild/toolchains/gompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gompic.py b/easybuild/toolchains/gompic.py index b6b0b26edb..984ea26272 100644 --- a/easybuild/toolchains/gompic.py +++ b/easybuild/toolchains/gompic.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -26,7 +26,7 @@ EasyBuild support for gompic compiler toolchain (includes GCC and OpenMPI and CUDA). @author: Kenneth Hoste (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ from easybuild.toolchains.gcccuda import GccCUDA diff --git a/easybuild/toolchains/goolf.py b/easybuild/toolchains/goolf.py index 2970099d71..6b51b34e66 100644 --- a/easybuild/toolchains/goolf.py +++ b/easybuild/toolchains/goolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goolfc.py b/easybuild/toolchains/goolfc.py index 96e2f49574..184272a6f0 100644 --- a/easybuild/toolchains/goolfc.py +++ b/easybuild/toolchains/goolfc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gpsmpi.py b/easybuild/toolchains/gpsmpi.py new file mode 100644 index 0000000000..0109d0b389 --- /dev/null +++ b/easybuild/toolchains/gpsmpi.py @@ -0,0 +1,37 @@ +## +# Copyright 2012-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gpsmpi compiler toolchain (includes GCC and Parastation MPICH). + +""" + +from easybuild.toolchains.gmpich import Gmpich + + +class Gpsmpi(Gmpich): + """Compiler toolchain with GCC and Parastation MPICH.""" + NAME = 'gpsmpi' + # Use Parastation naming + MPI_MODULE_NAME = ["psmpi"] diff --git a/easybuild/toolchains/gpsolf.py b/easybuild/toolchains/gpsolf.py new file mode 100644 index 0000000000..592556f148 --- /dev/null +++ b/easybuild/toolchains/gpsolf.py @@ -0,0 +1,41 @@ +## +# Copyright 2013-2015 Ghent University +# +# This file is triple-licensed under GPLv2 (see below), MIT, and +# BSD three-clause licenses. +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gmpolf compiler toolchain (includes GCC, Parastation MPICH, OpenBLAS, LAPACK, ScaLAPACK and FFTW). + +""" + +from easybuild.toolchains.gpsmpi import Gpsmpi +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.linalg.openblas import OpenBLAS +from easybuild.toolchains.linalg.scalapack import ScaLAPACK + + +class Gpsolf(Gpsmpi, OpenBLAS, ScaLAPACK, Fftw): + """Compiler toolchain with GCC, Parastation MPICH, OpenBLAS, ScaLAPACK and FFTW.""" + NAME = 'gpsolf' diff --git a/easybuild/toolchains/gqacml.py b/easybuild/toolchains/gqacml.py index 9731a5bfc7..f7426a9055 100644 --- a/easybuild/toolchains/gqacml.py +++ b/easybuild/toolchains/gqacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iccifort.py b/easybuild/toolchains/iccifort.py index 748fc8b4b1..4c688d4eb5 100644 --- a/easybuild/toolchains/iccifort.py +++ b/easybuild/toolchains/iccifort.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ictce.py b/easybuild/toolchains/ictce.py index 9cf59a1eaf..468741937d 100644 --- a/easybuild/toolchains/ictce.py +++ b/easybuild/toolchains/ictce.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iimpi.py b/easybuild/toolchains/iimpi.py new file mode 100644 index 0000000000..ef5dddeafe --- /dev/null +++ b/easybuild/toolchains/iimpi.py @@ -0,0 +1,42 @@ +## +# Copyright 2012-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for intel compiler toolchain (includes Intel compilers (icc, ifort), Intel MPI). + +@author: Stijn De Weirdt (Ghent University) +@author: Kenneth Hoste (Ghent University) +""" + +from easybuild.toolchains.compiler.inteliccifort import IntelIccIfort +from easybuild.toolchains.fft.intelfftw import IntelFFTW +from easybuild.toolchains.mpi.intelmpi import IntelMPI +from easybuild.toolchains.linalg.intelmkl import IntelMKL + + +class Iimpi(IntelIccIfort, IntelMPI): + """ + Compiler toolchain with Intel compilers (icc/ifort), Intel MPI. + """ + NAME = 'iimpi' diff --git a/easybuild/toolchains/iiqmpi.py b/easybuild/toolchains/iiqmpi.py index 58141a9471..4ef089d181 100644 --- a/easybuild/toolchains/iiqmpi.py +++ b/easybuild/toolchains/iiqmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/impich.py b/easybuild/toolchains/impich.py new file mode 100644 index 0000000000..f837f77325 --- /dev/null +++ b/easybuild/toolchains/impich.py @@ -0,0 +1,38 @@ +## +# Copyright 2013-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for impich compiler toolchain (includes Intel compilers (icc, ifort), MPICH. + +""" + +from easybuild.toolchains.compiler.inteliccifort import IntelIccIfort +from easybuild.toolchains.mpi.mpich import Mpich + + +class Impich(IntelIccIfort, Mpich): + """ + Compiler toolchain with Intel compilers (icc/ifort), MPICH. + """ + NAME = 'impich' diff --git a/easybuild/toolchains/impmkl.py b/easybuild/toolchains/impmkl.py index ba0f1521b6..7c8f3eaad2 100644 --- a/easybuild/toolchains/impmkl.py +++ b/easybuild/toolchains/impmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -23,20 +23,19 @@ # along with EasyBuild. If not, see . ## """ -EasyBuild support for impmkl compiler toolchain (includes Intel compilers (icc, ifort), MPICH2, +EasyBuild support for impmkl compiler toolchain (includes Intel compilers (icc, ifort), MPICH, Intel Math Kernel Library (MKL) , and Intel FFTW wrappers. @author: Kenneth Hoste (Ghent University) """ -from easybuild.toolchains.compiler.inteliccifort import IntelIccIfort +from easybuild.toolchains.impich import Impich from easybuild.toolchains.fft.intelfftw import IntelFFTW -from easybuild.toolchains.mpi.mpich2 import Mpich2 from easybuild.toolchains.linalg.intelmkl import IntelMKL -class Impmkl(IntelIccIfort, Mpich2, IntelMKL, IntelFFTW): +class Impmkl(Impich, IntelMKL, IntelFFTW): """ - Compiler toolchain with Intel compilers (icc/ifort), MPICH2, + Compiler toolchain with Intel compilers (icc/ifort), MPICH, Intel Math Kernel Library (MKL) and Intel FFTW wrappers. """ NAME = 'impmkl' diff --git a/easybuild/toolchains/intel-para.py b/easybuild/toolchains/intel-para.py new file mode 100644 index 0000000000..31e12d638b --- /dev/null +++ b/easybuild/toolchains/intel-para.py @@ -0,0 +1,42 @@ +## +# Copyright 2012-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for intel compiler toolchain (includes Intel compilers (icc, ifort), Parastation MPICH, +Intel Math Kernel Library (MKL), and Intel FFTW wrappers). + +""" + +from easybuild.toolchains.ipsmpi import Ipsmpi +from easybuild.toolchains.fft.intelfftw import IntelFFTW +from easybuild.toolchains.linalg.intelmkl import IntelMKL + + +class IntelPara(Ipsmpi, IntelMKL, IntelFFTW): + """ + Compiler toolchain with Intel compilers (icc/ifort), Parastation MPICH, + Intel Math Kernel Library (MKL) and Intel FFTW wrappers. + """ + NAME = 'intel-para' + diff --git a/easybuild/toolchains/intel.py b/easybuild/toolchains/intel.py old mode 100755 new mode 100644 index 25a88b6100..02c2bd6618 --- a/easybuild/toolchains/intel.py +++ b/easybuild/toolchains/intel.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2013 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iomkl.py b/easybuild/toolchains/iomkl.py index 57f41d7e35..a13fb71cea 100644 --- a/easybuild/toolchains/iomkl.py +++ b/easybuild/toolchains/iomkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iompi.py b/easybuild/toolchains/iompi.py new file mode 100644 index 0000000000..b64cc8e759 --- /dev/null +++ b/easybuild/toolchains/iompi.py @@ -0,0 +1,40 @@ +## +# Copyright 2012-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for iompi compiler toolchain (includes Intel compilers (icc, ifort) and OpenMPI. + +@author: Stijn De Weirdt (Ghent University) +@author: Kenneth Hoste (Ghent University) +""" + +from easybuild.toolchains.compiler.inteliccifort import IntelIccIfort +from easybuild.toolchains.mpi.openmpi import OpenMPI + + +class Iompi(IntelIccIfort, OpenMPI): + """ + Compiler toolchain with Intel compilers (icc/ifort) and OpenMPI. + """ + NAME = 'iompi' diff --git a/easybuild/toolchains/ipsmpi.py b/easybuild/toolchains/ipsmpi.py new file mode 100644 index 0000000000..de8b8806a1 --- /dev/null +++ b/easybuild/toolchains/ipsmpi.py @@ -0,0 +1,39 @@ +## +# Copyright 2012-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for intel compiler toolchain (includes Intel compilers (icc, ifort), Parastation MPICH). + +""" + +from easybuild.toolchains.impich import Impich + + +class Ipsmpi(Impich): + """ + Compiler toolchain with Intel compilers (icc/ifort), Parastation MPICH. + """ + NAME = 'ipsmpi' + # Use Parastation naming + MPI_MODULE_NAME = ["psmpi"] diff --git a/easybuild/toolchains/iqacml.py b/easybuild/toolchains/iqacml.py index dc6505112a..9529746985 100644 --- a/easybuild/toolchains/iqacml.py +++ b/easybuild/toolchains/iqacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ismkl.py b/easybuild/toolchains/ismkl.py index de3c965a46..41afc4a4f9 100644 --- a/easybuild/toolchains/ismkl.py +++ b/easybuild/toolchains/ismkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/__init__.py b/easybuild/toolchains/linalg/__init__.py index 9fdd109ac1..26041ec702 100644 --- a/easybuild/toolchains/linalg/__init__.py +++ b/easybuild/toolchains/linalg/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/acml.py b/easybuild/toolchains/linalg/acml.py index 62c18c0ef7..fb7c5def68 100644 --- a/easybuild/toolchains/linalg/acml.py +++ b/easybuild/toolchains/linalg/acml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,6 +34,7 @@ from easybuild.toolchains.compiler.inteliccifort import TC_CONSTANT_INTELCOMP from easybuild.toolchains.compiler.gcc import TC_CONSTANT_GCC +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.linalg import LinAlg @@ -57,10 +58,16 @@ class Acml(LinAlg): TC_CONSTANT_GCC: ['gfortran64', 'gfortran64_mp'], } + def __init__(self, *args, **kwargs): + """Toolchain constructor.""" + class_constants = kwargs.setdefault('class_constants', []) + class_constants.extend(['BLAS_LIB', 'BLAS_LIB_MT']) + super(Acml, self).__init__(*args, **kwargs) + def _set_blas_variables(self): """Fix the map a bit""" if self.options.get('32bit', None): - self.log.raiseException("_set_blas_variables: 32bit ACML not (yet) supported") + raise EasyBuildError("_set_blas_variables: 32bit ACML not (yet) supported") try: for root in self.get_software_root(self.BLAS_MODULE_NAME): subdirs = self.ACML_SUBDIRS_MAP[self.COMPILER_FAMILY] @@ -69,8 +76,8 @@ def _set_blas_variables(self): incdirs = [os.path.join(x, 'include') for x in subdirs] self.variables.append_exists('CPPFLAGS', root, incdirs, append_all=True) except: - self.log.raiseException(("_set_blas_variables: ACML set LDFLAGS/CPPFLAGS unknown entry in ACML_SUBDIRS_MAP" - " with compiler family %s") % self.COMPILER_FAMILY) + raise EasyBuildError("_set_blas_variables: ACML set LDFLAGS/CPPFLAGS unknown entry in ACML_SUBDIRS_MAP " + "with compiler family %s", self.COMPILER_FAMILY) # version before 5.x still featured the acml_mv library ver = self.get_software_version(self.BLAS_MODULE_NAME)[0] diff --git a/easybuild/toolchains/linalg/atlas.py b/easybuild/toolchains/linalg/atlas.py index 33313b4ae6..f1320600ac 100644 --- a/easybuild/toolchains/linalg/atlas.py +++ b/easybuild/toolchains/linalg/atlas.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/blacs.py b/easybuild/toolchains/linalg/blacs.py index 9bdebdf2e8..e7e708e392 100644 --- a/easybuild/toolchains/linalg/blacs.py +++ b/easybuild/toolchains/linalg/blacs.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/flame.py b/easybuild/toolchains/linalg/flame.py index e48848649e..e276e67327 100644 --- a/easybuild/toolchains/linalg/flame.py +++ b/easybuild/toolchains/linalg/flame.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/gotoblas.py b/easybuild/toolchains/linalg/gotoblas.py index 48719dcf73..c196d2fe5b 100644 --- a/easybuild/toolchains/linalg/gotoblas.py +++ b/easybuild/toolchains/linalg/gotoblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py index 415921f2e5..81ae047c63 100644 --- a/easybuild/toolchains/linalg/intelmkl.py +++ b/easybuild/toolchains/linalg/intelmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,11 +28,16 @@ @author: Stijn De Weirdt (Ghent University) @author: Kenneth Hoste (Ghent University) """ - from distutils.version import LooseVersion from easybuild.toolchains.compiler.inteliccifort import TC_CONSTANT_INTELCOMP from easybuild.toolchains.compiler.gcc import TC_CONSTANT_GCC +from easybuild.toolchains.mpi.intelmpi import TC_CONSTANT_INTELMPI +from easybuild.toolchains.mpi.mpich import TC_CONSTANT_MPICH +from easybuild.toolchains.mpi.mpich2 import TC_CONSTANT_MPICH2 +from easybuild.toolchains.mpi.mvapich2 import TC_CONSTANT_MVAPICH2 +from easybuild.toolchains.mpi.openmpi import TC_CONSTANT_OPENMPI +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.linalg import LinAlg @@ -63,11 +68,17 @@ class IntelMKL(LinAlg): SCALAPACK_MODULE_NAME = ['imkl'] SCALAPACK_LIB = ["mkl_scalapack%(lp64_sc)s"] SCALAPACK_LIB_MT = ["mkl_scalapack%(lp64_sc)s"] - SCALAPACK_LIB_MAP = {"lp64_sc":"_lp64"} + SCALAPACK_LIB_MAP = {'lp64_sc': '_lp64'} SCALAPACK_REQUIRES = ['LIBBLACS', 'LIBBLAS'] SCALAPACK_LIB_GROUP = True SCALAPACK_LIB_STATIC = True + def __init__(self, *args, **kwargs): + """Toolchain constructor.""" + class_constants = kwargs.setdefault('class_constants', []) + class_constants.extend(['BLAS_LIB_MAP', 'SCALAPACK_LIB', 'SCALAPACK_LIB_MT', 'SCALAPACK_LIB_MAP']) + super(IntelMKL, self).__init__(*args, **kwargs) + def _set_blas_variables(self): """Fix the map a bit""" interfacemap = { @@ -79,8 +90,8 @@ def _set_blas_variables(self): "interface": interfacemap[self.COMPILER_FAMILY], }) except: - self.log.raiseException(("_set_blas_variables: interface unsupported combination" - " with MPI family %s") % self.COMPILER_FAMILY) + raise EasyBuildError("_set_blas_variables: interface unsupported combination with MPI family %s", + self.COMPILER_FAMILY) interfacemap_mt = { TC_CONSTANT_INTELCOMP: 'intel', @@ -89,8 +100,8 @@ def _set_blas_variables(self): try: self.BLAS_LIB_MAP.update({"interface_mt":interfacemap_mt[self.COMPILER_FAMILY]}) except: - self.log.raiseException(("_set_blas_variables: interface_mt unsupported combination " - "with compiler family %s") % self.COMPILER_FAMILY) + raise EasyBuildError("_set_blas_variables: interface_mt unsupported combination with compiler family %s", + self.COMPILER_FAMILY) if self.options.get('32bit', None): @@ -112,8 +123,8 @@ def _set_blas_variables(self): self.BLAS_INCLUDE_DIR = ['include'] else: if self.options.get('32bit', None): - self.log.raiseException(("_set_blas_variables: 32-bit libraries not supported yet " - "for IMKL v%s (> v10.3)") % found_version) + raise EasyBuildError("_set_blas_variables: 32-bit libraries not supported yet for IMKL v%s (> v10.3)", + found_version) else: self.BLAS_LIB_DIR = ['mkl/lib/intel64', 'compiler/lib/intel64' ] @@ -123,16 +134,20 @@ def _set_blas_variables(self): def _set_blacs_variables(self): mpimap = { - "OpenMPI": '_openmpi', - "IntelMPI": '_intelmpi', - "MVAPICH2": '_intelmpi', - "MPICH2":'', + TC_CONSTANT_OPENMPI: '_openmpi', + TC_CONSTANT_INTELMPI: '_intelmpi', + TC_CONSTANT_MVAPICH2: '_intelmpi', + # use intelmpi MKL blacs library for both MPICH v2 and v3 + # cfr. https://software.intel.com/en-us/articles/intel-mkl-link-line-advisor + # note: MKL link advisor uses 'MPICH' for MPICH v1 + TC_CONSTANT_MPICH2: '_intelmpi', + TC_CONSTANT_MPICH: '_intelmpi', } try: self.BLACS_LIB_MAP.update({'mpi': mpimap[self.MPI_FAMILY]}) except: - self.log.raiseException(("_set_blacs_variables: mpi unsupported combination with" - " MPI family %s") % self.MPI_FAMILY) + raise EasyBuildError("_set_blacs_variables: mpi unsupported combination with MPI family %s", + self.MPI_FAMILY) self.BLACS_LIB_DIR = self.BLAS_LIB_DIR self.BLACS_INCLUDE_DIR = self.BLAS_INCLUDE_DIR @@ -153,5 +168,7 @@ def _set_scalapack_variables(self): # ilp64/i8 self.SCALAPACK_LIB_MAP.update({"lp64_sc":'_ilp64'}) - super(IntelMKL, self)._set_scalapack_variables() + self.SCALAPACK_LIB_DIR = self.BLAS_LIB_DIR + self.SCALAPACK_INCLUDE_DIR = self.BLAS_INCLUDE_DIR + super(IntelMKL, self)._set_scalapack_variables() diff --git a/easybuild/toolchains/linalg/lapack.py b/easybuild/toolchains/linalg/lapack.py index d8933bb158..0a95b9f456 100644 --- a/easybuild/toolchains/linalg/lapack.py +++ b/easybuild/toolchains/linalg/lapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/libsci.py b/easybuild/toolchains/linalg/libsci.py new file mode 100644 index 0000000000..d332b8d3d0 --- /dev/null +++ b/easybuild/toolchains/linalg/libsci.py @@ -0,0 +1,90 @@ +## +# Copyright 2014-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for Cray's LibSci library, which provides BLAS/LAPACK support. +cfr. https://www.nersc.gov/users/software/programming-libraries/math-libraries/libsci/ + +@author: Petar Forai (IMP/IMBA, Austria) +@author: Kenneth Hoste (Ghent University) +""" +import os + +from easybuild.tools.toolchain.linalg import LinAlg + + +CRAY_LIBSCI_MODULE_NAME = 'cray-libsci' + + +class LibSci(LinAlg): + """Support for Cray's LibSci library, which provides BLAS/LAPACK support.""" + # BLAS/LAPACK support + # via cray-libsci module, which gets loaded via the PrgEnv module + # see https://www.nersc.gov/users/software/programming-libraries/math-libraries/libsci/ + BLAS_MODULE_NAME = [CRAY_LIBSCI_MODULE_NAME] + + # no need to specify libraries, compiler driver takes care of linking the right libraries + # FIXME: need to revisit this, on numpy we ended up with a serial BLAS through the wrapper. + BLAS_LIB = [] + BLAS_LIB_MT = [] + + LAPACK_MODULE_NAME = [CRAY_LIBSCI_MODULE_NAME] + LAPACK_IS_BLAS = True + + BLACS_MODULE_NAME = [] + SCALAPACK_MODULE_NAME = [] + + def _get_software_root(self, name): + """Get install prefix for specified software name; special treatment for Cray modules.""" + if name == 'cray-libsci': + # Cray-provided LibSci module + env_var = 'CRAY_LIBSCI_PREFIX_DIR' + root = os.getenv(env_var, None) + if root is None: + raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_var) + else: + self.log.debug("Obtained install prefix for %s via $%s: %s", name, env_var, root) + else: + root = super(LibSci, self)._get_software_root(name) + + return root + + def _set_blacs_variables(self): + """Skip setting BLACS related variables""" + pass + + def _set_scalapack_variables(self): + """Skip setting ScaLAPACK related variables""" + pass + + def definition(self): + """ + Filter BLAS module from toolchain definition. + The cray-libsci module is loaded indirectly (and versionless) via the PrgEnv module, + and thus is not a direct toolchain component. + """ + tc_def = super(LibSci, self).definition() + tc_def['BLAS'] = [] + tc_def['LAPACK'] = [] + return tc_def diff --git a/easybuild/toolchains/linalg/openblas.py b/easybuild/toolchains/linalg/openblas.py index 798c63e7ac..1d1c6d4790 100644 --- a/easybuild/toolchains/linalg/openblas.py +++ b/easybuild/toolchains/linalg/openblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/scalapack.py b/easybuild/toolchains/linalg/scalapack.py index b3859346c2..eea44b1b45 100644 --- a/easybuild/toolchains/linalg/scalapack.py +++ b/easybuild/toolchains/linalg/scalapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/__init__.py b/easybuild/toolchains/mpi/__init__.py index 3f868dff91..e67a5cb97e 100644 --- a/easybuild/toolchains/mpi/__init__.py +++ b/easybuild/toolchains/mpi/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/craympich.py b/easybuild/toolchains/mpi/craympich.py new file mode 100644 index 0000000000..284281fdf7 --- /dev/null +++ b/easybuild/toolchains/mpi/craympich.py @@ -0,0 +1,76 @@ +## +# Copyright 2014-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +MPI support for the Cray Programming Environment (craype). + +@author: Petar Forai (IMP/IMBA, Austria) +@author: Kenneth Hoste (Ghent University) +""" +from easybuild.toolchains.compiler.craype import CrayPECompiler +from easybuild.toolchains.mpi.mpich import TC_CONSTANT_MPICH, TC_CONSTANT_MPI_TYPE_MPICH +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.toolchain.constants import COMPILER_VARIABLES, MPI_COMPILER_TEMPLATE, MPI_COMPILER_VARIABLES +from easybuild.tools.toolchain.constants import SEQ_COMPILER_TEMPLATE +from easybuild.tools.toolchain.mpi import Mpi + + +class CrayMPICH(Mpi): + """Generic support for using Cray compiler wrappers""" + # MPI support + # no separate module, Cray compiler drivers always provide MPI support + MPI_MODULE_NAME = [] + MPI_FAMILY = TC_CONSTANT_MPICH + MPI_TYPE = TC_CONSTANT_MPI_TYPE_MPICH + + MPI_COMPILER_MPICC = CrayPECompiler.COMPILER_CC + MPI_COMPILER_MPICXX = CrayPECompiler.COMPILER_CXX + MPI_COMPILER_MPIF77 = CrayPECompiler.COMPILER_F77 + MPI_COMPILER_MPIF90 = CrayPECompiler.COMPILER_F90 + MPI_COMPILER_MPIFC = CrayPECompiler.COMPILER_FC + + # no MPI wrappers, so no need to specify serial compiler + MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES]) + + def _set_mpi_compiler_variables(self): + """Set the MPI compiler variables""" + for var_tuple in COMPILER_VARIABLES: + c_var = var_tuple[0] # [1] is the description + var = MPI_COMPILER_TEMPLATE % {'c_var':c_var} + + value = getattr(self, 'MPI_COMPILER_%s' % var.upper(), None) + if value is None: + raise EasyBuildError("_set_mpi_compiler_variables: mpi compiler variable %s undefined", var) + self.variables.nappend_el(var, value) + + if self.options.get('usempi', None): + var_seq = SEQ_COMPILER_TEMPLATE % {'c_var': c_var} + seq_comp = self.variables[c_var] + self.log.debug('_set_mpi_compiler_variables: usempi set: defining %s as %s', var_seq, seq_comp) + self.variables[var_seq] = seq_comp + + if self.options.get('cciscxx', None): + self.log.debug("_set_mpi_compiler_variables: cciscxx set: switching MPICXX %s for MPICC value %s" % + (self.variables['MPICXX'], self.variables['MPICC'])) + self.variables['MPICXX'] = self.variables['MPICC'] diff --git a/easybuild/toolchains/mpi/intelmpi.py b/easybuild/toolchains/mpi/intelmpi.py index f2bb79a926..1137a9ddec 100644 --- a/easybuild/toolchains/mpi/intelmpi.py +++ b/easybuild/toolchains/mpi/intelmpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,6 +30,7 @@ """ from easybuild.toolchains.mpi.mpich2 import Mpich2 +from easybuild.tools.toolchain.constants import COMPILER_FLAGS, COMPILER_VARIABLES from easybuild.tools.toolchain.variables import CommandFlagList @@ -53,8 +54,8 @@ def _set_mpi_compiler_variables(self): """Add I_MPI_XXX variables to set.""" # this needs to be done first, otherwise e.g., CC is set to MPICC if the usempi toolchain option is enabled - for var in ["CC", "CXX", "F77", "F90"]: - self.variables.nappend("I_MPI_%s" % var, str(self.variables[var].get_first()), var_class=CommandFlagList) + for var, _ in COMPILER_VARIABLES: + self.variables.nappend('I_MPI_%s' % var, str(self.variables[var].get_first()), var_class=CommandFlagList) super(IntelMPI, self)._set_mpi_compiler_variables() @@ -66,5 +67,5 @@ def set_variables(self): # add -mt_mpi flag to ensure linking against thread-safe MPI library when OpenMP is enabled if self.options.get('openmp', None) and self.options.get('usempi', None): mt_mpi_option = ['mt_mpi'] - for flags_var in ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS']: + for flags_var, _ in COMPILER_FLAGS: self.variables.nappend(flags_var, mt_mpi_option) diff --git a/easybuild/toolchains/mpi/mpich.py b/easybuild/toolchains/mpi/mpich.py index 8c61e8fdf7..50e1a07229 100644 --- a/easybuild/toolchains/mpi/mpich.py +++ b/easybuild/toolchains/mpi/mpich.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,7 +30,9 @@ @author: Jens Timmerman (Ghent University) @author: Dmitri Gribenko (National Technical University of Ukraine "KPI") """ +from distutils.version import LooseVersion +from easybuild.tools.toolchain.constants import COMPILER_VARIABLES, MPI_COMPILER_VARIABLES from easybuild.tools.toolchain.mpi import Mpi from easybuild.tools.toolchain.variables import CommandFlagList @@ -40,25 +42,37 @@ class Mpich(Mpi): """MPICH MPI class""" - MPI_MODULE_NAME = ["MPICH"] + MPI_MODULE_NAME = ['MPICH'] MPI_FAMILY = TC_CONSTANT_MPICH MPI_TYPE = TC_CONSTANT_MPI_TYPE_MPICH MPI_LIBRARY_NAME = 'mpich' + # version-dependent, so defined at runtime + MPI_COMPILER_MPIF77 = None + MPI_COMPILER_MPIF90 = None + MPI_COMPILER_MPIFC = None + # clear MPI wrapper command options - MPI_SHARED_OPTION_MAP = { - '_opt_MPICC': '', - '_opt_MPICXX': '', - '_opt_MPIF77': '', - '_opt_MPIF90': '', - } + MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES]) def _set_mpi_compiler_variables(self): - """Set the MPICH_{CC, CXX, F77, F90} variables.""" + """Set the MPICH_{CC, CXX, F77, F90, FC} variables.""" + # determine MPI wrapper commands to use based on MPICH version + if self.MPI_COMPILER_MPIF77 is None and self.MPI_COMPILER_MPIF90 is None and self.MPI_COMPILER_MPIFC is None: + # mpif77/mpif90 for MPICH v3.1.0 and earlier, mpifort for MPICH v3.1.2 and newer + # see http://www.mpich.org/static/docs/v3.1/ vs http://www.mpich.org/static/docs/v3.1.2/ + if LooseVersion(self.get_software_version(self.MPI_MODULE_NAME)[0]) >= LooseVersion('3.1.2'): + self.MPI_COMPILER_MPIF77 = 'mpif77' + self.MPI_COMPILER_MPIF90 = 'mpifort' + self.MPI_COMPILER_MPIFC = 'mpifort' + else: + self.MPI_COMPILER_MPIF77 = 'mpif77' + self.MPI_COMPILER_MPIF90 = 'mpif90' + self.MPI_COMPILER_MPIFC = 'mpif90' # this needs to be done first, otherwise e.g., CC is set to MPICC if the usempi toolchain option is enabled - for var in ["CC", "CXX", "F77", "F90"]: - self.variables.nappend("MPICH_%s" % var, str(self.variables[var].get_first()), var_class=CommandFlagList) + for var, _ in COMPILER_VARIABLES: + self.variables.nappend('MPICH_%s' % var, str(self.variables[var].get_first()), var_class=CommandFlagList) super(Mpich, self)._set_mpi_compiler_variables() diff --git a/easybuild/toolchains/mpi/mpich2.py b/easybuild/toolchains/mpi/mpich2.py index 21dc78de60..84157e9eb9 100644 --- a/easybuild/toolchains/mpi/mpich2.py +++ b/easybuild/toolchains/mpi/mpich2.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,34 +30,24 @@ @author: Jens Timmerman (Ghent University) """ -from easybuild.tools.toolchain.mpi import Mpi -from easybuild.tools.toolchain.variables import CommandFlagList +from easybuild.toolchains.mpi.mpich import Mpich TC_CONSTANT_MPICH2 = "MPICH2" TC_CONSTANT_MPI_TYPE_MPICH = "MPI_TYPE_MPICH" -class Mpich2(Mpi): +class Mpich2(Mpich): """MPICH2 MPI class""" MPI_MODULE_NAME = ["MPICH2"] MPI_FAMILY = TC_CONSTANT_MPICH2 MPI_TYPE = TC_CONSTANT_MPI_TYPE_MPICH - MPI_LIBRARY_NAME = 'mpich' - - # clear MPI wrapper command options - MPI_SHARED_OPTION_MAP = { - '_opt_MPICC': '', - '_opt_MPICXX': '', - '_opt_MPIF77': '', - '_opt_MPIF90': '', - } - def _set_mpi_compiler_variables(self): - """Set the MPICH_{CC, CXX, F77, F90} variables.""" + """Set the MPICH_{CC, CXX, F77, F90, FC} variables.""" - # this needs to be done first, otherwise e.g., CC is set to MPICC if the usempi toolchain option is enabled - for var in ["CC", "CXX", "F77", "F90"]: - self.variables.nappend("MPICH_%s" % var, str(self.variables[var].get_first()), var_class=CommandFlagList) + # hardwire MPI wrapper commands (otherwise Mpich parent class sets them based on MPICH version) + self.MPI_COMPILER_MPIF77 = 'mpif77' + self.MPI_COMPILER_MPIF90 = 'mpif90' + self.MPI_COMPILER_MPIFC = 'mpif90' super(Mpich2, self)._set_mpi_compiler_variables() diff --git a/easybuild/toolchains/mpi/mvapich2.py b/easybuild/toolchains/mpi/mvapich2.py index b9cbf3561a..000be188a1 100644 --- a/easybuild/toolchains/mpi/mvapich2.py +++ b/easybuild/toolchains/mpi/mvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/openmpi.py b/easybuild/toolchains/mpi/openmpi.py index 12e1a9ade0..995e0b85e1 100644 --- a/easybuild/toolchains/mpi/openmpi.py +++ b/easybuild/toolchains/mpi/openmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,7 +28,9 @@ @author: Stijn De Weirdt (Ghent University) @author: Kenneth Hoste (Ghent University) """ +from distutils.version import LooseVersion +from easybuild.tools.toolchain.constants import COMPILER_VARIABLES, MPI_COMPILER_VARIABLES from easybuild.tools.toolchain.mpi import Mpi from easybuild.tools.toolchain.variables import CommandFlagList @@ -39,33 +41,37 @@ class OpenMPI(Mpi): """OpenMPI MPI class""" - MPI_MODULE_NAME = ["OpenMPI"] + MPI_MODULE_NAME = ['OpenMPI'] MPI_FAMILY = TC_CONSTANT_OPENMPI MPI_TYPE = TC_CONSTANT_MPI_TYPE_OPENMPI MPI_LIBRARY_NAME = 'mpi' - ## OpenMPI reads from CC etc env variables - MPI_SHARED_OPTION_MAP = { - '_opt_MPICC': '', - '_opt_MPICXX':'', - '_opt_MPICF77':'', - '_opt_MPICF90':'', - } + # version-dependent, so defined at runtime + MPI_COMPILER_MPIF77 = None + MPI_COMPILER_MPIF90 = None + MPI_COMPILER_MPIFC = None + + # OpenMPI reads from CC etc env variables + MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES]) MPI_LINK_INFO_OPTION = '-showme:link' def _set_mpi_compiler_variables(self): - """Add OMPI_XXX variables to set.""" + """Define MPI wrapper commands (depends on OpenMPI version) and add OMPI_* variables to set.""" + ompi_ver = self.get_software_version(self.MPI_MODULE_NAME)[0] + # version-dependent, see http://www.open-mpi.org/faq/?category=mpi-apps#override-wrappers-after-v1.0 + if LooseVersion(ompi_ver) >= LooseVersion('1.7'): + self.MPI_COMPILER_MPIF77 = 'mpifort' + self.MPI_COMPILER_MPIF90 = 'mpifort' + self.MPI_COMPILER_MPIFC = 'mpifort' + else: + self.MPI_COMPILER_MPIF77 = 'mpif77' + self.MPI_COMPILER_MPIF90 = 'mpif90' + self.MPI_COMPILER_MPIFC = 'mpif90' # this needs to be done first, otherwise e.g., CC is set to MPICC if the usempi toolchain option is enabled - for var in ["CC", "CXX", "F77", ("F90", "FC")]: - if isinstance(var, basestring): - source_var = var - target_var = var - else: - source_var = var[0] - target_var = var[1] - self.variables.nappend("OMPI_%s" % target_var, str(self.variables[source_var].get_first()), var_class=CommandFlagList) + for var, _ in COMPILER_VARIABLES: + self.variables.nappend('OMPI_%s' % var, str(self.variables[var].get_first()), var_class=CommandFlagList) super(OpenMPI, self)._set_mpi_compiler_variables() diff --git a/easybuild/toolchains/mpi/qlogicmpi.py b/easybuild/toolchains/mpi/qlogicmpi.py index 9118550664..2a7db39666 100644 --- a/easybuild/toolchains/mpi/qlogicmpi.py +++ b/easybuild/toolchains/mpi/qlogicmpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/__init__.py b/easybuild/tools/__init__.py index 15512e2f9c..6badf9fb34 100644 --- a/easybuild/tools/__init__.py +++ b/easybuild/tools/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/asyncprocess.py b/easybuild/tools/asyncprocess.py index 797478f70a..58e83de980 100644 --- a/easybuild/tools/asyncprocess.py +++ b/easybuild/tools/asyncprocess.py @@ -1,6 +1,6 @@ ## # Copyright 2005 Josiah Carlson -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # The Asynchronous Python Subprocess recipe was originally created by Josiah Carlson. # and released under the GPL v2 on March 14, 2012 diff --git a/easybuild/tools/build_details.py b/easybuild/tools/build_details.py index 44475ec9e8..4b6df10d39 100644 --- a/easybuild/tools/build_details.py +++ b/easybuild/tools/build_details.py @@ -1,4 +1,4 @@ -# Copyright 2014-2014 Ghent University +# Copyright 2014-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index e74931ff61..bd9f50853b 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -33,8 +33,10 @@ """ import os import sys +import tempfile from copy import copy from vsc.utils import fancylogger +from vsc.utils.exceptions import LoggedException from easybuild.tools.version import VERSION @@ -48,19 +50,37 @@ # allow some experimental experimental code EXPERIMENTAL = False +DEPRECATED_DOC_URL = 'http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html' -class EasyBuildError(Exception): + +class EasyBuildError(LoggedException): """ EasyBuildError is thrown when EasyBuild runs into something horribly wrong. """ - def __init__(self, msg): - Exception.__init__(self, msg) + LOC_INFO_TOP_PKG_NAMES = ['easybuild', 'vsc'] + LOC_INFO_LEVEL = 1 + + # use custom error logging method, to make sure EasyBuildError isn't being raised again to avoid infinite recursion + # only required because 'error' log method raises (should no longer be needed in EB v3.x) + LOGGING_METHOD_NAME = '_error_no_raise' + + def __init__(self, msg, *args): + """Constructor: initialise EasyBuildError instance.""" + if args: + msg = msg % args + LoggedException.__init__(self, msg) self.msg = msg def __str__(self): + """Return string representation of this EasyBuildError instance.""" return repr(self.msg) +def raise_easybuilderror(msg, *args): + """Raise EasyBuildError with given message, formatted by provided string arguments.""" + raise EasyBuildError(msg, *args) + + class EasyBuildLog(fancylogger.FancyLogger): """ The EasyBuild logger, with its own error and exception functions. @@ -80,6 +100,8 @@ def caller_info(self): filepath_dirs.remove(dirName) else: break + if not filepath_dirs: + filepath_dirs = ['?'] return "(at %s:%s in %s)" % (os.path.join(*filepath_dirs), line, function_name) def experimental(self, msg, *args, **kwargs): @@ -89,29 +111,54 @@ def experimental(self, msg, *args, **kwargs): self.warning(msg, *args, **kwargs) else: msg = 'Experimental functionality. Behaviour might change/be removed later (use --experimental option to enable). ' + msg - self.error(msg, *args) + raise EasyBuildError(msg, *args) def deprecated(self, msg, max_ver): """Print deprecation warning or raise an EasyBuildError, depending on max version allowed.""" + msg += "; see %s for more information" % DEPRECATED_DOC_URL fancylogger.FancyLogger.deprecated(self, msg, str(CURRENT_VERSION), max_ver, exception=EasyBuildError) + def nosupport(self, msg, ver): + """Print error message for no longer supported behaviour, and raise an EasyBuildError.""" + nosupport_msg = "NO LONGER SUPPORTED since v%s: %s; see %s for more information" + raise EasyBuildError(nosupport_msg, ver, msg, DEPRECATED_DOC_URL) + def error(self, msg, *args, **kwargs): """Print error message and raise an EasyBuildError.""" - newMsg = "EasyBuild crashed with an error %s: %s" % (self.caller_info(), msg) - fancylogger.FancyLogger.error(self, newMsg, *args, **kwargs) + ebmsg = "EasyBuild crashed with an error %s: " + msg + args = (self.caller_info(),) + args + + fancylogger.FancyLogger.error(self, ebmsg, *args, **kwargs) + if self.raiseError: - raise EasyBuildError(newMsg) + self.deprecated("Use 'raise EasyBuildError' rather than error() logging method that raises", '3.0') + raise EasyBuildError(ebmsg, *args) + + # FIXME: remove this when error() no longer raises EasyBuildError + def _error_no_raise(self, msg): + """Utility function to log an error with raising an exception.""" + + # make sure raising of error is disabled + orig_raise_error = self.raiseError + self.raiseError = False + + fancylogger.FancyLogger.error(self, msg) + + # reinstate previous raiseError setting + self.raiseError = orig_raise_error def exception(self, msg, *args): """Print exception message and raise EasyBuildError.""" # don't raise the exception from within error - newMsg = "EasyBuild encountered an exception %s: %s" % (self.caller_info(), msg) + ebmsg = "EasyBuild encountered an exception %s: " + msg + args = (self.caller_info(),) + args self.raiseError = False - fancylogger.FancyLogger.exception(self, newMsg, *args) + fancylogger.FancyLogger.exception(self, ebmsg, *args) self.raiseError = True - raise EasyBuildError(newMsg) + self.deprecated("Use 'raise EasyBuildError' rather than exception() logging method that raises", '3.0') + raise EasyBuildError(ebmsg, *args) # set format for logger @@ -132,15 +179,37 @@ def exception(self, msg, *args): _init_easybuildlog = fancylogger.getLogger(fname=False) +def init_logging(logfile, logtostdout=False, testing=False): + """Initialize logging.""" + if logtostdout: + fancylogger.logToScreen(enable=True, stdout=True) + else: + if logfile is None: + # mkstemp returns (fd,filename), fd is from os.open, not regular open! + fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-') + os.close(fd) + + fancylogger.logToFile(logfile) + print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing) + + log = fancylogger.getLogger(fname=False) + + return log, logfile + + +def stop_logging(logfile, logtostdout=False): + """Stop logging.""" + if logtostdout: + fancylogger.logToScreen(enable=False, stdout=True) + if logfile is not None: + fancylogger.logToFile(logfile, enable=False) + + def get_log(name=None): """ - Generate logger object + (NO LONGER SUPPORTED!) Generate logger object """ - # fname is always get_log, useless - log = fancylogger.getLogger(name, fname=False) - log.info("Logger started for %s." % name) - log.deprecated("get_log", "2.0") - return log + log.nosupport("Use of get_log function", '2.0') def print_msg(msg, log=None, silent=False, prefix=True): @@ -155,24 +224,23 @@ def print_msg(msg, log=None, silent=False, prefix=True): else: print msg + def print_error(message, log=None, exitCode=1, opt_parser=None, exit_on_error=True, silent=False): """ Print error message and exit EasyBuild """ if exit_on_error: if not silent: - print_msg("ERROR: %s\n" % message) if opt_parser: opt_parser.print_shorthelp() - print_msg("ERROR: %s\n" % message) + sys.stderr.write("ERROR: %s\n" % message) sys.exit(exitCode) elif log is not None: - log.error(message) + raise EasyBuildError(message) + def print_warning(message, silent=False): """ Print warning message. """ print_msg("WARNING: %s\n" % message, silent=silent) - - diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 50e0f5ac14..309268f618 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -40,69 +40,146 @@ import tempfile import time from vsc.utils import fancylogger -from vsc.utils.missing import nub, FrozenDictKnownKeys +from vsc.utils.missing import FrozenDictKnownKeys from vsc.utils.patterns import Singleton -import easybuild.tools.build_log # this import is required to obtain a correct (EasyBuild) logger! import easybuild.tools.environment as env -from easybuild.tools.environment import read_environment as _read_environment +from easybuild.tools import run +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.module_naming_scheme import GENERAL_CLASS from easybuild.tools.run import run_cmd _log = fancylogger.getLogger('config', fname=False) -# class constant to prepare migration to generaloption as only way of configuration (maybe for v2.X) -SUPPORT_OLDSTYLE = True - -DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log") +PKG_TOOL_FPM = 'fpm' +PKG_TYPE_RPM = 'rpm' +DEFAULT_JOB_BACKEND = 'PbsPython' +DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log") +DEFAULT_MNS = 'EasyBuildMNS' +DEFAULT_MODULE_SYNTAX = 'Tcl' +DEFAULT_MODULES_TOOL = 'EnvironmentModulesC' DEFAULT_PATH_SUBDIRS = { 'buildpath': 'build', 'installpath': '', + 'packagepath': 'packages', 'repositorypath': 'ebfiles_repo', 'sourcepath': 'sources', 'subdir_modules': 'modules', 'subdir_software': 'software', } - - -DEFAULT_BUILD_OPTIONS = { - 'aggregate_regtest': None, - 'allow_modules_tool_mismatch': False, - 'check_osdeps': True, - 'filter_deps': None, - 'cleanup_builddir': True, - 'command_line': None, - 'debug': False, - 'dry_run': False, - 'easyblock': None, - 'experimental': False, - 'force': False, - 'github_user': None, - 'group': None, - 'ignore_dirs': None, - 'modules_footer': None, - 'only_blocks': None, - 'optarch': None, - 'recursive_mod_unload': False, - 'regtest_output_dir': None, - 'retain_all_deps': False, - 'robot_path': None, - 'sequential': False, - 'set_gid_bit': False, - 'silent': False, - 'skip': None, - 'skip_test_cases': False, - 'sticky_bit': False, - 'stop': None, - 'suffix_modules_path': None, - 'test_report_env_filter': None, - 'umask': None, - 'valid_module_classes': None, - 'valid_stops': None, - 'validate': True, +DEFAULT_PKG_RELEASE = '1' +DEFAULT_PKG_TOOL = PKG_TOOL_FPM +DEFAULT_PKG_TYPE = PKG_TYPE_RPM +DEFAULT_PNS = 'EasyBuildPNS' +DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild") +DEFAULT_REPOSITORY = 'FileRepository' +DEFAULT_STRICT = run.WARN + + +# utility function for obtaining default paths +def mk_full_default_path(name, prefix=DEFAULT_PREFIX): + """Create full path, avoid '/' at the end.""" + args = [prefix] + path = DEFAULT_PATH_SUBDIRS[name] + if path: + args.append(path) + return os.path.join(*args) + +# build options that have a perfectly matching command line option, listed by default value +BUILD_OPTIONS_CMDLINE = { + None: [ + 'aggregate_regtest', + 'download_timeout', + 'dump_test_report', + 'easyblock', + 'external_modules_metadata', + 'filter_deps', + 'hide_deps', + 'from_pr', + 'github_user', + 'group', + 'ignore_dirs', + 'job_backend_config', + 'job_cores', + 'job_max_walltime', + 'job_output_dir', + 'job_polling_interval', + 'job_target_resource', + 'modules_footer', + 'only_blocks', + 'optarch', + 'parallel', + 'regtest_output_dir', + 'skip', + 'stop', + 'test_report_env_filter', + 'testoutput', + 'umask', + ], + False: [ + 'allow_modules_tool_mismatch', + 'debug', + 'dump_autopep8', + 'experimental', + 'force', + 'group_writable_installdir', + 'hidden', + 'module_only', + 'package', + 'read_only_installdir', + 'robot', + 'sequential', + 'set_gid_bit', + 'skip_test_cases', + 'sticky_bit', + 'upload_test_report', + 'update_modules_tool_cache', + ], + True: [ + 'cleanup_builddir', + 'cleanup_tmpdir', + ], + DEFAULT_STRICT: [ + 'strict', + ], + DEFAULT_PKG_RELEASE: [ + 'package_release', + ], + DEFAULT_PKG_TOOL: [ + 'package_tool', + ], + DEFAULT_PKG_TYPE: [ + 'package_type', + ], + GENERAL_CLASS: [ + 'suffix_modules_path', + ], +} +# build option that do not have a perfectly matching command line option +BUILD_OPTIONS_OTHER = { + None: [ + 'build_specs', + 'command_line', + 'pr_path', + 'robot_path', + 'valid_module_classes', + 'valid_stops', + ], + False: [ + 'dry_run', + 'recursive_mod_unload', + 'retain_all_deps', + 'silent', + 'try_to_generate', + ], + True: [ + 'check_osdeps', + 'validate', + ], } @@ -134,81 +211,45 @@ ] -OLDSTYLE_ENVIRONMENT_VARIABLES = { - 'build_path': 'EASYBUILDBUILDPATH', - 'config_file': 'EASYBUILDCONFIG', - 'install_path': 'EASYBUILDINSTALLPATH', - 'log_format': 'EASYBUILDLOGFORMAT', - 'log_dir': 'EASYBUILDLOGDIR', - 'source_path': 'EASYBUILDSOURCEPATH', - 'test_output_path': 'EASYBUILDTESTOUTPUT', -} - - -OLDSTYLE_NEWSTYLE_MAP = { - 'build_path': 'buildpath', - 'install_path': 'installpath', - 'log_dir': 'tmp_logdir', - 'config_file': 'config', - 'source_path': 'sourcepath', - 'log_format': 'logfile_format', - 'test_output_path': 'testoutput', - 'module_classes': 'moduleclasses', - 'repository_path': 'repositorypath', - 'modules_install_suffix': 'subdir_modules', - 'software_install_suffix': 'subdir_software', -} - - -def map_to_newstyle(adict): - """Map a dictionary with oldstyle keys to the new style.""" - res = {} - for key, val in adict.items(): - if key in OLDSTYLE_NEWSTYLE_MAP: - newkey = OLDSTYLE_NEWSTYLE_MAP.get(key) - _log.deprecated("oldstyle key %s usage found, replacing with newkey %s" % (key, newkey), "2.0") - key = newkey - res[key] = val - return res - - class ConfigurationVariables(FrozenDictKnownKeys): """This is a dict that supports legacy config names transparently.""" # singleton metaclass: only one instance is created __metaclass__ = Singleton + # list of known/required keys REQUIRED = [ - 'config', - 'prefix', 'buildpath', + 'config', 'installpath', - 'sourcepath', - 'repository', - 'repositorypath', + 'installpath_modules', + 'installpath_software', + 'job_backend', 'logfile_format', - 'tmp_logdir', 'moduleclasses', + 'module_naming_scheme', + 'module_syntax', + 'modules_tool', + 'packagepath', + 'package_naming_scheme', + 'prefix', + 'repository', + 'repositorypath', + 'sourcepath', 'subdir_modules', 'subdir_software', - 'modules_tool', - 'module_naming_scheme', + 'tmp_logdir', ] + KNOWN_KEYS = REQUIRED # KNOWN_KEYS must be defined for FrozenDictKnownKeys functionality - KNOWN_KEYS = nub(OLDSTYLE_NEWSTYLE_MAP.values() + REQUIRED) - - def get_items_check_required(self, no_missing=True): + def get_items_check_required(self): """ - For all REQUIRED, check if exists and return all key,value pairs. + For all known/required keys, check if exists and return all key/value pairs. no_missing: boolean, when True, will throw error message for missing values """ - missing = [x for x in self.REQUIRED if not x in self] + missing = [x for x in self.KNOWN_KEYS if x not in self] if len(missing) > 0: - msg = 'Cannot determine value for configuration variables %s. Please specify it.' % missing - if no_missing: - self.log.error(msg) - else: - self.log.debug(msg) + raise EasyBuildError("Cannot determine value for configuration variables %s. Please specify it.", missing) return self.items() @@ -219,94 +260,7 @@ class BuildOptions(FrozenDictKnownKeys): # singleton metaclass: only one instance is created __metaclass__ = Singleton - KNOWN_KEYS = DEFAULT_BUILD_OPTIONS.keys() - - -def get_user_easybuild_dir(): - """Return the per-user easybuild dir (e.g. to store config files)""" - oldpath = os.path.join(os.path.expanduser('~'), ".easybuild") - xdg_config_home = os.environ.get("XDG_CONFIG_HOME", os.path.join(os.path.expanduser('~'), ".config")) - newpath = os.path.join(xdg_config_home, "easybuild") - - if os.path.isdir(newpath): - return newpath - else: - _log.deprecated("The user easybuild dir has moved from %s to %s." % (oldpath, newpath), "2.0") - return oldpath - - -def get_default_oldstyle_configfile(): - """Get the default location of the oldstyle config file to be set as default in the options""" - # TODO these _log.debug here can't be controlled/set with the generaloption - # - check environment variable EASYBUILDCONFIG - # - next, check for an EasyBuild config in $HOME/.easybuild/config.py - # - last, use default config file easybuild_config.py in main.py directory - config_env_var = OLDSTYLE_ENVIRONMENT_VARIABLES['config_file'] - home_config_file = os.path.join(get_user_easybuild_dir(), "config.py") - if os.getenv(config_env_var): - _log.debug("Environment variable %s, so using that as config file." % config_env_var) - config_file = os.getenv(config_env_var) - elif os.path.exists(home_config_file): - config_file = home_config_file - _log.debug("Found EasyBuild configuration file at %s." % config_file) - else: - # this should be easybuild.tools.config, the default config file is - # part of framework in easybuild (ie in tool/..) - appPath = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - config_file = os.path.join(appPath, "easybuild_config.py") - _log.debug("Falling back to default config: %s" % config_file) - - _log.deprecated("get_default_oldstyle_configfile oldstyle configfile %s used" % config_file, "2.0") - - return config_file - - -def get_default_oldstyle_configfile_defaults(prefix=None): - """ - Return a dict with the defaults from the shipped legacy easybuild_config.py and/or environment variables - prefix: string, when provided, it used as prefix for the other defaults (where applicable) - """ - if prefix is None: - prefix = os.path.join(os.path.expanduser('~'), ".local", "easybuild") - - def mk_full_path(name): - """Create full path, avoid '/' at the end.""" - args = [prefix] - path = DEFAULT_PATH_SUBDIRS[name] - if path: - args.append(path) - return os.path.join(*args) - - # keys are the options dest - defaults = { - 'config': get_default_oldstyle_configfile(), - 'prefix': prefix, - 'buildpath': mk_full_path('buildpath'), - 'installpath': mk_full_path('installpath'), - 'sourcepath': mk_full_path('sourcepath'), - 'repository': 'FileRepository', - 'repositorypath': {'FileRepository': [mk_full_path('repositorypath')]}, - 'logfile_format': DEFAULT_LOGFILE_FORMAT[:], # make a copy - 'tmp_logdir': tempfile.gettempdir(), - 'moduleclasses': [x[0] for x in DEFAULT_MODULECLASSES], - 'subdir_modules': DEFAULT_PATH_SUBDIRS['subdir_modules'], - 'subdir_software': DEFAULT_PATH_SUBDIRS['subdir_software'], - 'modules_tool': 'EnvironmentModulesC', - 'module_naming_scheme': 'EasyBuildMNS', - } - - # sanity check - if not defaults['repository'] in defaults['repositorypath']: - _log.error('Failed to get repository path default for default %s' % (defaults['repository'])) - - _log.deprecated("get_default_oldstyle_configfile_defaults", "2.0") - - return defaults - - -def get_default_configfiles(): - """Return a list of default configfiles for tools.options/generaloption""" - return [os.path.join(get_user_easybuild_dir(), "config.cfg")] + KNOWN_KEYS = [k for kss in [BUILD_OPTIONS_CMDLINE, BUILD_OPTIONS_OTHER] for ks in kss.values() for k in ks] def get_pretend_installpath(): @@ -319,36 +273,7 @@ def init(options, config_options_dict): Gather all variables and check if they're valid Variables are read in this order of preference: generaloption > legacy environment > legacy config file """ - tmpdict = {} - - if SUPPORT_OLDSTYLE: - - _log.deprecated('oldstyle init with modifications to support oldstyle options', '2.0') - tmpdict.update(oldstyle_init(options.config)) - - # add the DEFAULT_MODULECLASSES as default (behavior is now that this extends the default list) - tmpdict['moduleclasses'] = nub(list(tmpdict.get('moduleclasses', [])) + - [x[0] for x in DEFAULT_MODULECLASSES]) - - # make sure we have new-style keys - tmpdict = map_to_newstyle(tmpdict) - - # all defaults are now set in generaloption - # distinguish between default generaloption values and values actually passed by generaloption - for dest in config_options_dict.keys(): - if not options._action_taken.get(dest, False): - if dest == 'installpath' and options.pretend: - # the installpath has been set by pretend option in postprocess - continue - # remove the default options if they are set in variables - # this way, all defaults are set - if dest in tmpdict: - _log.debug("Oldstyle support: no action for dest %s." % dest) - del config_options_dict[dest] - - # update the variables with the generaloption values - _log.debug("Updating config variables with generaloption dict %s" % config_options_dict) - tmpdict.update(config_options_dict) + tmpdict = copy.deepcopy(config_options_dict) # make sure source path is a list sourcepath = tmpdict['sourcepath'] @@ -356,7 +281,7 @@ def init(options, config_options_dict): tmpdict['sourcepath'] = sourcepath.split(':') _log.debug("Converted source path ('%s') to a list of paths: %s" % (sourcepath, tmpdict['sourcepath'])) elif not isinstance(sourcepath, (tuple, list)): - _log.error("Value for sourcepath has invalid type (%s): %s" % (type(sourcepath), sourcepath)) + raise EasyBuildError("Value for sourcepath has invalid type (%s): %s", type(sourcepath), sourcepath) # initialize configuration variables (any future calls to ConfigurationVariables() will yield the same instance variables = ConfigurationVariables(tmpdict, ignore_unknown_keys=True) @@ -364,12 +289,46 @@ def init(options, config_options_dict): _log.debug("Config variables: %s" % variables) -def init_build_options(build_options=None): +def init_build_options(build_options=None, cmdline_options=None): """Initialize build options.""" - # seed in defaults to make sure all build options are defined, and that build_option() doesn't fail on valid keys - bo = copy.deepcopy(DEFAULT_BUILD_OPTIONS) + + active_build_options = {} + + if cmdline_options is not None: + # building a dependency graph implies force, so that all dependencies are retained + # and also skips validation of easyconfigs (e.g. checking os dependencies) + retain_all_deps = False + if cmdline_options.dep_graph: + _log.info("Enabling force to generate dependency graph.") + cmdline_options.force = True + retain_all_deps = True + + if cmdline_options.dep_graph or cmdline_options.dry_run or cmdline_options.dry_run_short: + _log.info("Ignoring OS dependencies for --dep-graph/--dry-run") + cmdline_options.ignore_osdeps = True + + cmdline_build_option_names = [k for ks in BUILD_OPTIONS_CMDLINE.values() for k in ks] + active_build_options.update(dict([(key, getattr(cmdline_options, key)) for key in cmdline_build_option_names])) + # other options which can be derived but have no perfectly matching cmdline option + active_build_options.update({ + 'check_osdeps': not cmdline_options.ignore_osdeps, + 'dry_run': cmdline_options.dry_run or cmdline_options.dry_run_short, + 'recursive_mod_unload': cmdline_options.recursive_module_unload, + 'retain_all_deps': retain_all_deps, + 'validate': not cmdline_options.force, + 'valid_module_classes': module_classes(), + }) + if build_options is not None: - bo.update(build_options) + active_build_options.update(build_options) + + # seed in defaults to make sure all build options are defined, and that build_option() doesn't fail on valid keys + bo = {} + for build_options_by_default in [BUILD_OPTIONS_CMDLINE, BUILD_OPTIONS_OTHER]: + for default in build_options_by_default: + bo.update(dict([(opt, default) for opt in build_options_by_default[default]])) + bo.update(active_build_options) + # BuildOptions is a singleton, so any future calls to BuildOptions will yield the same instance return BuildOptions(bo) @@ -394,11 +353,8 @@ def source_paths(): def source_path(): - """ - Return the source path (deprecated) - """ - _log.deprecated("Use of source_path() is deprecated, use source_paths() instead.", '2.0') - return source_paths() + """NO LONGER SUPPORTED: use source_paths instead""" + _log.nosupport("source_path() is replaced by source_paths()", '2.0') def install_path(typ=None): @@ -407,26 +363,27 @@ def install_path(typ=None): - subdir 'software' for actual installation (default) - subdir 'modules' for environment modules (typ='mod') """ - variables = ConfigurationVariables() - if typ is None: typ = 'software' - if typ == 'mod': + elif typ == 'mod': typ = 'modules' - key = "subdir_%s" % typ - if key in variables: - suffix = variables[key] + known_types = ['modules', 'software'] + if typ not in known_types: + raise EasyBuildError("Unknown type specified in install_path(): %s (known: %s)", typ, ', '.join(known_types)) + + variables = ConfigurationVariables() + + key = 'installpath_%s' % typ + res = variables[key] + if res is None: + key = 'subdir_%s' % typ + res = os.path.join(variables['installpath'], variables[key]) + _log.debug("%s install path as specified by 'installpath' and '%s': %s", typ, key, res) else: - # TODO remove default setting. it should have been set through options - _log.deprecated('%s not set in config, returning default' % key, "2.0") - defaults = get_default_oldstyle_configfile_defaults() - try: - suffix = defaults[key] - except: - _log.error('install_path trying to get unknown suffix %s' % key) + _log.debug("%s install path as specified by '%s': %s", typ, key, res) - return os.path.join(variables['installpath'], suffix) + return res def get_repository(): @@ -443,6 +400,20 @@ def get_repositorypath(): return ConfigurationVariables()['repositorypath'] +def get_package_naming_scheme(): + """ + Return the package naming scheme + """ + return ConfigurationVariables()['package_naming_scheme'] + + +def package_path(): + """ + Return the path where built packages are copied to + """ + return ConfigurationVariables()['packagepath'] + + def get_modules_tool(): """ Return modules tool (EnvironmentModulesC, Lmod, ...) @@ -453,24 +424,30 @@ def get_modules_tool(): def get_module_naming_scheme(): """ - Return module naming scheme (EasyBuild, ...) + Return module naming scheme (EasyBuildMNS, HierarchicalMNS, ...) """ return ConfigurationVariables()['module_naming_scheme'] +def get_job_backend(): + """ + Return job execution backend (PBS, GC3Pie, ...) + """ + # 'job_backend' key will only be present after EasyBuild config is initialized + return ConfigurationVariables().get('job_backend', None) + + +def get_module_syntax(): + """ + Return module syntax (Lua, Tcl) + """ + return ConfigurationVariables()['module_syntax'] + + def log_file_format(return_directory=False): """Return the format for the logfile or the directory""" idx = int(not return_directory) - - variables = ConfigurationVariables() - if 'logfile_format' in variables: - res = variables['logfile_format'][idx] - else: - # TODO remove default setting. it should have been set through options - _log.deprecated('logfile_format not set in config, returning default', "2.0") - defaults = get_default_oldstyle_configfile_defaults() - res = defaults['logfile_format'][idx] - return res + return ConfigurationVariables()['logfile_format'][idx] def log_format(): @@ -490,16 +467,14 @@ def log_path(): def get_build_log_path(): """ - return temporary log directory + Return (temporary) directory for build log """ variables = ConfigurationVariables() - if 'tmp_logdir' in variables: - return variables['tmp_logdir'] + if variables['tmp_logdir'] is not None: + res = variables['tmp_logdir'] else: - # TODO remove default setting. it should have been set through options - _log.deprecated('tmp_logdir not set in config, returning default', "2.0") - defaults = get_default_oldstyle_configfile_defaults() - return defaults['tmp_logdir'] + res = tempfile.gettempdir() + return res def get_log_filename(name, version, add_salt=False): @@ -532,109 +507,30 @@ def get_log_filename(name, version, add_salt=False): return filepath -def read_only_installdir(): - """ - Return whether installation dir should be fully read-only after installation. - """ - # FIXME (see issue #123): add a config option to set this, should be True by default (?) - # this also needs to be checked when --force is used; - # install dir will have to (temporarily) be made writeable again for owner in that case - return False - - def module_classes(): """ Return list of module classes specified in config file. """ - variables = ConfigurationVariables() - if 'moduleclasses' in variables: - return variables['moduleclasses'] - else: - # TODO remove default setting. it should have been set through options - _log.deprecated('moduleclasses not set in config, returning default', "2.0") - defaults = get_default_oldstyle_configfile_defaults() - return defaults['moduleclasses'] + return ConfigurationVariables()['moduleclasses'] def read_environment(env_vars, strict=False): - """Depreacted location for read_environment, use easybuild.tools.environment""" - _log.deprecated("Deprecated location for read_environment, use easybuild.tools.environment", '2.0') - return _read_environment(env_vars, strict) - + """NO LONGER SUPPORTED: use read_environment from easybuild.tools.environment instead""" + _log.nosupport("read_environment has moved to easybuild.tools.environment", '2.0') -def oldstyle_init(filename, **kwargs): - """ - Gather all variables and check if they're valid - Variables are read in this order of preference: CLI option > environment > config file - """ - res = {} - _log.deprecated("oldstyle_init filename %s kwargs %s" % (filename, kwargs), "2.0") - _log.debug('variables before oldstyle_init %s' % res) - res.update(oldstyle_read_configuration(filename)) # config file - _log.debug('variables after oldstyle_init read_configuration (%s) %s' % (filename, res)) - res.update(oldstyle_read_environment()) # environment - _log.debug('variables after oldstyle_init read_environment %s' % res) - if kwargs: - res.update(kwargs) # CLI options - _log.debug('variables after oldstyle_init kwargs (passed %s) %s' % (kwargs, res)) - - return res - - -def oldstyle_read_configuration(filename): - """ - Read variables from the config file - """ - _log.deprecated("oldstyle_read_configuration filename %s" % filename, "2.0") - - # import avail_repositories here to avoid cyclic dependencies - # this block of code is going to be removed in EB v2.0 - from easybuild.tools.repository.repository import avail_repositories - file_variables = avail_repositories(check_useable=False) - try: - execfile(filename, {}, file_variables) - except (IOError, SyntaxError), err: - _log.exception("Failed to read config file %s %s" % (filename, err)) - - return file_variables - - -def oldstyle_read_environment(env_vars=None, strict=False): - """ - Read variables from the environment - - strict=True enforces that all possible environment variables are found - """ - _log.deprecated(('Adapt code to use read_environment from easybuild.tools.utilities ' - 'and do not use oldstyle environment variables'), '2.0') - if env_vars is None: - env_vars = OLDSTYLE_ENVIRONMENT_VARIABLES - result = {} - for key in env_vars.keys(): - env_var = env_vars[key] - if env_var in os.environ: - result[key] = os.environ[env_var] - _log.deprecated("Found oldstyle environment variable %s for %s: %s" % (env_var, key, result[key]), "2.0") - elif strict: - _log.error("Can't determine value for %s. Environment variable %s is missing" % (key, env_var)) - else: - _log.debug("Old style env var %s not defined." % env_var) - - return result - - -def set_tmpdir(tmpdir=None): +def set_tmpdir(tmpdir=None, raise_error=False): """Set temporary directory to be used by tempfile and others.""" try: if tmpdir is not None: if not os.path.exists(tmpdir): os.makedirs(tmpdir) - current_tmpdir = tempfile.mkdtemp(prefix='easybuild-', dir=tmpdir) + current_tmpdir = tempfile.mkdtemp(prefix='eb-', dir=tmpdir) else: # use tempfile default parent dir - current_tmpdir = tempfile.mkdtemp(prefix='easybuild-') + current_tmpdir = tempfile.mkdtemp(prefix='eb-') except OSError, err: - _log.error("Failed to create temporary directory (tmpdir: %s): %s" % (tmpdir, err)) + raise EasyBuildError("Failed to create temporary directory (tmpdir: %s): %s", tmpdir, err) _log.info("Temporary directory used in this EasyBuild run: %s" % current_tmpdir) @@ -652,12 +548,15 @@ def set_tmpdir(tmpdir=None): if not run_cmd(tmptest_file, simple=True, log_ok=False, regexp=False): msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir() msg += "This can cause problems in the build process, consider using --tmpdir." - _log.warning(msg) + if raise_error: + raise EasyBuildError(msg) + else: + _log.warning(msg) else: _log.debug("Temporary directory %s allows to execute files, good!" % tempfile.gettempdir()) os.remove(tmptest_file) except OSError, err: - _log.error("Failed to test whether temporary directory allows to execute files: %s" % err) + raise EasyBuildError("Failed to test whether temporary directory allows to execute files: %s", err) return current_tmpdir diff --git a/easybuild/tools/convert.py b/easybuild/tools/convert.py index b9b845bfe3..b4a2304597 100644 --- a/easybuild/tools/convert.py +++ b/easybuild/tools/convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2014 Ghent University +# Copyright 2014-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,6 +34,9 @@ from vsc.utils.missing import get_subclasses, nub from vsc.utils.wrapper import Wrapper +from easybuild.tools.build_log import EasyBuildError + + _log = fancylogger.getLogger('tools.convert', fname=False) @@ -56,7 +59,7 @@ def __init__(self, obj): if isinstance(obj, basestring): self.data = self._from_string(obj) else: - self.log.error('unsupported type %s for %s: %s' % (type(obj), self.__class__.__name__, obj)) + raise EasyBuildError("unsupported type %s for %s: %s", type(obj), self.__class__.__name__, obj) super(Convert, self).__init__(self.data) def _split_string(self, txt, sep=None, max=0): @@ -66,7 +69,7 @@ def _split_string(self, txt, sep=None, max=0): """ if sep is None: if self.SEPARATOR is None: - self.log.error('No SEPARATOR set, also no separator passed') + raise EasyBuildError("No SEPARATOR set, also no separator passed") else: sep = self.SEPARATOR return [x.strip() for x in re.split(r'' + sep, txt, maxsplit=max)] @@ -221,4 +224,4 @@ def get_convert_class(class_name): if len(res) == 1: return res[0] else: - _log.error('More then one Convert subclass found for name %s: %s' % (class_name, res)) + raise EasyBuildError("More than one Convert subclass found for name %s: %s", class_name, res) diff --git a/easybuild/tools/deprecated/__init__.py b/easybuild/tools/deprecated/__init__.py new file mode 100644 index 0000000000..379c01f066 --- /dev/null +++ b/easybuild/tools/deprecated/__init__.py @@ -0,0 +1,38 @@ +## +# Copyright 2009-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +This declares the namespace for the tools.deprecated submodule of EasyBuild, +which provides deprecated functionality. + +@author: Stijn De Weirdt (Ghent University) +@author: Dries Verdegem (Ghent University) +@author: Kenneth Hoste (Ghent University) +@author: Pieter De Baets (Ghent University) +@author: Jens Timmerman (Ghent University) +""" +from pkgutil import extend_path + +# we're not the only ones in this namespace +__path__ = extend_path(__path__, __name__) #@ReservedAssignment diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py new file mode 100644 index 0000000000..dead250516 --- /dev/null +++ b/easybuild/tools/docs.py @@ -0,0 +1,274 @@ +# # +# Copyright 2009-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Documentation-related functionality + +@author: Stijn De Weirdt (Ghent University) +@author: Dries Verdegem (Ghent University) +@author: Kenneth Hoste (Ghent University) +@author: Pieter De Baets (Ghent University) +@author: Jens Timmerman (Ghent University) +@author: Toon Willems (Ghent University) +@author: Ward Poelmans (Ghent University) +""" +import copy +import inspect +import os +from vsc.utils.docs import mk_rst_table + +from easybuild.framework.easyconfig.default import DEFAULT_CONFIG, HIDDEN, sorted_categories +from easybuild.framework.easyconfig.easyconfig import get_easyblock_class +from easybuild.tools.filetools import read_file +from easybuild.tools.ordereddict import OrderedDict +from easybuild.tools.utilities import import_available_modules, quote_str + + +FORMAT_RST = 'rst' +FORMAT_TXT = 'txt' + + +def det_col_width(entries, title): + """Determine column width based on column title and list of entries.""" + return max(map(len, entries + [title])) + + +def avail_easyconfig_params_rst(title, grouped_params): + """ + Compose overview of available easyconfig parameters, in RST format. + """ + # main title + lines = [ + title, + '=' * len(title), + '', + ] + + for grpname in grouped_params: + # group section title + lines.append("%s parameters" % grpname) + lines.extend(['-' * len(lines[-1]), '']) + + titles = ["**Parameter name**", "**Description**", "**Default value**"] + values = [ + ['``' + name + '``' for name in grouped_params[grpname].keys()], # parameter name + [x[0] for x in grouped_params[grpname].values()], # description + [str(quote_str(x[1])) for x in grouped_params[grpname].values()] # default value + ] + + lines.extend(mk_rst_table(titles, values)) + lines.append('') + + return '\n'.join(lines) + + +def avail_easyconfig_params_txt(title, grouped_params): + """ + Compose overview of available easyconfig parameters, in plain text format. + """ + # main title + lines = [ + '%s:' % title, + '', + ] + + for grpname in grouped_params: + # group section title + lines.append(grpname.upper()) + lines.append('-' * len(lines[-1])) + + # determine width of 'name' column, to left-align descriptions + nw = max(map(len, grouped_params[grpname].keys())) + + # line by parameter + for name, (descr, dflt) in sorted(grouped_params[grpname].items()): + lines.append("{0:<{nw}} {1:} [default: {2:}]".format(name, descr, str(quote_str(dflt)), nw=nw)) + lines.append('') + + return '\n'.join(lines) + + +def avail_easyconfig_params(easyblock, output_format): + """ + Compose overview of available easyconfig parameters, in specified format. + """ + params = copy.deepcopy(DEFAULT_CONFIG) + + # include list of extra parameters (if any) + extra_params = {} + app = get_easyblock_class(easyblock, default_fallback=False) + if app is not None: + extra_params = app.extra_options() + params.update(extra_params) + + # compose title + title = "Available easyconfig parameters" + if extra_params: + title += " (* indicates specific to the %s easyblock)" % app.__name__ + + # group parameters by category + grouped_params = OrderedDict() + for category in sorted_categories(): + # exclude hidden parameters + if category[1].upper() in [HIDDEN]: + continue + + grpname = category[1] + grouped_params[grpname] = {} + for name, (dflt, descr, cat) in params.items(): + if cat == category: + if name in extra_params: + # mark easyblock-specific parameters + name = '%s*' % name + grouped_params[grpname].update({name: (descr, dflt)}) + + if not grouped_params[grpname]: + del grouped_params[grpname] + + # compose output, according to specified format (txt, rst, ...) + avail_easyconfig_params_functions = { + FORMAT_RST: avail_easyconfig_params_rst, + FORMAT_TXT: avail_easyconfig_params_txt, + } + return avail_easyconfig_params_functions[output_format](title, grouped_params) + + +def gen_easyblocks_overview_rst(package_name, path_to_examples, common_params={}, doc_functions=[]): + """ + Compose overview of all easyblocks in the given package in rst format + """ + modules = import_available_modules(package_name) + docs = [] + all_blocks = [] + + # get all blocks + for mod in modules: + for name,obj in inspect.getmembers(mod, inspect.isclass): + eb_class = getattr(mod, name) + # skip imported classes that are not easyblocks + if eb_class.__module__.startswith(package_name) and eb_class not in all_blocks: + all_blocks.append(eb_class) + + for eb_class in sorted(all_blocks, key=lambda c: c.__name__): + docs.append(gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc_functions, all_blocks)) + + title = 'Overview of generic easyblocks' + + heading = [ + '*(this page was generated automatically using* ``easybuild.tools.docs.gen_easyblocks_overview_rst()`` *)*', + '', + '=' * len(title), + title, + '=' * len(title), + '', + ] + + contents = [":ref:`" + b.__name__ + "`" for b in sorted(all_blocks, key=lambda b: b.__name__)] + toc = ' - '.join(contents) + heading.append(toc) + heading.append('') + + return heading + docs + + +def gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc_functions, all_blocks): + """ + Compose overview of one easyblock given class object of the easyblock in rst format + """ + classname = eb_class.__name__ + + lines = [ + '.. _' + classname + ':', + '', + '``' + classname + '``', + '=' * (len(classname)+4), + '', + ] + + bases = [] + for b in eb_class.__bases__: + base = ':ref:`' + b.__name__ +'`' if b in all_blocks else b.__name__ + bases.append(base) + + derived = '(derives from ' + ', '.join(bases) + ')' + lines.extend([derived, '']) + + # Description (docstring) + lines.extend([eb_class.__doc__.strip(), '']) + + # Add extra options, if any + if eb_class.extra_options(): + extra_parameters = 'Extra easyconfig parameters specific to ``' + classname + '`` easyblock' + lines.extend([extra_parameters, '-' * len(extra_parameters), '']) + ex_opt = eb_class.extra_options() + + titles = ['easyconfig parameter', 'description', 'default value'] + values = [ + ['``' + key + '``' for key in ex_opt], # parameter name + [val[1] for val in ex_opt.values()], # description + ['``' + str(quote_str(val[0])) + '``' for val in ex_opt.values()] # default value + ] + + lines.extend(mk_rst_table(titles, values)) + + # Add commonly used parameters + if classname in common_params: + commonly_used = 'Commonly used easyconfig parameters with ``' + classname + '`` easyblock' + lines.extend([commonly_used, '-' * len(commonly_used)]) + + titles = ['easyconfig parameter', 'description'] + values = [ + [opt for opt in common_params[classname]], + [DEFAULT_CONFIG[opt][1] for opt in common_params[classname]], + ] + + lines.extend(mk_rst_table(titles, values)) + + lines.append('') + + # Add docstring for custom steps + custom = [] + inh = '' + f = None + for func in doc_functions: + if func in eb_class.__dict__: + f = eb_class.__dict__[func] + + if f.__doc__: + custom.append('* ``' + func + '`` - ' + f.__doc__.strip() + inh) + + if custom: + title = 'Customised steps in ``' + classname + '`` easyblock' + lines.extend([title, '-' * len(title)] + custom) + lines.append('') + + # Add example if available + if os.path.exists(os.path.join(path_to_examples, '%s.eb' % classname)): + title = 'Example for ``' + classname + '`` easyblock' + lines.extend(['', title, '-' * len(title), '', '::', '']) + for line in read_file(os.path.join(path_to_examples, classname+'.eb')).split('\n'): + lines.append(' ' + line.strip()) + lines.append('') # empty line after literal block + + return '\n'.join(lines) diff --git a/easybuild/tools/environment.py b/easybuild/tools/environment.py index 073c43829d..cd6b290d77 100644 --- a/easybuild/tools/environment.py +++ b/easybuild/tools/environment.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,10 +28,17 @@ @author: Toon Willems (Ghent University) @author: Ward Poelmans (Ghent University) """ +import copy import os from vsc.utils import fancylogger from vsc.utils.missing import shell_quote +from easybuild.tools.build_log import EasyBuildError + + +# take copy of original environemt, so we can restore (parts of) it later +ORIG_OS_ENVIRON = copy.deepcopy(os.environ) + _log = fancylogger.getLogger('environment', fname=False) @@ -53,7 +60,7 @@ def write_changes(filename): except IOError, err: if script is not None: script.close() - _log.error("Failed to write to %s: %s" % (filename, err)) + raise EasyBuildError("Failed to write to %s: %s", filename, err) reset_changes() @@ -71,15 +78,20 @@ def get_changes(): """ return _changes + def setvar(key, value): """ put key in the environment with value tracks added keys until write_changes has been called """ + if key in os.environ: + oldval_info = "previous value: '%s'" % os.environ[key] + else: + oldval_info = "previously undefined" # os.putenv() is not necessary. os.environ will call this. os.environ[key] = value _changes[key] = value - _log.info("Environment variable %s set to %s" % (key, value)) + _log.info("Environment variable %s set to %s (%s)", key, value, oldval_info) def unset_env_vars(keys): @@ -102,7 +114,6 @@ def restore_env_vars(env_keys): """ Restore the environment by setting the keys in the env_keys dict again with their old value """ - for key in env_keys: if env_keys[key] is not None: _log.info("Restoring environment variable %s (value: %s)" % (key, env_keys[key])) @@ -121,7 +132,7 @@ def read_environment(env_vars, strict=False): missing = ','.join(["%s / %s" % (k, v) for k, v in env_vars.items() if not k in result]) msg = 'Following name/variable not found in environment: %s' % missing if strict: - _log.error(msg) + raise EasyBuildError(msg) else: _log.debug(msg) @@ -150,3 +161,10 @@ def modify_env(old, new): _log.debug("Key in old environment found that is not in new one: %s (%s)" % (key, old[key])) os.unsetenv(key) del os.environ[key] + + +def restore_env(env): + """ + Restore active environment based on specified dictionary. + """ + modify_env(os.environ, env) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index bd390a646b..287540acae 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -32,20 +32,22 @@ @author: Jens Timmerman (Ghent University) @author: Toon Willems (Ghent University) @author: Ward Poelmans (Ghent University) +@author: Fotis Georgatos (Uni.Lu, NTUA) +@author: Sotiris Fragkiskos (NTUA, CERN) """ -import errno +import glob +import hashlib import os import re import shutil import stat import time -import urllib +import urllib2 import zlib from vsc.utils import fancylogger -from vsc.utils.missing import all +from vsc.utils.missing import nub -import easybuild.tools.environment as env -from easybuild.tools.build_log import print_msg # import build_log must stay, to activate use of EasyBuildLog +from easybuild.tools.build_log import EasyBuildError, print_msg # import build_log must stay, to use of EasyBuildLog from easybuild.tools.config import build_option from easybuild.tools import run @@ -92,23 +94,14 @@ r'~': "_tilde_", } -try: - # preferred over md5/sha modules, but only available in Python 2.5 and more recent - import hashlib - md5_class = hashlib.md5 - sha1_class = hashlib.sha1 -except ImportError: - import md5, sha - md5_class = md5.md5 - sha1_class = sha.sha # default checksum for source and patch files DEFAULT_CHECKSUM = 'md5' # map of checksum types to checksum functions CHECKSUM_FUNCTIONS = { - 'md5': lambda p: calc_block_checksum(p, md5_class()), - 'sha1': lambda p: calc_block_checksum(p, sha1_class()), + 'md5': lambda p: calc_block_checksum(p, hashlib.md5()), + 'sha1': lambda p: calc_block_checksum(p, hashlib.sha1()), 'adler32': lambda p: calc_block_checksum(p, ZlibChecksum(zlib.adler32)), 'crc32': lambda p: calc_block_checksum(p, ZlibChecksum(zlib.crc32)), 'size': lambda p: os.path.getsize(p), @@ -148,25 +141,37 @@ def read_file(path, log_error=True): if f is not None: f.close() if log_error: - _log.error("Failed to read %s: %s" % (path, err)) + raise EasyBuildError("Failed to read %s: %s", path, err) else: return None -def write_file(path, txt): +def write_file(path, txt, append=False): """Write given contents to file at given path (overwrites current file contents!).""" f = None # note: we can't use try-except-finally, because Python 2.4 doesn't support it as a single block try: mkdir(os.path.dirname(path), parents=True) - f = open(path, 'w') + if append: + f = open(path, 'a') + else: + f = open(path, 'w') f.write(txt) f.close() except IOError, err: # make sure file handle is always closed if f is not None: f.close() - _log.error("Failed to write to %s: %s" % (path, err)) + raise EasyBuildError("Failed to write to %s: %s", path, err) + + +def remove_file(path): + """Remove file at specified path.""" + try: + if os.path.exists(path): + os.remove(path) + except OSError, err: + raise EasyBuildError("Failed to remove %s: %s", path, err) def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False): @@ -175,19 +180,19 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False): - returns the directory name in case of success """ if not os.path.isfile(fn): - _log.error("Can't extract file %s: no such file" % fn) + raise EasyBuildError("Can't extract file %s: no such file", fn) mkdir(dest, parents=True) # use absolute pathnames from now on - absDest = os.path.abspath(dest) + abs_dest = os.path.abspath(dest) # change working directory try: - _log.debug("Unpacking %s in directory %s." % (fn, absDest)) - os.chdir(absDest) + _log.debug("Unpacking %s in directory %s.", fn, abs_dest) + os.chdir(abs_dest) except OSError, err: - _log.error("Can't change to directory %s: %s" % (absDest, err)) + raise EasyBuildError("Can't change to directory %s: %s", abs_dest, err) if not cmd: cmd = extract_cmd(fn, overwrite=overwrite) @@ -195,7 +200,7 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False): # complete command template with filename cmd = cmd % fn if not cmd: - _log.error("Can't extract file %s with unknown filetype" % fn) + raise EasyBuildError("Can't extract file %s with unknown filetype", fn) if extra_options: cmd = "%s %s" % (cmd, extra_options) @@ -221,7 +226,7 @@ def which(cmd): def det_common_path_prefix(paths): """Determine common path prefix for a given list of paths.""" if not isinstance(paths, list): - _log.error("det_common_path_prefix: argument must be of type list (got %s: %s)" % (type(paths), paths)) + raise EasyBuildError("det_common_path_prefix: argument must be of type list (got %s: %s)", type(paths), paths) elif not paths: return None @@ -242,37 +247,54 @@ def det_common_path_prefix(paths): def download_file(filename, url, path): """Download a file from the given URL, to the specified path.""" - _log.debug("Downloading %s from %s to %s" % (filename, url, path)) + _log.debug("Trying to download %s from %s to %s", filename, url, path) + + timeout = build_option('download_timeout') + if timeout is None: + # default to 10sec timeout if none was specified + # default system timeout (used is nothing is specified) may be infinite (?) + timeout = 10 + _log.debug("Using timeout of %s seconds for initiating download" % timeout) # make sure directory exists basedir = os.path.dirname(path) mkdir(basedir, parents=True) + # try downloading, three times max. downloaded = False + max_attempts = 3 attempt_cnt = 0 - - # try downloading three times max. - while not downloaded and attempt_cnt < 3: - - (_, httpmsg) = urllib.urlretrieve(url, path) - - if httpmsg.type == "text/html" and not filename.endswith('.html'): - _log.warning("HTML file downloaded but not expecting it, so assuming invalid download.") - _log.debug("removing downloaded file %s from %s" % (filename, path)) - try: - os.remove(path) - except OSError, err: - _log.error("Failed to remove downloaded file:" % err) - else: - _log.info("Downloading file %s from url %s: done" % (filename, url)) + while not downloaded and attempt_cnt < max_attempts: + try: + # urllib2 does the right thing for http proxy setups, urllib does not! + url_fd = urllib2.urlopen(url, timeout=timeout) + _log.debug('response code for given url %s: %s' % (url, url_fd.getcode())) + write_file(path, url_fd.read()) + _log.info("Downloaded file %s from url %s to %s" % (filename, url, path)) downloaded = True - return path - - attempt_cnt += 1 - _log.warning("Downloading failed at attempt %s, retrying..." % attempt_cnt) - - # failed to download after multiple attempts - return None + url_fd.close() + except urllib2.HTTPError as err: + if 400 <= err.code <= 499: + _log.warning("URL %s was not found (HTTP response code %s), not trying again" % (url, err.code)) + break + else: + _log.warning("HTTPError occured while trying to download %s to %s: %s" % (url, path, err)) + attempt_cnt += 1 + except IOError as err: + _log.warning("IOError occurred while trying to download %s to %s: %s" % (url, path, err)) + attempt_cnt += 1 + except Exception, err: + raise EasyBuildError("Unexpected error occurred when trying to download %s to %s: %s", url, path, err) + + if not downloaded and attempt_cnt < max_attempts: + _log.info("Attempt %d of downloading %s to %s failed, trying again..." % (attempt_cnt, url, path)) + + if downloaded: + _log.info("Successful download of file %s from url %s to path %s" % (filename, url, path)) + return path + else: + _log.warning("Download of %s to %s failed, done trying" % (url, path)) + return None def find_easyconfigs(path, ignore_dirs=None): @@ -310,7 +332,11 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False): if ignore_dirs is None: ignore_dirs = ['.git', '.svn'] if not isinstance(ignore_dirs, list): - _log.error("search_file: ignore_dirs (%s) should be of type list, not %s" % (ignore_dirs, type(ignore_dirs))) + raise EasyBuildError("search_file: ignore_dirs (%s) should be of type list, not %s", + ignore_dirs, type(ignore_dirs)) + + # compile regex, case-insensitive + query = re.compile(query, re.I) var_lines = [] hit_lines = [] @@ -319,18 +345,16 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False): for path in paths: hits = [] hit_in_path = False - print_msg("Searching (case-insensitive) for '%s' in %s " % (query, path), log=_log, silent=silent) + print_msg("Searching (case-insensitive) for '%s' in %s " % (query.pattern, path), log=_log, silent=silent) - query = query.lower() for (dirpath, dirnames, filenames) in os.walk(path, topdown=True): for filename in filenames: - filename = os.path.join(dirpath, filename) - if filename.lower().find(query) != -1: + if query.search(filename): if not hit_in_path: var = "CFGS%d" % var_index var_index += 1 hit_in_path = True - hits.append(filename) + hits.append(os.path.join(dirpath, filename)) # do not consider (certain) hidden directories # note: we still need to consider e.g., .local ! @@ -338,6 +362,8 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False): # see http://stackoverflow.com/questions/13454164/os-walk-without-hidden-folders dirnames[:] = [d for d in dirnames if not d in ignore_dirs] + hits = sorted(hits) + if hits: common_prefix = det_common_path_prefix(hits) if short and common_prefix is not None and len(common_prefix) > len(var) * 2: @@ -358,12 +384,13 @@ def compute_checksum(path, checksum_type=DEFAULT_CHECKSUM): @param checksum_type: Type of checksum ('adler32', 'crc32', 'md5' (default), 'sha1', 'size') """ if not checksum_type in CHECKSUM_FUNCTIONS: - _log.error("Unknown checksum type (%s), supported types are: %s" % (checksum_type, CHECKSUM_FUNCTIONS.keys())) + raise EasyBuildError("Unknown checksum type (%s), supported types are: %s", + checksum_type, CHECKSUM_FUNCTIONS.keys()) try: checksum = CHECKSUM_FUNCTIONS[checksum_type](path) except IOError, err: - _log.error("Failed to read %s: %s" % (path, err)) + raise EasyBuildError("Failed to read %s: %s", path, err) except MemoryError, err: _log.warning("A memory error occured when computing the checksum for %s: %s" % (path, err)) checksum = 'dummy_checksum_due_to_memory_error' @@ -388,7 +415,7 @@ def calc_block_checksum(path, algorithm): algorithm.update(block) f.close() except IOError, err: - _log.error("Failed to read %s: %s" % (path, err)) + raise EasyBuildError("Failed to read %s: %s", path, err) return algorithm.hexdigest() @@ -415,7 +442,8 @@ def verify_checksum(path, checksums): elif isinstance(checksum, tuple) and len(checksum) == 2: typ, checksum = checksum else: - _log.error("Invalid checksum spec '%s', should be a string (MD5) or 2-tuple (type, value)." % checksum) + raise EasyBuildError("Invalid checksum spec '%s', should be a string (MD5) or 2-tuple (type, value).", + checksum) actual_checksum = compute_checksum(path, typ) _log.debug("Computed %s checksum for %s: %s (correct checksum: %s)" % (typ, path, actual_checksum, checksum)) @@ -436,12 +464,11 @@ def find_base_dir(): """ def get_local_dirs_purged(): # e.g. always purge the log directory - ignoreDirs = ["easybuild"] + # and hidden directories + ignoredirs = ["easybuild"] lst = os.listdir(os.getcwd()) - for ignDir in ignoreDirs: - if ignDir in lst: - lst.remove(ignDir) + lst = [d for d in lst if not d.startswith('.') and d not in ignoredirs] return lst lst = get_local_dirs_purged() @@ -454,7 +481,7 @@ def get_local_dirs_purged(): try: os.chdir(new_dir) except OSError, err: - _log.exception("Changing to dir %s from current dir %s failed: %s" % (new_dir, os.getcwd(), err)) + raise EasyBuildError("Changing to dir %s from current dir %s failed: %s", new_dir, os.getcwd(), err) lst = get_local_dirs_purged() # make sure it's a directory, and not a (single) file that was in a tarball for example @@ -468,59 +495,47 @@ def get_local_dirs_purged(): def extract_cmd(filepath, overwrite=False): """ - Determines the file type of file fn, returns extract cmd - - based on file suffix - - better to use Python magic? + Determines the file type of file at filepath, returns extract cmd based on file suffix """ filename = os.path.basename(filepath) - exts = [x.lower() for x in filename.split('.')] - target = '.'.join(exts[:-1]) - cmd_tmpl = None - - # gzipped or gzipped tarball - if exts[-1] in ['gz']: - if exts[-2] in ['tar']: - # unzip .tar.gz in one go - cmd_tmpl = "tar xzf %(filepath)s" - else: - cmd_tmpl = "gunzip -c %(filepath)s > %(target)s" - - elif exts[-1] in ['tgz', 'gtgz']: - cmd_tmpl = "tar xzf %(filepath)s" - - # bzipped or bzipped tarball - elif exts[-1] in ['bz2']: - if exts[-2] in ['tar']: - cmd_tmpl = 'tar xjf %(filepath)s' - else: - cmd_tmpl = "bunzip2 %(filepath)s" - - elif exts[-1] in ['tbz', 'tbz2', 'tb2']: - cmd_tmpl = "tar xjf %(filepath)s" - - # xzipped or xzipped tarball - elif exts[-1] in ['xz']: - if exts[-2] in ['tar']: - cmd_tmpl = "unxz %(filepath)s --stdout | tar x" - else: - cmd_tmpl = "unxz %(filepath)s" - elif exts[-1] in ['txz']: - cmd_tmpl = "unxz %(filepath)s --stdout | tar x" + extract_cmds = { + # gzipped or gzipped tarball + '.gtgz': "tar xzf %(filepath)s", + '.gz': "gunzip -c %(filepath)s > %(target)s", + '.tar.gz': "tar xzf %(filepath)s", + '.tgz': "tar xzf %(filepath)s", + # bzipped or bzipped tarball + '.bz2': "bunzip2 %(filepath)s", + '.tar.bz2': "tar xjf %(filepath)s", + '.tb2': "tar xjf %(filepath)s", + '.tbz': "tar xjf %(filepath)s", + '.tbz2': "tar xjf %(filepath)s", + # xzipped or xzipped tarball + '.tar.xz': "unxz %(filepath)s --stdout | tar x", + '.txz': "unxz %(filepath)s --stdout | tar x", + '.xz': "unxz %(filepath)s", + # tarball + '.tar': "tar xf %(filepath)s", + # zip file + '.zip': "unzip -qq -o %(filepath)s" if overwrite else "unzip -qq %(filepath)s", + # iso file + '.iso': "7z x %(filepath)s", + # tar.Z: using compress (LZW) + '.tar.z': "tar xZf %(filepath)s", + } - # tarball - elif exts[-1] in ['tar']: - cmd_tmpl = "tar xf %(filepath)s" + suffixes = sorted(extract_cmds.keys(), key=len, reverse=True) + pat = r'(?P%s)$' % '|'.join([ext.replace('.', '\\.') for ext in suffixes]) + res = re.search(pat, filename, flags=re.IGNORECASE) + if res: + ext = res.group('ext') + else: + raise EasyBuildError('Unknown file type for file %s', filename) - # zip file - elif exts[-1] in ['zip']: - if overwrite: - cmd_tmpl = "unzip -qq -o %(filepath)s" - else: - cmd_tmpl = "unzip -qq %(filepath)s" + target = filename.rstrip(ext) - if cmd_tmpl is None: - _log.error('Unknown file type for file %s (%s)' % (filepath, exts)) + cmd_tmpl = extract_cmds[ext.lower()] return cmd_tmpl % {'filepath': filepath, 'target': target} @@ -536,16 +551,20 @@ def det_patched_files(path=None, txt=None, omit_ab_prefix=False): txt = f.read() f.close() except IOError, err: - _log.error("Failed to read patch %s: %s" % (path, err)) + raise EasyBuildError("Failed to read patch %s: %s", path, err) elif txt is None: - _log.error("Either a file path or a string representing a patch should be supplied to det_patched_files") + raise EasyBuildError("Either a file path or a string representing a patch should be supplied") patched_files = [] for match in patched_regex.finditer(txt): patched_file = match.group('file') if not omit_ab_prefix and match.group('ab_prefix') is not None: patched_file = match.group('ab_prefix') + patched_file - patched_files.append(patched_file) + + if patched_file in ['/dev/null']: + _log.debug("Ignoring patched file %s" % patched_file) + else: + patched_files.append(patched_file) return patched_files @@ -579,15 +598,15 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None): """ if not os.path.isfile(patch_file): - _log.error("Can't find patch %s: no such file" % patch_file) + raise EasyBuildError("Can't find patch %s: no such file", patch_file) return if fn and not os.path.isfile(fn): - _log.error("Can't patch file %s: no such file" % fn) + raise EasyBuildError("Can't patch file %s: no such file", fn) return if not os.path.isdir(dest): - _log.error("Can't patch directory %s: no such directory" % dest) + raise EasyBuildError("Can't patch directory %s: no such directory", dest) return # copy missing files @@ -597,7 +616,7 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None): _log.debug("Copied patch %s to dir %s" % (patch_file, dest)) return 'ok' except IOError, err: - _log.error("Failed to copy %s to dir %s: %s" % (patch_file, dest, err)) + raise EasyBuildError("Failed to copy %s to dir %s: %s", patch_file, dest, err) return # use absolute paths @@ -612,14 +631,14 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None): patched_files = det_patched_files(path=apatch) if not patched_files: - _log.error("Can't guess patchlevel from patch %s: no testfile line found in patch" % apatch) + raise EasyBuildError("Can't guess patchlevel from patch %s: no testfile line found in patch", apatch) return patch_level = guess_patch_level(patched_files, adest) if patch_level is None: # patch_level can also be 0 (zero), so don't use "not patch_level" # no match - _log.error("Can't determine patch level for patch %s from directory %s" % (patch_file, adest)) + raise EasyBuildError("Can't determine patch level for patch %s from directory %s", patch_file, adest) else: _log.debug("Guessed patch level %d for patch %s" % (patch_level, patch_file)) @@ -631,24 +650,21 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None): os.chdir(adest) _log.debug("Changing to directory %s" % adest) except OSError, err: - _log.error("Can't change to directory %s: %s" % (adest, err)) + raise EasyBuildError("Can't change to directory %s: %s", adest, err) return patch_cmd = "patch -b -p%d -i %s" % (patch_level, apatch) result = run.run_cmd(patch_cmd, simple=True) if not result: - _log.error("Patching with patch %s failed" % patch_file) + raise EasyBuildError("Patching with patch %s failed", patch_file) return return result def modify_env(old, new): - """ - Compares 2 os.environ dumps. Adapts final environment. - """ - _log.deprecated("moved modify_env to tools.environment", "2.0") - return env.modify_env(old, new) + """NO LONGER SUPPORTED: use modify_env from easybuild.tools.environment instead""" + _log.nosupport("moved modify_env to easybuild.tools.environment", "2.0") def convert_name(name, upper=False): @@ -732,14 +748,14 @@ def adjust_permissions(name, permissionBits, add=True, onlyfiles=False, onlydirs failed_paths.append(path) if failed_paths: - _log.error("Failed to chmod/chown several paths: %s (last error: %s)" % (failed_paths, err)) + raise EasyBuildError("Failed to chmod/chown several paths: %s (last error: %s)", failed_paths, err) # we ignore some errors, but if there are to many, something is definitely wrong fail_ratio = fail_cnt / float(len(allpaths)) max_fail_ratio = 0.5 if fail_ratio > max_fail_ratio: - _log.error("%.2f%% of permissions/owner operations failed (more than %.2f%%), something must be wrong..." % - (100 * fail_ratio, 100 * max_fail_ratio)) + raise EasyBuildError("%.2f%% of permissions/owner operations failed (more than %.2f%%), " + "something must be wrong...", 100 * fail_ratio, 100 * max_fail_ratio) elif fail_cnt > 0: _log.debug("%.2f%% of permissions/owner operations failed, ignoring that..." % (100 * fail_ratio)) @@ -784,8 +800,7 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): # exit early if path already exists if not os.path.exists(path): - tup = (path, parents, set_gid, sticky) - _log.info("Creating directory %s (parents: %s, set_gid: %s, sticky: %s)" % tup) + _log.info("Creating directory %s (parents: %s, set_gid: %s, sticky: %s)", path, parents, set_gid, sticky) # set_gid and sticky bits are only set on new directories, so we need to determine the existing parent path existing_parent_path = os.path.dirname(path) try: @@ -797,7 +812,7 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): else: os.mkdir(path) except OSError, err: - _log.error("Failed to create directory %s: %s" % (path, err)) + raise EasyBuildError("Failed to create directory %s: %s", path, err) # set group ID and sticky bits, if desired bits = 0 @@ -811,11 +826,63 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): new_path = os.path.join(existing_parent_path, new_subdir.split(os.path.sep)[0]) adjust_permissions(new_path, bits, add=True, relative=True, recursive=True, onlydirs=True) except OSError, err: - _log.error("Failed to set groud ID/sticky bit: %s" % err) + raise EasyBuildError("Failed to set groud ID/sticky bit: %s", err) else: _log.debug("Not creating existing path %s" % path) +def expand_glob_paths(glob_paths): + """Expand specified glob paths to a list of unique non-glob paths to only files.""" + paths = [] + for glob_path in glob_paths: + paths.extend([f for f in glob.glob(glob_path) if os.path.isfile(f)]) + + return nub(paths) + + +def weld_paths(path1, path2): + """Weld two paths together, taking into account overlap between tail of 1st path with head of 2nd path.""" + # strip path1 for use in comparisons + path1s = path1.rstrip(os.path.sep) + + # init part2 head/tail/parts + path2_head = path2.rstrip(os.path.sep) + path2_tail = '' + path2_parts = path2.split(os.path.sep) + # if path2 is an absolute path, make sure it stays that way + if path2_parts[0] == '': + path2_parts[0] = os.path.sep + + while path2_parts and not path1s.endswith(path2_head): + path2_tail = os.path.join(path2_parts.pop(), path2_tail) + if path2_parts: + # os.path.join requires non-empty list + path2_head = os.path.join(*path2_parts) + else: + path2_head = None + + return os.path.join(path1, path2_tail) + + +def symlink(source_path, symlink_path): + """Create a symlink at the specified path to the given path.""" + try: + os.symlink(os.path.abspath(source_path), symlink_path) + _log.info("Symlinked %s to %s", source_path, symlink_path) + except OSError as err: + raise EasyBuildError("Symlinking %s to %s failed: %s", source_path, symlink_path, err) + + +def path_matches(path, paths): + """Check whether given path matches any of the provided paths.""" + if not os.path.exists(path): + return False + for somepath in paths: + if os.path.exists(somepath) and os.path.samefile(path, somepath): + return True + return False + + def rmtree2(path, n=3): """Wrapper around shutil.rmtree to make it more robust when used on NFS mounted file systems.""" @@ -829,20 +896,63 @@ def rmtree2(path, n=3): _log.debug("Failed to remove path %s with shutil.rmtree at attempt %d: %s" % (path, n, err)) time.sleep(2) if not ok: - _log.error("Failed to remove path %s with shutil.rmtree, even after %d attempts." % (path, n)) + raise EasyBuildError("Failed to remove path %s with shutil.rmtree, even after %d attempts.", path, n) else: _log.info("Path %s successfully removed." % path) +def move_logs(src_logfile, target_logfile): + """Move log file(s).""" + mkdir(os.path.dirname(target_logfile), parents=True) + src_logfile_len = len(src_logfile) + try: + + # there may be multiple log files, due to log rotation + app_logs = glob.glob('%s*' % src_logfile) + for app_log in app_logs: + # retain possible suffix + new_log_path = target_logfile + app_log[src_logfile_len:] + + # retain old logs + if os.path.exists(new_log_path): + i = 0 + oldlog_backup = "%s_%d" % (new_log_path, i) + while os.path.exists(oldlog_backup): + i += 1 + oldlog_backup = "%s_%d" % (new_log_path, i) + shutil.move(new_log_path, oldlog_backup) + _log.info("Moved existing log file %s to %s" % (new_log_path, oldlog_backup)) + + # move log to target path + shutil.move(app_log, new_log_path) + _log.info("Moved log file %s to %s" % (src_logfile, new_log_path)) + + except (IOError, OSError), err: + raise EasyBuildError("Failed to move log file(s) %s* to new log file %s*: %s" , + src_logfile, target_logfile, err) + + def cleanup(logfile, tempdir, testing): - """Cleanup the specified log file and the tmp directory""" - if not testing and logfile is not None: - os.remove(logfile) - print_msg('temporary log file %s has been removed.' % (logfile), log=None, silent=testing) + """Cleanup the specified log file and the tmp directory, if desired.""" + + if build_option('cleanup_tmpdir') and not testing: + if logfile is not None: + try: + for log in [logfile] + glob.glob('%s.[0-9]*' % logfile): + os.remove(log) + except OSError, err: + raise EasyBuildError("Failed to remove log file(s) %s*: %s", logfile, err) + print_msg("Temporary log file(s) %s* have been removed." % (logfile), log=None, silent=testing) - if not testing and tempdir is not None: - shutil.rmtree(tempdir, ignore_errors=True) - print_msg('temporary directory %s has been removed.' % (tempdir), log=None, silent=testing) + if tempdir is not None: + try: + shutil.rmtree(tempdir, ignore_errors=True) + except OSError, err: + raise EasyBuildError("Failed to remove temporary directory %s: %s", tempdir, err) + print_msg("Temporary directory %s has been removed." % tempdir, log=None, silent=testing) + + else: + print_msg("Keeping temporary log file(s) %s* and directory %s." % (logfile, tempdir), log=None, silent=testing) def copytree(src, dst, symlinks=False, ignore=None): @@ -970,19 +1080,17 @@ def decode_class_name(name): def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None): - """Legacy wrapper/placeholder for run.run_cmd""" - return run.run_cmd(cmd, log_ok=log_ok, log_all=log_all, simple=simple, - inp=inp, regexp=regexp, log_output=log_output, path=path) + """NO LONGER SUPPORTED: use run_cmd from easybuild.tools.run instead""" + _log.nosupport("run_cmd was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0') def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None): - """Legacy wrapper/placeholder for run.run_cmd_qa""" - return run.run_cmd_qa(cmd, qa, no_qa=no_qa, log_ok=log_ok, log_all=log_all, - simple=simple, regexp=regexp, std_qa=std_qa, path=path) + """NO LONGER SUPPORTED: use run_cmd_qa from easybuild.tools.run instead""" + _log.nosupport("run_cmd_qa was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0') def parse_log_for_error(txt, regExp=None, stdout=True, msg=None): - """Legacy wrapper/placeholder for run.parse_log_for_error""" - return run.parse_log_for_error(txt, regExp=regExp, stdout=stdout, msg=msg) + """NO LONGER SUPPORTED: use parse_log_for_error from easybuild.tools.run instead""" + _log.nosupport("parse_log_for_error was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0') def det_size(path): diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 5e0f1f0729..f6d0705b42 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -27,17 +27,19 @@ @author: Jens Timmerman (Ghent University) @author: Kenneth Hoste (Ghent University) +@author: Toon Willems (Ghent University) """ import base64 import os -import re import socket import tempfile -import urllib -import urllib2 from vsc.utils import fancylogger from vsc.utils.patterns import Singleton +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option +from easybuild.tools.filetools import det_patched_files, download_file, extract_file, mkdir, read_file, write_file + _log = fancylogger.getLogger('github', fname=False) @@ -56,14 +58,14 @@ _log.warning("Failed to import from 'vsc.utils.rest' Python module: %s" % err) HAVE_GITHUB_API = False -from easybuild.tools.filetools import det_patched_files, mkdir - +GITHUB_URL = 'https://github.com' GITHUB_API_URL = 'https://api.github.com' GITHUB_DIR_TYPE = u'dir' GITHUB_EB_MAIN = 'hpcugent' GITHUB_EASYCONFIGS_REPO = 'easybuild-easyconfigs' GITHUB_FILE_TYPE = u'file' +GITHUB_MAX_PER_PAGE = 100 GITHUB_MERGEABLE_STATE_CLEAN = 'clean' GITHUB_RAW = 'https://raw.githubusercontent.com' GITHUB_STATE_CLOSED = 'closed' @@ -139,7 +141,7 @@ def listdir(self, path): return listing[1] else: self.log.warning("error: %s" % str(listing)) - self.log.exception("Invalid response from github (I/O error)") + raise EasyBuildError("Invalid response from github (I/O error)") def walk(self, top=None, topdown=True): """ @@ -174,8 +176,8 @@ def read(self, path, api=True): # https://raw.github.com/hpcugent/easybuild/master/README.rst if not api: outfile = tempfile.mkstemp()[1] - url = ("http://raw.github.com/%s/%s/%s/%s" % (self.githubuser, self.reponame, self.branchname, path)) - urllib.urlretrieve(url, outfile) + url = '/'.join([GITHUB_RAW, self.githubuser, self.reponame, self.branchname, path]) + download_file(os.path.basename(path), url, outfile) return outfile else: obj = self.get_path(path).get(ref=self.branchname)[1] @@ -189,28 +191,108 @@ class GithubError(Exception): pass -def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): - """Fetch patched easyconfig files for a particular PR.""" +def github_api_get_request(request_f, github_user=None, **kwargs): + """ + Helper method, for performing get requests to GitHub API. + @param request_f: function that should be called to compose request, providing a RestClient instance + @param github_user: GitHub user name (to try and obtain matching GitHub token) + @return: tuple with return status and data + """ + if github_user is None: + github_user = build_option('github_user') - def download(url, path=None): - """Download file from specified URL to specified path.""" - if path is not None: - try: - _, httpmsg = urllib.urlretrieve(url, path) - _log.debug("Downloaded %s to %s" % (url, path)) - except IOError, err: - _log.error("Failed to download %s to %s: %s" % (url, path, err)) + token = fetch_github_token(github_user) + url = request_f(RestClient(GITHUB_API_URL, username=github_user, token=token)) - if not httpmsg.type == 'text/plain': - _log.error("Unexpected file type for %s: %s" % (path, httpmsg.type)) - else: - try: - return urllib2.urlopen(url).read() - except urllib2.URLError, err: - _log.error("Failed to open %s for reading: %s" % (url, err)) + try: + status, data = url.get(**kwargs) + except socket.gaierror, err: + _log.warning("Error occured while performing get request: %s" % err) + status, data = 0, None + + _log.debug("get request result for %s: status: %d, data: %s" % (url, status, data)) + return (status, data) + + +def fetch_latest_commit_sha(repo, account, branch='master'): + """ + Fetch latest SHA1 for a specified repository and branch. + @param repo: GitHub repository + @param account: GitHub account + @param branch: branch to fetch latest SHA1 for + @return: latest SHA1 + """ + status, data = github_api_get_request(lambda x: x.repos[account][repo].branches) + if not status == HTTP_STATUS_OK: + raise EasyBuildError("Failed to get latest commit sha for branch %s from %s/%s (status: %d %s)", + branch, account, repo, status, data) + + res = None + for entry in data: + if entry[u'name'] == branch: + res = entry['commit']['sha'] + break + + if res is None: + raise EasyBuildError("No branch with name %s found in repo %s/%s (%s)", branch, account, repo, data) + + return res + + +def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='master', account=GITHUB_EB_MAIN, path=None): + """ + Download entire GitHub repo as a tar.gz archive, and extract it into specified path. + @param repo: repo to download + @param branch: branch to download + @param account: GitHub account to download repo from + @param path: path to extract to + """ + # make sure path exists, create it if necessary + if path is None: + path = tempfile.mkdtemp() - # a GitHub token is optional here, but can be used if available in order to be less susceptible to rate limiting - github_token = fetch_github_token(github_user) + # add account subdir + path = os.path.join(path, account) + mkdir(path, parents=True) + + extracted_dir_name = '%s-%s' % (repo, branch) + base_name = '%s.tar.gz' % branch + latest_commit_sha = fetch_latest_commit_sha(repo, account, branch) + + expected_path = os.path.join(path, extracted_dir_name) + latest_sha_path = os.path.join(expected_path, 'latest-sha') + + # check if directory already exists, don't download if 'latest-sha' file indicates that it's up to date + if os.path.exists(latest_sha_path): + sha = read_file(latest_sha_path).split('\n')[0].rstrip() + if latest_commit_sha == sha: + _log.debug("Not redownloading %s/%s as it already exists: %s" % (account, repo, expected_path)) + return expected_path + + url = URL_SEPARATOR.join([GITHUB_URL, account, repo, 'archive', base_name]) + + target_path = os.path.join(path, base_name) + _log.debug("downloading repo %s/%s as archive from %s to %s" % (account, repo, url, target_path)) + download_file(base_name, url, target_path) + _log.debug("%s downloaded to %s, extracting now" % (base_name, path)) + + extracted_path = os.path.join(extract_file(target_path, path), extracted_dir_name) + # check if extracted_path exists + if not os.path.isdir(extracted_path): + raise EasyBuildError("%s should exist and contain the repo %s at branch %s", extracted_path, repo, branch) + + write_file(latest_sha_path, latest_commit_sha) + + _log.debug("Repo %s at branch %s extracted into %s" % (repo, branch, extracted_path)) + return extracted_path + + +def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): + """Fetch patched easyconfig files for a particular PR.""" + if github_user is None: + github_user = build_option('github_user') + if path is None: + path = build_option('pr_path') if path is None: path = tempfile.mkdtemp() @@ -219,38 +301,40 @@ def download(url, path=None): mkdir(path, parents=True) _log.debug("Fetching easyconfigs from PR #%s into %s" % (pr, path)) + pr_url = lambda g: g.repos[GITHUB_EB_MAIN][GITHUB_EASYCONFIGS_REPO].pulls[pr] - # fetch data for specified PR - g = RestClient(GITHUB_API_URL, username=github_user, token=github_token) - pr_url = g.repos[GITHUB_EB_MAIN][GITHUB_EASYCONFIGS_REPO].pulls[pr] - try: - status, pr_data = pr_url.get() - except socket.gaierror, err: - status, pr_data = 0, None - _log.debug("status: %d, data: %s" % (status, pr_data)) + status, pr_data = github_api_get_request(pr_url, github_user) if not status == HTTP_STATUS_OK: - tup = (pr, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, status, pr_data) - _log.error("Failed to get data for PR #%d from %s/%s (status: %d %s)" % tup) + raise EasyBuildError("Failed to get data for PR #%d from %s/%s (status: %d %s)", + pr, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, status, pr_data) # 'clean' on successful (or missing) test, 'unstable' on failed tests stable = pr_data['mergeable_state'] == GITHUB_MERGEABLE_STATE_CLEAN if not stable: - tup = (pr, GITHUB_MERGEABLE_STATE_CLEAN, pr_data['mergeable_state']) - _log.warning("Mergeable state for PR #%d is not '%s': %s." % tup) + _log.warning("Mergeable state for PR #%d is not '%s': %s.", + pr, GITHUB_MERGEABLE_STATE_CLEAN, pr_data['mergeable_state']) for key, val in sorted(pr_data.items()): _log.debug("\n%s:\n\n%s\n" % (key, val)) # determine list of changed files via diff - diff_txt = download(pr_data['diff_url']) + diff_fn = os.path.basename(pr_data['diff_url']) + diff_filepath = os.path.join(path, diff_fn) + download_file(diff_fn, pr_data['diff_url'], diff_filepath) + diff_txt = read_file(diff_filepath) + os.remove(diff_filepath) patched_files = det_patched_files(txt=diff_txt, omit_ab_prefix=True) - _log.debug("List of patches files: %s" % patched_files) + _log.debug("List of patched files: %s" % patched_files) # obtain last commit - status, commits_data = pr_url.commits.get() + # get all commits, increase to (max of) 100 per page + if pr_data['commits'] > GITHUB_MAX_PER_PAGE: + raise EasyBuildError("PR #%s contains more than %s commits, can't obtain last commit", pr, GITHUB_MAX_PER_PAGE) + status, commits_data = github_api_get_request(lambda g: pr_url(g).commits, github_user, + per_page=GITHUB_MAX_PER_PAGE) last_commit = commits_data[-1] - _log.debug("Commits: %s" % commits_data) + _log.debug("Commits: %s, last commit: %s" % (commits_data, last_commit['sha'])) # obtain most recent version of patched files for patched_file in patched_files: @@ -258,12 +342,12 @@ def download(url, path=None): sha = last_commit['sha'] full_url = URL_SEPARATOR.join([GITHUB_RAW, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, sha, patched_file]) _log.info("Downloading %s from %s" % (fn, full_url)) - download(full_url, path=os.path.join(path, fn)) + download_file(fn, full_url, path=os.path.join(path, fn)) all_files = [os.path.basename(x) for x in patched_files] tmp_files = os.listdir(path) if not sorted(tmp_files) == sorted(all_files): - _log.error("Not all patched files were downloaded to %s: %s vs %s" % (path, tmp_files, all_files)) + raise EasyBuildError("Not all patched files were downloaded to %s: %s vs %s", path, tmp_files, all_files) ec_files = [os.path.join(path, fn) for fn in tmp_files] @@ -289,7 +373,7 @@ def create_gist(txt, fn, descr=None, github_user=None): status, data = g.gists.post(body=body) if not status == HTTP_STATUS_CREATED: - _log.error("Failed to create gist; status %s, data: %s" % (status, data)) + raise EasyBuildError("Failed to create gist; status %s, data: %s", status, data) return data['html_url'] @@ -300,7 +384,7 @@ def post_comment_in_issue(issue, txt, repo=GITHUB_EASYCONFIGS_REPO, github_user= try: issue = int(issue) except ValueError, err: - _log.error("Failed to parse specified pull request number '%s' as an int: %s; " % (issue, err)) + raise EasyBuildError("Failed to parse specified pull request number '%s' as an int: %s; ", issue, err) github_token = fetch_github_token(github_user) g = RestClient(GITHUB_API_URL, username=github_user, token=github_token) @@ -308,7 +392,7 @@ def post_comment_in_issue(issue, txt, repo=GITHUB_EASYCONFIGS_REPO, github_user= status, data = pr_url.comments.post(body={'body': txt}) if not status == HTTP_STATUS_CREATED: - _log.error("Failed to create comment in PR %s#%d; status %s, data: %s" % (repo, issue, status, data)) + raise EasyBuildError("Failed to create comment in PR %s#%d; status %s, data: %s", repo, issue, status, data) class GithubToken(object): diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py new file mode 100644 index 0000000000..dbde8e067f --- /dev/null +++ b/easybuild/tools/include.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python +# # +# Copyright 2015-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Support for including additional Python modules, for easyblocks, module naming schemes and toolchains. + +@author: Kenneth Hoste (Ghent University) +""" +import os +import sys +from vsc.utils import fancylogger + +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import expand_glob_paths, symlink +# these are imported just to we can reload them later +import easybuild.tools.module_naming_scheme +import easybuild.toolchains +import easybuild.toolchains.compiler +import easybuild.toolchains.fft +import easybuild.toolchains.linalg +import easybuild.toolchains.mpi +# importing easyblocks namespace may fail if easybuild-easyblocks is not available +# for now, we don't really care +try: + import easybuild.easyblocks + import easybuild.easyblocks.generic +except ImportError: + pass + + +_log = fancylogger.getLogger('tools.include', fname=False) + + +# body for __init__.py file in package directory, which takes care of making sure the package can be distributed +# across multiple directories +PKG_INIT_BODY = """ +from pkgutil import extend_path + +# extend path so Python knows this is not the only place to look for modules in this package +__path__ = extend_path(__path__, __name__) +""" + +# more extensive __init__.py specific to easybuild.easyblocks package; +# this is required because of the way in which the easyblock Python modules are organised in the easybuild-easyblocks +# repository, i.e. in first-letter subdirectories +EASYBLOCKS_PKG_INIT_BODY = """ +from pkgutil import extend_path + +# extend path so Python finds our easyblocks in the subdirectories where they are located +subdirs = [chr(l) for l in range(ord('a'), ord('z') + 1)] + ['0'] +for subdir in subdirs: + __path__ = extend_path(__path__, '%s.%s' % (__name__, subdir)) + +# extend path so Python knows this is not the only place to look for modules in this package +__path__ = extend_path(__path__, __name__) + +del subdir, subdirs, l +""" + + +def create_pkg(path, pkg_init_body=None): + """Write package __init__.py file at specified path.""" + init_path = os.path.join(path, '__init__.py') + try: + # note: can't use mkdir, since that required build options to be initialised + if not os.path.exists(path): + os.makedirs(path) + + # put __init__.py files in place, with required pkgutil.extend_path statement + # note: can't use write_file, since that required build options to be initialised + with open(init_path, 'w') as handle: + if pkg_init_body is None: + handle.write(PKG_INIT_BODY) + else: + handle.write(pkg_init_body) + + except (IOError, OSError) as err: + raise EasyBuildError("Failed to create package at %s: %s", path, err) + + +def set_up_eb_package(parent_path, eb_pkg_name, subpkgs=None, pkg_init_body=None): + """ + Set up new easybuild subnamespace in specified path. + + @param parent_path: directory to create package in, using 'easybuild' namespace + @param eb_pkg_name: full package name, must start with 'easybuild' + @param subpkgs: list of subpackages to create + @parak pkg_init_body: body of package's __init__.py file (does not apply to subpackages) + """ + if not eb_pkg_name.startswith('easybuild'): + raise EasyBuildError("Specified EasyBuild package name does not start with 'easybuild': %s", eb_pkg_name) + + pkgpath = os.path.join(parent_path, eb_pkg_name.replace('.', os.path.sep)) + + # handle subpackages first + if subpkgs: + for subpkg in subpkgs: + create_pkg(os.path.join(pkgpath, subpkg)) + + # creata package dirs on each level + while pkgpath != parent_path: + create_pkg(pkgpath, pkg_init_body=pkg_init_body) + pkgpath = os.path.dirname(pkgpath) + + +def verify_imports(pymods, pypkg, from_path): + """Verify that import of specified modules from specified package and expected location works.""" + for pymod in pymods: + pymod_spec = '%s.%s' % (pypkg, pymod) + try: + pymod = __import__(pymod_spec, fromlist=[pypkg]) + # different types of exceptions may be thrown, not only ImportErrors + # e.g. when module being imported contains syntax errors or undefined variables + except Exception as err: + raise EasyBuildError("Failed to import easyblock %s from %s: %s", pymod_spec, from_path, err) + + if not os.path.samefile(os.path.dirname(pymod.__file__), from_path): + raise EasyBuildError("Module %s not imported from expected location (%s): %s", + pymod_spec, from_path, pymod.__file__) + + _log.debug("Import of %s from %s verified", pymod_spec, from_path) + + +def include_easyblocks(tmpdir, paths): + """Include generic and software-specific easyblocks found in specified locations.""" + easyblocks_path = os.path.join(tmpdir, 'included-easyblocks') + + set_up_eb_package(easyblocks_path, 'easybuild.easyblocks', + subpkgs=['generic'], pkg_init_body=EASYBLOCKS_PKG_INIT_BODY) + + easyblocks_dir = os.path.join(easyblocks_path, 'easybuild', 'easyblocks') + + allpaths = expand_glob_paths(paths) + for easyblock_module in allpaths: + filename = os.path.basename(easyblock_module) + + # generic easyblocks are expected to be in a directory named 'generic' + parent_dir = os.path.basename(os.path.dirname(easyblock_module)) + if parent_dir == 'generic': + target_path = os.path.join(easyblocks_dir, 'generic', filename) + else: + target_path = os.path.join(easyblocks_dir, filename) + + symlink(easyblock_module, target_path) + + included_ebs = [x for x in os.listdir(easyblocks_dir) if x not in ['__init__.py', 'generic']] + included_generic_ebs = [x for x in os.listdir(os.path.join(easyblocks_dir, 'generic')) if x != '__init__.py'] + _log.debug("Included generic easyblocks: %s", included_generic_ebs) + _log.debug("Included software-specific easyblocks: %s", included_ebs) + + # inject path into Python search path, and reload modules to get it 'registered' in sys.modules + sys.path.insert(0, easyblocks_path) + reload(easybuild) + if 'easybuild.easyblocks' in sys.modules: + reload(easybuild.easyblocks) + reload(easybuild.easyblocks.generic) + + # sanity check: verify that included easyblocks can be imported (from expected location) + for subdir, ebs in [('', included_ebs), ('generic', included_generic_ebs)]: + pkg = '.'.join(['easybuild', 'easyblocks', subdir]).strip('.') + loc = os.path.join(easyblocks_dir, subdir) + verify_imports([os.path.splitext(eb)[0] for eb in ebs], pkg, loc) + + return easyblocks_path + + +def include_module_naming_schemes(tmpdir, paths): + """Include module naming schemes at specified locations.""" + mns_path = os.path.join(tmpdir, 'included-module-naming-schemes') + + set_up_eb_package(mns_path, 'easybuild.tools.module_naming_scheme') + + mns_dir = os.path.join(mns_path, 'easybuild', 'tools', 'module_naming_scheme') + + allpaths = expand_glob_paths(paths) + for mns_module in allpaths: + filename = os.path.basename(mns_module) + target_path = os.path.join(mns_dir, filename) + symlink(mns_module, target_path) + + included_mns = [x for x in os.listdir(mns_dir) if x not in ['__init__.py']] + _log.debug("Included module naming schemes: %s", included_mns) + + # inject path into Python search path, and reload modules to get it 'registered' in sys.modules + sys.path.insert(0, mns_path) + reload(easybuild.tools.module_naming_scheme) + + # sanity check: verify that included module naming schemes can be imported (from expected location) + verify_imports([os.path.splitext(mns)[0] for mns in included_mns], 'easybuild.tools.module_naming_scheme', mns_dir) + + return mns_path + + +def include_toolchains(tmpdir, paths): + """Include toolchains and toolchain components at specified locations.""" + toolchains_path = os.path.join(tmpdir, 'included-toolchains') + toolchain_subpkgs = ['compiler', 'fft', 'linalg', 'mpi'] + + set_up_eb_package(toolchains_path, 'easybuild.toolchains', subpkgs=toolchain_subpkgs) + + tcs_dir = os.path.join(toolchains_path, 'easybuild', 'toolchains') + + allpaths = expand_glob_paths(paths) + for toolchain_module in allpaths: + filename = os.path.basename(toolchain_module) + + parent_dir = os.path.basename(os.path.dirname(toolchain_module)) + + # toolchain components are expected to be in a directory named according to the type of component + if parent_dir in toolchain_subpkgs: + target_path = os.path.join(tcs_dir, parent_dir, filename) + else: + target_path = os.path.join(tcs_dir, filename) + + symlink(toolchain_module, target_path) + + included_toolchains = [x for x in os.listdir(tcs_dir) if x not in ['__init__.py'] + toolchain_subpkgs] + _log.debug("Included toolchains: %s", included_toolchains) + + included_subpkg_modules = {} + for subpkg in toolchain_subpkgs: + included_subpkg_modules[subpkg] = [x for x in os.listdir(os.path.join(tcs_dir, subpkg)) if x != '__init__.py'] + _log.debug("Included toolchain %s components: %s", subpkg, included_subpkg_modules[subpkg]) + + # inject path into Python search path, and reload modules to get it 'registered' in sys.modules + sys.path.insert(0, toolchains_path) + reload(easybuild.toolchains) + for subpkg in toolchain_subpkgs: + reload(sys.modules['easybuild.toolchains.%s' % subpkg]) + + # sanity check: verify that included toolchain modules can be imported (from expected location) + verify_imports([os.path.splitext(mns)[0] for mns in included_toolchains], 'easybuild.toolchains', tcs_dir) + for subpkg in toolchain_subpkgs: + pkg = '.'.join(['easybuild', 'toolchains', subpkg]) + loc = os.path.join(tcs_dir, subpkg) + verify_imports([os.path.splitext(tcmod)[0] for tcmod in included_subpkg_modules[subpkg]], pkg, loc) + + return toolchains_path diff --git a/easybuild/tools/jenkins.py b/easybuild/tools/jenkins.py index fe8e9cf242..0cc0ee4d1a 100644 --- a/easybuild/tools/jenkins.py +++ b/easybuild/tools/jenkins.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,6 +34,7 @@ from datetime import datetime from vsc.utils import fancylogger +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.version import FRAMEWORK_VERSION, EASYBLOCKS_VERSION @@ -108,7 +109,7 @@ def create_success(name, stats): root.writexml(output_file) output_file.close() except IOError, err: - _log.error("Failed to write out XML file %s: %s" % (filename, err)) + raise EasyBuildError("Failed to write out XML file %s: %s", filename, err) def aggregate_xml_in_dirs(base_dir, output_filename): @@ -142,14 +143,14 @@ def aggregate_xml_in_dirs(base_dir, output_filename): total = 0 for d in dirs: - xml_file = glob.glob(os.path.join(d, "*.xml")) + xml_file = sorted(glob.glob(os.path.join(d, "*.xml"))) if xml_file: # take the first one (should be only one present) xml_file = xml_file[0] try: dom = xml.parse(xml_file) except IOError, err: - _log.error("Failed to read/parse XML file %s: %s" % (xml_file, err)) + raise EasyBuildError("Failed to read/parse XML file %s: %s", xml_file, err) # only one should be present, we are just discarding the rest testcase = dom.getElementsByTagName("testcase")[0] root.firstChild.appendChild(testcase) @@ -165,6 +166,6 @@ def aggregate_xml_in_dirs(base_dir, output_filename): root.writexml(output_file, addindent="\t", newl="\n") output_file.close() except IOError, err: - _log.error("Failed to write out XML file %s: %s" % (output_filename, err)) + raise EasyBuildError("Failed to write out XML file %s: %s", output_filename, err) print "Aggregate regtest results written to %s" % output_filename diff --git a/easybuild/tools/job/__init__.py b/easybuild/tools/job/__init__.py new file mode 100644 index 0000000000..ce6a4a99ce --- /dev/null +++ b/easybuild/tools/job/__init__.py @@ -0,0 +1,34 @@ +## +# Copyright 2011-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Declares easybuild.tools.job namespace, in an extendable way. + +@author: Jens Timmerman (Ghent University) +@author: Kenneth Hoste (Ghent University) +""" +from pkgutil import extend_path + +# we're not the only ones in this namespace +__path__ = extend_path(__path__, __name__) #@ReservedAssignment diff --git a/easybuild/tools/job/backend.py b/easybuild/tools/job/backend.py new file mode 100644 index 0000000000..ba381005a4 --- /dev/null +++ b/easybuild/tools/job/backend.py @@ -0,0 +1,119 @@ +## +# Copyright 2015-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Abstract interface for submitting jobs and related utilities. + +@author: Riccardo Murri (University of Zurich) +@author: Kenneth Hoste (Ghent University) +""" + +from abc import ABCMeta, abstractmethod + +from vsc.utils import fancylogger +from vsc.utils.missing import get_subclasses + +from easybuild.tools.config import get_job_backend +from easybuild.tools.utilities import import_available_modules + + +class JobBackend(object): + __metaclass__ = ABCMeta + + def __init__(self): + """Constructor.""" + self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) + self._check_version() + + @abstractmethod + def _check_version(self): + """Check whether version of backend complies with required version.""" + pass + + @abstractmethod + def init(self): + """ + Initialise the job backend, to start a bulk job submission. + + Jobs may be queued and only actually submitted when `complete()` + is called. + """ + pass + + @abstractmethod + def make_job(self, script, name, env_vars=None, hours=None, cores=None): + """ + Create and return a `Job` object with the given parameters. + + See the `Job`:class: constructor for an explanation of what + the arguments are. + """ + pass + + @abstractmethod + def queue(self, job, dependencies=frozenset()): + """ + Add a job to the queue. + + If second optional argument `dependencies` is given, it must be a + sequence of jobs that must be successfully terminated before + the new job can run. + + Note that actual submission may be delayed until `complete()` is + called. + """ + pass + + @abstractmethod + def complete(self): + """ + Complete a bulk job submission. + + Releases any jobs that were possibly queued since the last + `init()` call. + + No more job submissions should be attempted after `complete()` + has been called, until a `init()` is invoked again. + """ + pass + + +def avail_job_backends(check_usable=True): + """ + Return all known job execution backends. + """ + import_available_modules('easybuild.tools.job') + class_dict = dict([(x.__name__, x) for x in get_subclasses(JobBackend)]) + return class_dict + + +def job_backend(): + """ + Return interface to job server, or `None` if none is available. + """ + job_backend = get_job_backend() + if job_backend is None: + return None + job_backend_class = avail_job_backends().get(job_backend) + return job_backend_class() diff --git a/easybuild/tools/job/gc3pie.py b/easybuild/tools/job/gc3pie.py new file mode 100644 index 0000000000..833fb9beb4 --- /dev/null +++ b/easybuild/tools/job/gc3pie.py @@ -0,0 +1,269 @@ +## +# Copyright 2015-2015 Ghent University +# Copyright 2015 S3IT, University of Zurich +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Interface for submitting jobs via GC3Pie. + +@author: Riccardo Murri (University of Zurich) +@author: Kenneth Hoste (Ghent University) +""" +from distutils.version import LooseVersion +from time import gmtime, strftime +import re +import time + +from vsc.utils import fancylogger + +from easybuild.tools.build_log import EasyBuildError, print_msg +from easybuild.tools.config import build_option +from easybuild.tools.job.backend import JobBackend +from easybuild.tools.utilities import only_if_module_is_available + + +_log = fancylogger.getLogger('gc3pie', fname=False) + + +try: + import gc3libs + import gc3libs.exceptions + from gc3libs import Application, Run, create_engine + from gc3libs.core import Engine + from gc3libs.quantity import hours as hr + from gc3libs.workflow import DependentTaskCollection + + # inject EasyBuild logger into GC3Pie + gc3libs.log = fancylogger.getLogger('gc3pie', fname=False) + # make handling of log.error compatible with stdlib logging + gc3libs.log.raiseError = False + + # instruct GC3Pie to not ignore errors, but raise exceptions instead + gc3libs.UNIGNORE_ALL_ERRORS = True + +except ImportError as err: + _log.debug("Failed to import gc3libs from GC3Pie." + " Silently ignoring, this is a real issue only when GC3Pie is used as backend for --job") + + +# eb --job --job-backend=GC3Pie +class GC3Pie(JobBackend): + """ + Use the GC3Pie framework to submit and monitor compilation jobs, + see http://gc3pie.readthedocs.org/. + + In contrast with accessing an external service, GC3Pie implements + its own workflow manager, which means ``eb --job + --job-backend=GC3Pie`` will keep running until all jobs have + terminated. + """ + + REQ_VERSION = '2.4.0' + DEVELOPMENT_VERSION = 'development' # 'magic' version string indicated non-released version + REQ_SVN_REVISION = 4287 # use integer value, not a string! + + @only_if_module_is_available('gc3libs', pkgname='gc3pie') + def __init__(self, *args, **kwargs): + """GC3Pie constructor.""" + super(GC3Pie, self).__init__(*args, **kwargs) + + # _check_version is called by __init__, so guard it (too) with the decorator + @only_if_module_is_available('gc3libs', pkgname='gc3pie') + def _check_version(self): + """Check whether GC3Pie version complies with required version.""" + # location of __version__ to use may change, depending on the minimal required SVN revision for development versions + version_str = gc3libs.core.__version__ + + version_regex = re.compile(r'^(?P\S*) version \(SVN \$Revision: (?P\d+)\s*\$\)') + res = version_regex.search(version_str) + if res: + version = res.group('version') + svn_rev = int(res.group('svn_rev')) + self.log.debug("Parsed GC3Pie version info: '%s' (SVN rev: '%s')", version, svn_rev) + + if version == self.DEVELOPMENT_VERSION: + # fall back to checking SVN revision for development versions + if svn_rev < self.REQ_SVN_REVISION: + raise EasyBuildError("Found GC3Pie SVN revision %d, but revision %d or newer is required", + svn_rev, self.REQ_SVN_REVISION) + else: + if LooseVersion(version) < LooseVersion(self.REQ_VERSION): + raise EasyBuildError("Found GC3Pie version %s, but version %s or more recent is required", + version, self.REQ_VERSION) + else: + raise EasyBuildError("Failed to parse GC3Pie version string '%s' using pattern %s", + version_str, version_regex.pattern) + + def init(self): + """ + Initialise the GC3Pie job backend. + """ + # List of config files for GC3Pie; non-existing ones will be + # silently ignored. The list here copies GC3Pie's default, + # for the principle of minimal surprise, but there is no + # strict requirement that this be done and EB could actually + # choose to use a completely distinct set of conf. files. + self.config_files = gc3libs.Default.CONFIG_FILE_LOCATIONS[:] + cfgfile = build_option('job_backend_config') + if cfgfile: + self.config_files.append(cfgfile) + + self.output_dir = build_option('job_output_dir') + self.jobs = DependentTaskCollection(output_dir=self.output_dir) + self.job_cnt = 0 + + # after polling for job status, sleep for this time duration + # before polling again (in seconds) + self.poll_interval = build_option('job_polling_interval') + + def make_job(self, script, name, env_vars=None, hours=None, cores=None): + """ + Create and return a job object with the given parameters. + + First argument `server` is an instance of the corresponding + `JobBackend` class, i.e., a `GC3Pie`:class: instance in this case. + + Second argument `script` is the content of the job script + itself, i.e., the sequence of shell commands that will be + executed. + + Third argument `name` sets the job human-readable name. + + Fourth (optional) argument `env_vars` is a dictionary with + key-value pairs of environment variables that should be passed + on to the job. + + Fifth and sixth (optional) arguments `hours` and `cores` should be + integer values: + * hours must be in the range 1 .. MAX_WALLTIME; + * cores depends on which cluster the job is being run. + """ + named_args = { + 'jobname': name, # job name in GC3Pie + 'name': name, # job name in EasyBuild + } + + # environment + if env_vars: + named_args['environment'] = env_vars + + # input/output files for job (none) + named_args['inputs'] = [] + named_args['outputs'] = [] + + # job logs + named_args.update({ + # join stdout/stderr in a single log + 'join': True, + # location for log file + 'output_dir': self.output_dir, + # log file name (including timestamp to try and ensure unique filename) + 'stdout': 'eb-%s-gc3pie-job-%s.log' % (name, strftime("%Y%M%d-UTC-%H-%M-%S", gmtime())) + }) + + # walltime + max_walltime = build_option('job_max_walltime') + if hours is None: + hours = max_walltime + if hours > max_walltime: + self.log.warn("Specified %s hours, but this is impossible. (resetting to %s hours)" % (hours, max_walltime)) + hours = max_walltime + named_args['requested_walltime'] = hours * hr + + if cores: + named_args['requested_cores'] = cores + elif build_option('job_cores'): + named_args['requested_cores'] = build_option('job_cores') + else: + self.log.warn("Number of cores to request not specified, falling back to whatever GC3Pie does by default") + + return Application(['/bin/sh', '-c', script], **named_args) + + def queue(self, job, dependencies=frozenset()): + """ + Add a job to the queue, optionally specifying dependencies. + + @param dependencies: jobs on which this job depends. + """ + self.jobs.add(job, dependencies) + # since it's not trivial to determine the correct job count from self.jobs, we keep track of a count ourselves + self.job_cnt += 1 + + def complete(self): + """ + Complete a bulk job submission. + + Create engine, and progress it until all jobs have terminated. + """ + # create an instance of `Engine` using the list of configuration files + try: + self._engine = create_engine(*self.config_files, resource_errors_are_fatal=True) + + except gc3libs.exceptions.Error as err: + raise EasyBuildError("Failed to create GC3Pie engine: %s", err) + + # make sure that all job log files end up in the same directory, rather than renaming the output directory + # see https://gc3pie.readthedocs.org/en/latest/programmers/api/gc3libs/core.html#gc3libs.core.Engine + self._engine.retrieve_overwrites = True + + # Add your application to the engine. This will NOT submit + # your application yet, but will make the engine *aware* of + # the application. + self._engine.add(self.jobs) + + # in case you want to select a specific resource, call + target_resource = build_option('job_target_resource') + if target_resource: + res = self._engine.select_resource(target_resource) + if res == 0: + raise EasyBuildError("Failed to select target resource '%s' in GC3Pie", target_resource) + + # Periodically check the status of your application. + while self.jobs.execution.state != Run.State.TERMINATED: + # `Engine.progress()` will do the GC3Pie magic: + # submit new jobs, update status of submitted jobs, get + # results of terminating jobs etc... + self._engine.progress() + + # report progress + self._print_status_report() + + # Wait a few seconds... + time.sleep(self.poll_interval) + + # final status report + print_msg("Done processing jobs", log=self.log, silent=build_option('silent')) + self._print_status_report() + + def _print_status_report(self): + """ + Print a job status report to STDOUT and the log file. + + The number of jobs in each state is reported; the + figures are extracted from the `stats()` method of the + currently-running GC3Pie engine. + """ + stats = self._engine.stats(only=Application) + states = ', '.join(["%d %s" % (stats[s], s.lower()) for s in stats if s != 'total' and stats[s]]) + print_msg("GC3Pie job overview: %s (total: %s)" % (states, self.job_cnt), + log=self.log, silent=build_option('silent')) diff --git a/easybuild/tools/pbs_job.py b/easybuild/tools/job/pbs_python.py similarity index 54% rename from easybuild/tools/pbs_job.py rename to easybuild/tools/job/pbs_python.py index 930c059af9..2e581a9845 100644 --- a/easybuild/tools/pbs_job.py +++ b/easybuild/tools/job/pbs_python.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,85 +29,176 @@ @author: Toon Willems (Ghent University) @author: Kenneth Hoste (Ghent University) """ - +from distutils.version import LooseVersion import os +import re import tempfile -import time from vsc.utils import fancylogger +from easybuild.tools.build_log import EasyBuildError, print_msg +from easybuild.tools.config import build_option +from easybuild.tools.job.backend import JobBackend +from easybuild.tools.utilities import only_if_module_is_available + -_log = fancylogger.getLogger('pbs_job', fname=False) +_log = fancylogger.getLogger('pbs_python', fname=False) -MAX_WALLTIME = 72 # extend paramater should be 'NULL' in some functions because this is required by the python api NULL = 'NULL' # list of known hold types KNOWN_HOLD_TYPES = [] -pbs_import_failed = None try: - from PBSQuery import PBSQuery import pbs + from PBSQuery import PBSQuery KNOWN_HOLD_TYPES = [pbs.USER_HOLD, pbs.OTHER_HOLD, pbs.SYSTEM_HOLD] -except ImportError: - _log.debug("Failed to import pbs from pbs_python. Silently ignoring, is only a real issue with --job") - pbs_import_failed = ("PBSQuery or pbs modules not available. " - "Please make sure pbs_python is installed and usable.") +except ImportError as err: + _log.debug("Failed to import pbs/PBSQuery from pbs_python." + " Silently ignoring, this is a real issue only when pbs_python is used as backend for --job") + + +class PbsPython(JobBackend): + """ + Manage PBS server communication and create `PbsJob` objects. + """ + + # pbs_python 4.1.0 introduces the pbs.version variable we rely on + REQ_VERSION = '4.1.0' + + @only_if_module_is_available('pbs', pkgname='pbs_python') + def __init__(self, *args, **kwargs): + """PbsPython constructor.""" + super(PbsPython, self).__init__(*args, **kwargs) + + # _check_version is called by __init__, so guard it (too) with the decorator + @only_if_module_is_available('pbs', pkgname='pbs_python') + def _check_version(self): + """Check whether pbs_python version complies with required version.""" + version_regex = re.compile('pbs_python version (?P.*)') + res = version_regex.search(pbs.version) + if res: + version = res.group('version') + if LooseVersion(version) < LooseVersion(self.REQ_VERSION): + raise EasyBuildError("Found pbs_python version %s, but version %s or more recent is required", + version, self.REQ_VERSION) + else: + raise EasyBuildError("Failed to parse pbs_python version string '%s' using pattern %s", + pbs.version, version_regex.pattern) + + def __init__(self, *args, **kwargs): + """Constructor.""" + pbs_server = kwargs.pop('pbs_server', None) + + super(PbsPython, self).__init__(*args, **kwargs) + + self.pbs_server = pbs_server or build_option('job_target_resource') or pbs.pbs_default() + self.conn = None + self._ppn = None + + def init(self): + """ + Initialise the job backend. + + Connect to the PBS server & reset list of submitted jobs. + """ + self.connect_to_server() + self._submitted = [] + + def connect_to_server(self): + """Connect to PBS server, set and return connection.""" + if not self.conn: + self.conn = pbs.pbs_connect(self.pbs_server) + return self.conn + + def queue(self, job, dependencies=frozenset()): + """ + Add a job to the queue. + + @param dependencies: jobs on which this job depends. + """ + if dependencies: + job.add_dependencies(dependencies) + job._submit() + self._submitted.append(job) + + def complete(self): + """ + Complete a bulk job submission. + + Release all user holds on submitted jobs, and disconnect from server. + """ + for job in self._submitted: + if job.has_holds(): + self.log.info("releasing user hold on job %s" % job.jobid) + job.release_hold() + + self.disconnect_from_server() -def connect_to_server(pbs_server=None): - """Connect to PBS server and return connection.""" - if pbs_import_failed: - _log.error(pbs_import_failed) - return None + # print list of submitted jobs + submitted_jobs = '; '.join(["%s (%s): %s" % (job.name, job.module, job.jobid) for job in self._submitted]) + print_msg("List of submitted jobs (%d): %s" % (len(self._submitted), submitted_jobs), log=self.log) - if not pbs_server: - pbs_server = pbs.pbs_default() - return pbs.pbs_connect(pbs_server) + # determine leaf nodes in dependency graph, and report them + all_deps = set() + for job in self._submitted: + all_deps = all_deps.union(job.deps) + leaf_nodes = [] + for job in self._submitted: + if job.jobid not in all_deps: + leaf_nodes.append(str(job.jobid).split('.')[0]) -def disconnect_from_server(conn): - """Disconnect a given connection.""" - if pbs_import_failed: - _log.error(pbs_import_failed) - return None + self.log.info("Job ids of leaf nodes in dep. graph: %s" % ','.join(leaf_nodes)) - pbs.pbs_disconnect(conn) + def disconnect_from_server(self): + """Disconnect current connection.""" + pbs.pbs_disconnect(self.conn) + self.conn = None + def _get_ppn(self): + """Guess PBS' `ppn` value for a full node.""" + # cache this value as it's not likely going to change over the + # `eb` script runtime ... + if not self._ppn: + pq = PBSQuery() + node_vals = pq.getnodes().values() # only the values, not the names + interesting_nodes = ('free', 'job-exclusive',) + res = {} + for np in [int(x['np'][0]) for x in node_vals if x['state'][0] in interesting_nodes]: + res.setdefault(np, 0) + res[np] += 1 -def get_ppn(): - """Guess the ppn for full node""" + # return most frequent + freq_count, freq_np = max([(j, i) for i, j in res.items()]) + self.log.debug("Found most frequent np %s (%s times) in interesting nodes %s" % (freq_np, freq_count, interesting_nodes)) - log = fancylogger.getLogger('pbs_job.get_ppn') + self._ppn = freq_np - pq = PBSQuery() - node_vals = pq.getnodes().values() # only the values, not the names - interesting_nodes = ('free', 'job-exclusive',) - res = {} - for np in [int(x['np'][0]) for x in node_vals if x['state'][0] in interesting_nodes]: - res.setdefault(np, 0) - res[np] += 1 + return self._ppn - # return most frequent - freq_count, freq_np = max([(j, i) for i, j in res.items()]) - log.debug("Found most frequent np %s (%s times) in interesting nodes %s" % (freq_np, freq_count, interesting_nodes)) + ppn = property(_get_ppn) - return freq_np + def make_job(self, script, name, env_vars=None, hours=None, cores=None): + """Create and return a `PbsJob` object with the given parameters.""" + return PbsJob(self, script, name, env_vars, hours, cores, conn=self.conn, ppn=self.ppn) class PbsJob(object): """Interaction with TORQUE""" - def __init__(self, script, name, env_vars=None, resources={}, conn=None, ppn=None): + def __init__(self, server, script, name, env_vars=None, + hours=None, cores=None, conn=None, ppn=None): """ create a new Job to be submitted to PBS env_vars is a dictionary with key-value pairs of environment variables that should be passed on to the job - resources is a dictionary with optional keys: ['hours', 'cores'] both of these should be integer values. - hours can be 1 - MAX_WALLTIME, cores depends on which cluster it is being run. + hours and cores should be integer values. + hours can be 1 - (max walltime), cores depends on which cluster it is being run. """ - self.clean_conn = True self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) + + self._server = server self.script = script if env_vars: self.env_vars = env_vars.copy() @@ -115,46 +206,38 @@ def __init__(self, script, name, env_vars=None, resources={}, conn=None, ppn=Non self.env_vars = {} self.name = name - if pbs_import_failed: - self.log.error(pbs_import_failed) - try: - self.pbs_server = pbs.pbs_default() - if conn: - self.pbsconn = conn - self.clean_conn = False - else: - self.pbsconn = pbs.pbs_connect(self.pbs_server) + self.pbsconn = self._server.connect_to_server() except Exception, err: - self.log.error("Failed to connect to the default pbs server: %s" % err) + raise EasyBuildError("Failed to connect to the default pbs server: %s", err) # setup the resources requested # validate requested resources! - hours = resources.get('hours', MAX_WALLTIME) - if hours > MAX_WALLTIME: - self.log.warn("Specified %s hours, but this is impossible. (resetting to %s hours)" % (hours, MAX_WALLTIME)) - hours = MAX_WALLTIME + max_walltime = build_option('job_max_walltime') + if hours is None: + hours = max_walltime + if hours > max_walltime: + self.log.warn("Specified %s hours, but this is impossible. (resetting to %s hours)" % (hours, max_walltime)) + hours = max_walltime if ppn is None: - max_cores = get_ppn() + max_cores = server.ppn else: max_cores = ppn - cores = resources.get('cores', max_cores) + if cores is None: + cores = max_cores if cores > max_cores: self.log.warn("number of requested cores (%s) was greater than available (%s) " % (cores, max_cores)) cores = max_cores # only allow cores and hours for now. self.resources = { - "walltime": "%s:00:00" % hours, - "nodes": "1:ppn=%s" % cores - } - # set queue based on the hours requested - if hours >= 12: - self.queue = 'long' - else: - self.queue = 'short' + 'walltime': '%s:00:00' % hours, + 'nodes': '1:ppn=%s' % cores, + } + # don't specify any queue name to submit to, use the default + self.queue = None # job id of this job self.jobid = None # list of dependencies for this job @@ -162,53 +245,61 @@ def __init__(self, script, name, env_vars=None, resources={}, conn=None, ppn=Non # list of holds that are placed on this job self.holds = [] - def add_dependencies(self, job_ids): + def __str__(self): + """Return the job ID as a string.""" + return (str(self.jobid) if self.jobid is not None + else repr(self)) + + def add_dependencies(self, jobs): """ Add dependencies to this job. - job_ids is an array of job ids (e.g.: 8453.master2.gengar....) - if only one job_id is provided this function will also work - """ - if isinstance(job_ids, str): - job_ids = list(job_ids) - self.deps.extend(job_ids) + Argument `jobs` is a sequence of `PbsJob` objects. + """ + self.deps.extend(jobs) - def submit(self, with_hold=False): + def _submit(self): """Submit the jobscript txt, set self.jobid""" txt = self.script self.log.debug("Going to submit script %s" % txt) # Build default pbs_attributes list - pbs_attributes = pbs.new_attropl(1) + pbs_attributes = pbs.new_attropl(3) pbs_attributes[0].name = pbs.ATTR_N # Job_Name pbs_attributes[0].value = self.name + output_dir = build_option('job_output_dir') + pbs_attributes[1].name = pbs.ATTR_o + pbs_attributes[1].value = os.path.join(output_dir, '%s.o$PBS_JOBID' % self.name) + + pbs_attributes[2].name = pbs.ATTR_e + pbs_attributes[2].value = os.path.join(output_dir, '%s.e$PBS_JOBID' % self.name) + # set resource requirements - resourse_attributes = pbs.new_attropl(len(self.resources)) + resource_attributes = pbs.new_attropl(len(self.resources)) idx = 0 for k, v in self.resources.items(): - resourse_attributes[idx].name = pbs.ATTR_l # Resource_List - resourse_attributes[idx].resource = k - resourse_attributes[idx].value = v + resource_attributes[idx].name = pbs.ATTR_l # Resource_List + resource_attributes[idx].resource = k + resource_attributes[idx].value = v idx += 1 - pbs_attributes.extend(resourse_attributes) + pbs_attributes.extend(resource_attributes) # add job dependencies to attributes if self.deps: deps_attributes = pbs.new_attropl(1) deps_attributes[0].name = pbs.ATTR_depend - deps_attributes[0].value = ",".join(["afterany:%s" % dep for dep in self.deps]) + deps_attributes[0].value = ",".join(["afterany:%s" % dep.jobid for dep in self.deps]) pbs_attributes.extend(deps_attributes) self.log.debug("Job deps attributes: %s" % deps_attributes[0].value) - # submit job with (user) hold if requested - if with_hold: - hold_attributes = pbs.new_attropl(1) - hold_attributes[0].name = pbs.ATTR_h - hold_attributes[0].value = pbs.USER_HOLD - pbs_attributes.extend(hold_attributes) - self.holds.append(pbs.USER_HOLD) - self.log.debug("Job hold attributes: %s" % hold_attributes[0].value) + # submit job with (user) hold + hold_attributes = pbs.new_attropl(1) + hold_attributes[0].name = pbs.ATTR_h + hold_attributes[0].value = pbs.USER_HOLD + pbs_attributes.extend(hold_attributes) + self.holds.append(pbs.USER_HOLD) + self.log.debug("Job hold attributes: %s" % hold_attributes[0].value) # add a bunch of variables (added by qsub) # also set PBS_O_WORKDIR to os.getcwd() @@ -245,7 +336,7 @@ def submit(self, with_hold=False): jobid = pbs.pbs_submit(self.pbsconn, pbs_attributes, scriptfn, self.queue, NULL) is_error, errormsg = pbs.error() if is_error or jobid is None: - self.log.error("Failed to submit job script %s (job id: %s, error %s)" % (scriptfn, jobid, errormsg)) + raise EasyBuildError("Failed to submit job script %s (job id: %s, error %s)", scriptfn, jobid, errormsg) else: self.log.debug("Succesful job submission returned jobid %s" % jobid) self.jobid = jobid @@ -260,13 +351,13 @@ def set_hold(self, hold_type=None): # only set hold if it wasn't set before if hold_type not in self.holds: if hold_type not in KNOWN_HOLD_TYPES: - self.log.error("set_hold: unknown hold type: %s (supported: %s)" % (hold_type, KNOWN_HOLD_TYPES)) + raise EasyBuildError("set_hold: unknown hold type: %s (supported: %s)", hold_type, KNOWN_HOLD_TYPES) # set hold, check for errors, and keep track of this hold ec = pbs.pbs_holdjob(self.pbsconn, self.jobid, hold_type, NULL) is_error, errormsg = pbs.error() if is_error or ec: - tup = (hold_type, self.jobid, is_error, ec, errormsg) - self.log.error("Failed to set hold of type %s on job %s (is_error: %s, exit code: %s, msg: %s)" % tup) + raise EasyBuildError("Failed to set hold of type %s on job %s (is_error: %s, exit code: %s, msg: %s)", + hold_type, self.jobid, is_error, ec, errormsg) else: self.holds.append(hold_type) else: @@ -281,14 +372,14 @@ def release_hold(self, hold_type=None): # only release hold if it was set if hold_type in self.holds: if hold_type not in KNOWN_HOLD_TYPES: - self.log.error("release_hold: unknown hold type: %s (supported: %s)" % (hold_type, KNOWN_HOLD_TYPES)) + raise EasyBuildError("release_hold: unknown hold type: %s (supported: %s)", hold_type, KNOWN_HOLD_TYPES) # release hold, check for errors, remove from list of holds ec = pbs.pbs_rlsjob(self.pbsconn, self.jobid, hold_type, NULL) self.log.debug("Released hold of type %s for job %s" % (hold_type, self.jobid)) is_error, errormsg = pbs.error() if is_error or ec: - tup = (hold_type, self.jobid, is_error, ec, errormsg) - self.log.error("Failed to release hold type %s on job %s (is_error: %s, exit code: %s, msg: %s)" % tup) + raise EasyBuildError("Failed to release hold type %s on job %s (is_error: %s, exit code: %s, msg: %s)", + hold_type, self.jobid, is_error, ec, errormsg) else: self.holds.remove(hold_type) else: @@ -360,11 +451,6 @@ def info(self, types=None): for idx, attr in enumerate(types): jobattr[idx].name = attr - - # get a new connection (otherwise this seems to fail) - if self.clean_conn: - pbs.pbs_disconnect(self.pbsconn) - self.pbsconn = pbs.pbs_connect(self.pbs_server) jobs = pbs.pbs_statjob(self.pbsconn, self.jobid, jobattr, NULL) if len(jobs) == 0: # no job found, return None info @@ -374,7 +460,7 @@ def info(self, types=None): elif len(jobs) == 1: self.log.debug("Request for jobid %s returned one result %s" % (self.jobid, jobs)) else: - self.log.error("Request for jobid %s returned more then one result %s" % (self.jobid, jobs)) + raise EasyBuildError("Request for jobid %s returned more then one result %s", self.jobid, jobs) # only expect to have a list with one element j = jobs[0] @@ -389,12 +475,6 @@ def remove(self): """Remove the job with id jobid""" result = pbs.pbs_deljob(self.pbsconn, self.jobid, '') # use empty string, not NULL if result: - self.log.error("Failed to delete job %s: error %s" % (self.jobid, result)) + raise EasyBuildError("Failed to delete job %s: error %s", self.jobid, result) else: self.log.debug("Succesfully deleted job %s" % self.jobid) - - def cleanup(self): - """Cleanup: disconnect from server.""" - if self.clean_conn: - self.log.debug("Disconnecting from server.") - pbs.pbs_disconnect(self.pbsconn) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 525a43d508..5958508430 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,16 +30,19 @@ @author: Kenneth Hoste (Ghent University) @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) -@author: Fotis Georgatos (Uni.Lu) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ import os +import re +import sys import tempfile from vsc.utils import fancylogger +from vsc.utils.missing import get_subclasses -from easybuild.framework.easyconfig.easyconfig import ActiveMNS -from easybuild.tools import config -from easybuild.tools.config import build_option -from easybuild.tools.filetools import mkdir +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option, get_module_syntax, install_path +from easybuild.tools.filetools import mkdir, read_file +from easybuild.tools.modules import modules_tool from easybuild.tools.utilities import quote_str @@ -50,45 +53,108 @@ class ModuleGenerator(object): """ Class for generating module files. """ + SYNTAX = None + + # chars we want to escape in the generated modulefiles + CHARS_TO_ESCAPE = None + + MODULE_FILE_EXTENSION = None + MODULE_HEADER = None + def __init__(self, application, fake=False): + """ModuleGenerator constructor.""" self.app = application - self.fake = fake - self.tmpdir = None - self.filename = None - self.class_mod_file = None - self.module_path = None + self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) + self.fake_mod_path = tempfile.mkdtemp() + + def get_modules_path(self, fake=False, mod_path_suffix=None): + """Return path to directory where module files should be generated in.""" + mod_path = install_path('mod') + if fake: + self.log.debug("Fake mode: using %s (instead of %s)" % (self.fake_mod_path, mod_path)) + mod_path = self.fake_mod_path - def prepare(self): + if mod_path_suffix is None: + mod_path_suffix = build_option('suffix_modules_path') + + return os.path.join(mod_path, mod_path_suffix) + + def get_module_filepath(self, fake=False, mod_path_suffix=None): + """Return path to module file.""" + mod_path = self.get_modules_path(fake=fake, mod_path_suffix=mod_path_suffix) + full_mod_name = self.app.full_mod_name + self.MODULE_FILE_EXTENSION + return os.path.join(mod_path, full_mod_name) + + def prepare(self, fake=False): """ - Creates the absolute filename for the module. + Prepare for generating module file: Creates the absolute filename for the module. """ - mod_path_suffix = build_option('suffix_modules_path') + mod_path = self.get_modules_path(fake=fake) # module file goes in general moduleclass category - self.filename = os.path.join(self.module_path, mod_path_suffix, self.app.full_mod_name) # make symlink in moduleclass category - mod_symlink_paths = ActiveMNS().det_module_symlink_paths(self.app.cfg) - self.class_mod_files = [os.path.join(self.module_path, p, self.app.full_mod_name) for p in mod_symlink_paths] - # create directories and links - for path in [os.path.dirname(x) for x in [self.filename] + self.class_mod_files]: - mkdir(path, parents=True) + mod_filepath = self.get_module_filepath(fake=fake) + mkdir(os.path.dirname(mod_filepath), parents=True) - # remove module file if it's there (it'll be recreated), see Application.make_module - if os.path.exists(self.filename): - os.remove(self.filename) + # remove module file if it's there (it'll be recreated), see EasyBlock.make_module + if os.path.exists(mod_filepath): + self.log.debug("Removing existing module file %s", mod_filepath) + os.remove(mod_filepath) - return os.path.join(self.module_path, mod_path_suffix) + return mod_path - def create_symlinks(self): + def create_symlinks(self, mod_symlink_paths, fake=False): """Create moduleclass symlink(s) to actual module file.""" + mod_filepath = self.get_module_filepath(fake=fake) + class_mod_files = [self.get_module_filepath(fake=fake, mod_path_suffix=p) for p in mod_symlink_paths] try: - # remove symlink if its there (even if it's broken) - for class_mod_file in self.class_mod_files: + for class_mod_file in class_mod_files: + # remove symlink if its there (even if it's broken) if os.path.lexists(class_mod_file): + self.log.debug("Removing existing symlink %s", class_mod_file) os.remove(class_mod_file) - os.symlink(self.filename, class_mod_file) + + mkdir(os.path.dirname(class_mod_file), parents=True) + os.symlink(mod_filepath, class_mod_file) + except OSError, err: - _log.error("Failed to create symlinks from %s to %s: %s" % (self.class_mod_files, self.filename, err)) + raise EasyBuildError("Failed to create symlinks from %s to %s: %s", class_mod_files, mod_filepath, err) + + def comment(self, msg): + """Return given string formatted as a comment.""" + raise NotImplementedError + + def conditional_statement(self, condition, body, negative=False): + """Return formatted conditional statement, with given condition and body.""" + raise NotImplementedError + + +class ModuleGeneratorTcl(ModuleGenerator): + """ + Class for generating Tcl module files. + """ + SYNTAX = 'Tcl' + MODULE_FILE_EXTENSION = '' # no suffix for Tcl module files + MODULE_HEADER = '#%Module' + CHARS_TO_ESCAPE = ['$'] + + LOAD_REGEX = r"^\s*module\s+load\s+(\S+)" + LOAD_TEMPLATE = "module load %(mod_name)s" + + def comment(self, msg): + """Return string containing given message as a comment.""" + return "# %s\n" % msg + + def conditional_statement(self, condition, body, negative=False): + """Return formatted conditional statement, with given condition and body.""" + if negative: + lines = ["if { ![ %s ] } {" % condition] + else: + lines = ["if { [ %s ] } {" % condition] + + lines.append(' ' + body) + lines.extend(['}', '']) + return '\n'.join(lines) def get_description(self, conflict=True): """ @@ -97,33 +163,30 @@ def get_description(self, conflict=True): description = "%s - Homepage: %s" % (self.app.cfg['description'], self.app.cfg['homepage']) lines = [ - "#%%Module", # double % to escape string formatting! - "", + self.MODULE_HEADER.replace('%', '%%'), "proc ModulesHelp { } {", - " puts stderr { %(description)s", + " puts stderr { %(description)s", " }", - "}", - "", + '}', + '', "module-whatis {Description: %(description)s}", - "", - "set root %(installdir)s", - "", + '', + "set root %(installdir)s", ] if self.app.cfg['moduleloadnoconflict']: - lines.extend([ - "if { ![is-loaded %(name)s/%(version)s] } {", - " if { [is-loaded %(name)s] } {", - " module unload %(name)s", - " }", - "}", - "", - ]) + cond_unload = self.conditional_statement("is-loaded %(name)s", "module unload %(name)s") + lines.extend(['', self.conditional_statement("is-loaded %(name)s/%(version)s", cond_unload, negative=True)]) elif conflict: - lines.append("conflict %s\n" % self.app.name) + # conflict on 'name' part of module name (excluding version part at the end) + # examples: + # - 'conflict GCC' for 'GCC/4.8.3' + # - 'conflict Core/GCC' for 'Core/GCC/4.8.2' + # - 'conflict Compiler/GCC/4.8.2/OpenMPI' for 'Compiler/GCC/4.8.2/OpenMPI/1.6.4' + lines.extend(['', "conflict %s" % os.path.dirname(self.app.short_mod_name)]) - txt = '\n'.join(lines) % { + txt = '\n'.join(lines + ['']) % { 'name': self.app.name, 'version': self.app.version, 'description': description, @@ -136,50 +199,55 @@ def load_module(self, mod_name, recursive_unload=False): """ Generate load statements for module. """ - if recursive_unload: + if build_option('recursive_mod_unload') or recursive_unload: # not wrapping the 'module load' with an is-loaded guard ensures recursive unloading; - # when "module unload" is called on the module in which the depedency "module load" is present, + # when "module unload" is called on the module in which the dependency "module load" is present, # it will get translated to "module unload" - load_statement = ["module load %(mod_name)s"] + load_statement = [self.LOAD_TEMPLATE, ''] else: - load_statement = [ - "if { ![is-loaded %(mod_name)s] } {", - " module load %(mod_name)s", - "}", - ] - return '\n'.join([""] + load_statement + [""]) % {'mod_name': mod_name} + load_statement = [self.conditional_statement("is-loaded %(mod_name)s", self.LOAD_TEMPLATE, negative=True)] + return '\n'.join([''] + load_statement) % {'mod_name': mod_name} def unload_module(self, mod_name): """ Generate unload statements for module. """ - return '\n'.join([ - "", - "if { [is-loaded %(mod_name)s] } {", - " module unload %(mod_name)s", - "}", - "", - ]) % {'mod_name': mod_name} + cond_unload = self.conditional_statement("is-loaded %(mod)s", "module unload %(mod)s") % {'mod': mod_name} + return '\n'.join(['', cond_unload]) - def prepend_paths(self, key, paths, allow_abs=False): + def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True): """ Generate prepend-path statements for the given list of paths. + + @param key: environment variable to prepend paths to + @param paths: list of paths to prepend + @param allow_abs: allow providing of absolute paths + @param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir) """ template = "prepend-path\t%s\t\t%s\n" if isinstance(paths, basestring): - _log.info("Wrapping %s into a list before using it to prepend path %s" % (paths, key)) + self.log.debug("Wrapping %s into a list before using it to prepend path %s" % (paths, key)) paths = [paths] - # make sure only relative paths are passed - for i in xrange(len(paths)): - if os.path.isabs(paths[i]) and not allow_abs: - _log.error("Absolute path %s passed to prepend_paths which only expects relative paths." % paths[i]) - elif not os.path.isabs(paths[i]): - # prepend $root (= installdir) for relative paths - paths[i] = "$root/%s" % paths[i] + abspaths = [] + for path in paths: + if os.path.isabs(path) and not allow_abs: + raise EasyBuildError("Absolute path %s passed to prepend_paths which only expects relative paths.", + path) + elif not os.path.isabs(path): + # prepend $root (= installdir) for (non-empty) relative paths + if path: + if expand_relpaths: + abspaths.append(os.path.join('$root', path)) + else: + abspaths.append(path) + else: + abspaths.append('$root') + else: + abspaths.append(path) - statements = [template % (key, p) for p in paths] + statements = [template % (key, p) for p in abspaths] return ''.join(statements) def use(self, paths): @@ -188,34 +256,31 @@ def use(self, paths): """ use_statements = [] for path in paths: - use_statements.append("module use %s" % path) - return '\n'.join(use_statements) + use_statements.append("module use %s\n" % path) + return ''.join(use_statements) - def set_environment(self, key, value): + def set_environment(self, key, value, relpath=False): """ Generate setenv statement for the given key/value pair. """ # quotes are needed, to ensure smooth working of EBDEVEL* modulefiles - return 'setenv\t%s\t\t%s\n' % (key, quote_str(value)) - + if relpath: + if value: + val = quote_str(os.path.join('$root', value)) + else: + val = '"$root"' + else: + val = quote_str(value) + return 'setenv\t%s\t\t%s\n' % (key, val) + def msg_on_load(self, msg): """ Add a message that should be printed when loading the module. """ - return '\n'.join([ - "", - "if [ module-info mode load ] {", - ' puts stderr "%s"' % msg, - "}", - "", - ]) - - def add_tcl_footer(self, tcltxt): - """ - Append whatever Tcl code you want to your modulefile - """ - # nothing to do here, but this should fail in the context of generating Lua modules - return tcltxt + # escape any (non-escaped) characters with special meaning by prefixing them with a backslash + msg = re.sub(r'((? 0: + # recursively determine dependencies for these dependency modules, until depth is non-positive + moddeps = [dependencies_for(mod, depth=depth - 1) for mod in mods] + else: + # ignore any deeper dependencies + moddeps = [] + + # add dependencies of dependency modules only if they're not there yet + for moddepdeps in moddeps: + for dep in moddepdeps: + if not dep in mods: + mods.append(dep) - def is_fake(self): - """Return whether this ModuleGenerator instance generates fake modules or not.""" - return self.fake + return mods diff --git a/easybuild/tools/module_naming_scheme/__init__.py b/easybuild/tools/module_naming_scheme/__init__.py index 964cf582aa..084cf7f165 100644 --- a/easybuild/tools/module_naming_scheme/__init__.py +++ b/easybuild/tools/module_naming_scheme/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2014 Ghent University +# Copyright 2011-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/categorized_hmns.py b/easybuild/tools/module_naming_scheme/categorized_hmns.py new file mode 100644 index 0000000000..203fa4e41f --- /dev/null +++ b/easybuild/tools/module_naming_scheme/categorized_hmns.py @@ -0,0 +1,110 @@ +## +# Copyright (c) 2015 Forschungszentrum Juelich GmbH, Germany +# +# All rights reserved. +# +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of Forschungszentrum Juelich GmbH, nor the names of +# its contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# License: 3-clause BSD +## +""" +Implementation of a hierarchical module naming scheme using module classes. + +@author: Markus Geimer (Juelich Supercomputing Centre) +""" + +import os + +from easybuild.tools.module_naming_scheme.hierarchical_mns import HierarchicalMNS +from easybuild.tools.config import build_option + + +class CategorizedHMNS(HierarchicalMNS): + """ + Class implementing an extended hierarchical module naming scheme using the + 'moduleclass' easyconfig parameter to categorize modulefiles on each level + of the hierarchy. + """ + + REQUIRED_KEYS = ['name', 'version', 'versionsuffix', 'toolchain', 'moduleclass'] + + def det_module_subdir(self, ec): + """ + Determine module subdirectory, relative to the top of the module path. + This determines the separation between module names exposed to users, + and what's part of the $MODULEPATH. This implementation appends the + 'moduleclass' easyconfig parameter to the base path of the corresponding + hierarchy level. + + Examples: + Core/compiler, Compiler/GCC/4.8.3/mpi, MPI/GCC/4.8.3/OpenMPI/1.6.5/bio + """ + moduleclass = ec['moduleclass'] + basedir = super(CategorizedHMNS, self).det_module_subdir(ec) + + return os.path.join(basedir, moduleclass) + + def det_modpath_extensions(self, ec): + """ + Determine module path extensions, if any. Appends all known (valid) + module classes to the base path of the corresponding hierarchy level. + + Examples: + Compiler/GCC/4.8.3/ (for GCC/4.8.3 module), + MPI/GCC/4.8.3/OpenMPI/1.6.5/ (for OpenMPI/1.6.5 module) + """ + basepaths = super(CategorizedHMNS, self).det_modpath_extensions(ec) + + return self.categorize_paths(basepaths) + + def det_init_modulepaths(self, ec): + """ + Determine list of initial module paths (i.e., top of the hierarchy). + Appends all known (valid) module classes to the top-level base path. + + Examples: + Core/ + """ + basepaths = super(CategorizedHMNS, self).det_init_modulepaths(ec) + + return self.categorize_paths(basepaths) + + def categorize_paths(self, basepaths): + """ + Returns a list of paths where all known (valid) module classes have + been added to each of the given base paths. + """ + valid_module_classes = build_option('valid_module_classes') + + paths = [] + for path in basepaths: + for moduleclass in valid_module_classes: + paths.extend([os.path.join(path, moduleclass)]) + + return paths diff --git a/easybuild/tools/module_naming_scheme/easybuild_mns.py b/easybuild/tools/module_naming_scheme/easybuild_mns.py index 864002431f..801ac6b4d5 100644 --- a/easybuild/tools/module_naming_scheme/easybuild_mns.py +++ b/easybuild/tools/module_naming_scheme/easybuild_mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/hierarchical_mns.py b/easybuild/tools/module_naming_scheme/hierarchical_mns.py index 48a4bc9e05..7215d1c01b 100644 --- a/easybuild/tools/module_naming_scheme/hierarchical_mns.py +++ b/easybuild/tools/module_naming_scheme/hierarchical_mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,8 +30,10 @@ """ import os +import re from vsc.utils import fancylogger +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.module_naming_scheme import ModuleNamingScheme from easybuild.tools.module_naming_scheme.toolchain import det_toolchain_compilers, det_toolchain_mpi @@ -40,14 +42,21 @@ COMPILER = 'Compiler' MPI = 'MPI' +MODULECLASS_COMPILER = 'compiler' +MODULECLASS_MPI = 'mpi' -_log = fancylogger.getLogger('HierarchicalMNS') +# note: names in keys are ordered alphabetically +COMP_NAME_VERSION_TEMPLATES = { + 'icc,ifort': ('intel', '%(icc)s'), + 'Clang,GCC': ('Clang-GCC', '%(Clang)s-%(GCC)s'), + 'CUDA,GCC': ('GCC-CUDA', '%(GCC)s-%(CUDA)s'), +} class HierarchicalMNS(ModuleNamingScheme): """Class implementing an example hierarchical module naming scheme.""" - REQUIRED_KEYS = ['name', 'version', 'versionsuffix', 'toolchain', 'moduleclass'] + REQUIRED_KEYS = ['name', 'versionprefix', 'version', 'versionsuffix', 'toolchain', 'moduleclass'] def requires_toolchain_details(self): """ @@ -68,27 +77,38 @@ def det_short_module_name(self, ec): Determine short module name, i.e. the name under which modules will be exposed to users. Examples: GCC/4.8.3, OpenMPI/1.6.5, OpenBLAS/0.2.9, HPL/2.1, Python/2.7.5 """ - return os.path.join(ec['name'], ec['version'] + ec['versionsuffix']) + return os.path.join(ec['name'], self.det_full_version(ec)) + + def det_full_version(self, ec): + """Determine full version, taking into account version prefix/suffix.""" + # versionprefix is not always available (e.g., for toolchains) + versionprefix = ec.get('versionprefix', '') + return versionprefix + ec['version'] + ec['versionsuffix'] def det_toolchain_compilers_name_version(self, tc_comps): """ Determine toolchain compiler tag, for given list of compilers. """ - if len(tc_comps) == 1: - tc_comp_name = tc_comps[0]['name'] - tc_comp_ver = tc_comps[0]['version'] + if tc_comps is None: + # no compiler in toolchain, dummy toolchain + res = None + elif len(tc_comps) == 1: + res = (tc_comps[0]['name'], tc_comps[0]['version']) else: - tc_comp_names = [comp['name'] for comp in tc_comps] - if set(tc_comp_names) == set(['icc', 'ifort']): - tc_comp_name = 'intel' - if tc_comps[0]['version'] == tc_comps[1]['version']: - tc_comp_ver = tc_comps[0]['version'] - else: - _log.error("Bumped into different versions for toolchain compilers: %s" % tc_comps) + comp_versions = dict([(comp['name'], self.det_full_version(comp)) for comp in tc_comps]) + comp_names = comp_versions.keys() + key = ','.join(sorted(comp_names)) + if key in COMP_NAME_VERSION_TEMPLATES: + tc_comp_name, tc_comp_ver_tmpl = COMP_NAME_VERSION_TEMPLATES[key] + tc_comp_ver = tc_comp_ver_tmpl % comp_versions + # make sure that icc/ifort versions match + if tc_comp_name == 'intel' and comp_versions['icc'] != comp_versions['ifort']: + raise EasyBuildError("Bumped into different versions for Intel compilers: %s", comp_versions) else: - mns = self.__class__.__name__ - _log.error("Unknown set of toolchain compilers, %s needs to be enhanced first." % mns) - return tc_comp_name, tc_comp_ver + raise EasyBuildError("Unknown set of toolchain compilers, module naming scheme needs work: %s", + comp_names) + res = (tc_comp_name, tc_comp_ver) + return res def det_module_subdir(self, ec): """ @@ -96,39 +116,79 @@ def det_module_subdir(self, ec): This determines the separation between module names exposed to users, and what's part of the $MODULEPATH. Examples: Core, Compiler/GCC/4.8.3, MPI/GCC/4.8.3/OpenMPI/1.6.5 """ - # determine prefix based on type of toolchain used tc_comps = det_toolchain_compilers(ec) - if tc_comps is None: + tc_comp_info = self.det_toolchain_compilers_name_version(tc_comps) + # determine prefix based on type of toolchain used + if tc_comp_info is None: # no compiler in toolchain, dummy toolchain => Core module subdir = CORE else: - tc_comp_name, tc_comp_ver = self.det_toolchain_compilers_name_version(tc_comps) + tc_comp_name, tc_comp_ver = tc_comp_info tc_mpi = det_toolchain_mpi(ec) if tc_mpi is None: # compiler-only toolchain => Compiler// namespace subdir = os.path.join(COMPILER, tc_comp_name, tc_comp_ver) else: # compiler-MPI toolchain => MPI//// namespace - tc_mpi_fullver = tc_mpi['version'] + tc_mpi['versionsuffix'] + tc_mpi_fullver = self.det_full_version(tc_mpi) subdir = os.path.join(MPI, tc_comp_name, tc_comp_ver, tc_mpi['name'], tc_mpi_fullver) return subdir + def det_module_symlink_paths(self, ec): + """ + Determine list of paths in which symlinks to module files must be created. + """ + # symlinks are not very useful in the context of a hierarchical MNS + return [] + def det_modpath_extensions(self, ec): """ Determine module path extensions, if any. Examples: Compiler/GCC/4.8.3 (for GCC/4.8.3 module), MPI/GCC/4.8.3/OpenMPI/1.6.5 (for OpenMPI/1.6.5 module) """ modclass = ec['moduleclass'] + tc_comps = det_toolchain_compilers(ec) + tc_comp_info = self.det_toolchain_compilers_name_version(tc_comps) paths = [] - if modclass == 'compiler': - paths.append(os.path.join(COMPILER, ec['name'], ec['version'])) - elif modclass == 'mpi': - tc_comps = det_toolchain_compilers(ec) - tc_comp_name, tc_comp_ver = self.det_toolchain_compilers_name_version(tc_comps) - fullver = ec['version'] + ec['versionsuffix'] - paths.append(os.path.join(MPI, tc_comp_name, tc_comp_ver, ec['name'], fullver)) + if modclass == MODULECLASS_COMPILER or ec['name'] in ['CUDA']: + # obtain list of compilers based on that extend $MODULEPATH in some way other than / + extend_comps = [] + # exclude GCC for which / is used as $MODULEPATH extension + excluded_comps = ['GCC'] + for comps in COMP_NAME_VERSION_TEMPLATES.keys(): + extend_comps.extend([comp for comp in comps.split(',') if comp not in excluded_comps]) + + comp_name_ver = None + if ec['name'] in extend_comps: + for key in COMP_NAME_VERSION_TEMPLATES: + if ec['name'] in key.split(','): + comp_name, comp_ver_tmpl = COMP_NAME_VERSION_TEMPLATES[key] + comp_versions = {ec['name']: self.det_full_version(ec)} + if ec['name'] == 'ifort': + # 'icc' key should be provided since it's the only one used in the template + comp_versions.update({'icc': self.det_full_version(ec)}) + if tc_comp_info is not None: + # also provide toolchain version for non-dummy toolchains + comp_versions.update({tc_comp_info[0]: tc_comp_info[1]}) + + comp_name_ver = [comp_name, comp_ver_tmpl % comp_versions] + break + else: + comp_name_ver = [ec['name'], self.det_full_version(ec)] + + paths.append(os.path.join(COMPILER, *comp_name_ver)) + + elif modclass == MODULECLASS_MPI: + if tc_comp_info is None: + raise EasyBuildError("No compiler available in toolchain %s used to install MPI library %s v%s, " + "which is required by the active module naming scheme.", + ec['toolchain'], ec['name'], ec['version']) + else: + tc_comp_name, tc_comp_ver = tc_comp_info + fullver = self.det_full_version(ec) + paths.append(os.path.join(MPI, tc_comp_name, tc_comp_ver, ec['name'], fullver)) return paths diff --git a/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py b/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py new file mode 100644 index 0000000000..747303d9c5 --- /dev/null +++ b/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py @@ -0,0 +1,37 @@ +## +# Copyright 2013-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Implementation of a module naming scheme that can be used to migrate from EasyBuildMNS to HierarchicalMNS. + +@author: Kenneth Hoste (Ghent University) +""" +from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS +from easybuild.tools.module_naming_scheme.hierarchical_mns import HierarchicalMNS + +class MigrateFromEBToHMNS(HierarchicalMNS, EasyBuildMNS): + + def det_install_subdir(self, ec): + """Determine name of software installation subdirectory of install path, using EasyBuild MNS.""" + return EasyBuildMNS.det_full_module_name(self, ec) diff --git a/easybuild/tools/module_naming_scheme/mns.py b/easybuild/tools/module_naming_scheme/mns.py index 48da9c8a8d..670493fb78 100644 --- a/easybuild/tools/module_naming_scheme/mns.py +++ b/easybuild/tools/module_naming_scheme/mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2014 Ghent University +# Copyright 2011-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,9 +28,12 @@ @author: Jens Timmerman (Ghent University) @author: Kenneth Hoste (Ghent University) """ +import re from vsc.utils import fancylogger from vsc.utils.patterns import Singleton +from easybuild.tools.build_log import EasyBuildError + class ModuleNamingScheme(object): """Abstract class for a module naming scheme implementation.""" @@ -49,7 +52,8 @@ def is_sufficient(self, keys): if self.REQUIRED_KEYS is not None: return set(keys).issuperset(set(self.REQUIRED_KEYS)) else: - self.log.error("Constant REQUIRED_KEYS is not defined, should specify required easyconfig parameters.") + raise EasyBuildError("Constant REQUIRED_KEYS is not defined, " + "should specify required easyconfig parameters.") def requires_toolchain_details(self): """ @@ -80,6 +84,18 @@ def det_short_module_name(self, ec): # by default: full module name doesn't include a $MODULEPATH subdir return self.det_full_module_name(ec) + def det_install_subdir(self, ec): + """ + Determine name of software installation subdirectory of install path. + + @param ec: dict-like object with easyconfig parameter values; for now only the 'name', + 'version', 'versionsuffix' and 'toolchain' parameters are guaranteed to be available + + @return: string with name of subdirectory, e.g.: '///' + """ + # by default: use full module name as name for install subdir + return self.det_full_module_name(ec) + def det_module_subdir(self, ec): """ Determine subdirectory for module file in $MODULEPATH. @@ -123,3 +139,17 @@ def expand_toolchain_load(self): """ # by default: just include a load statement for the toolchain return False + + def is_short_modname_for(self, short_modname, name): + """ + Determine whether the specified (short) module name is a module for software with the specified name. + Default implementation checks via a strict regex pattern, and assumes short module names are of the form: + /[-] + """ + modname_regex = re.compile('^%s/\S+$' % re.escape(name)) + res = bool(modname_regex.match(short_modname)) + + self.log.debug("Checking whether '%s' is a module name for software with name '%s' via regex %s: %s", + short_modname, name, modname_regex.pattern, res) + + return res diff --git a/easybuild/tools/module_naming_scheme/toolchain.py b/easybuild/tools/module_naming_scheme/toolchain.py index d835fcf2a9..5c63be336a 100644 --- a/easybuild/tools/module_naming_scheme/toolchain.py +++ b/easybuild/tools/module_naming_scheme/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2014 Ghent University +# Copyright 2014-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,6 +30,7 @@ from vsc.utils import fancylogger from easybuild.framework.easyconfig.easyconfig import process_easyconfig, robot_find_easyconfig +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME @@ -77,7 +78,7 @@ def det_toolchain_element_details(tc, elem): if tc_ec['name'] == elem: tc_elem_details = tc_ec else: - _log.error("No toolchain element '%s' found for toolchain %s: %s" % (elem, tc.as_dict(), tc_ec)) + raise EasyBuildError("No toolchain element '%s' found for toolchain %s: %s", elem, tc.as_dict(), tc_ec) _toolchain_details_cache[key] = tc_elem_details _log.debug("Obtained details for '%s' in toolchain '%s', added to cache" % (elem, tc_dict)) return _toolchain_details_cache[key] @@ -95,14 +96,15 @@ def det_toolchain_compilers(ec): tc_comps = None elif not TOOLCHAIN_COMPILER in tc_elems: # every toolchain should have at least a compiler - _log.error("No compiler found in toolchain %s: %s" % (ec.toolchain.as_dict(), tc_elems)) + raise EasyBuildError("No compiler found in toolchain %s: %s", ec.toolchain.as_dict(), tc_elems) elif tc_elems[TOOLCHAIN_COMPILER]: tc_comps = [] for comp_elem in tc_elems[TOOLCHAIN_COMPILER]: tc_comps.append(det_toolchain_element_details(ec.toolchain, comp_elem)) else: - _log.error("Empty list of compilers for %s toolchain definition: %s" % (ec.toolchain.as_dict(), tc_elems)) - _log.debug("Found compilers %s for toolchain %s (%s)" % (tc_comps, ec.toolchain.name, ec.toolchain.as_dict())) + raise EasyBuildError("Empty list of compilers for %s toolchain definition: %s", + ec.toolchain.as_dict(), tc_elems) + _log.debug("Found compilers %s for toolchain %s (%s)", tc_comps, ec.toolchain.name, ec.toolchain.as_dict()) return tc_comps @@ -116,7 +118,8 @@ def det_toolchain_mpi(ec): tc_elems = ec.toolchain.definition() if TOOLCHAIN_MPI in tc_elems: if not tc_elems[TOOLCHAIN_MPI]: - _log.error("Empty list of MPI libs for %s toolchain definition: %s" % (ec.toolchain.as_dict(), tc_elems)) + raise EasyBuildError("Empty list of MPI libs for %s toolchain definition: %s", + ec.toolchain.as_dict(), tc_elems) # assumption: only one MPI toolchain element tc_mpi = det_toolchain_element_details(ec.toolchain, tc_elems[TOOLCHAIN_MPI][0]) else: diff --git a/easybuild/tools/module_naming_scheme/utilities.py b/easybuild/tools/module_naming_scheme/utilities.py index db121c4744..da3711ad00 100644 --- a/easybuild/tools/module_naming_scheme/utilities.py +++ b/easybuild/tools/module_naming_scheme/utilities.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,7 +30,7 @@ @author: Kenneth Hoste (Ghent University) @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) -@author: Fotis Georgatos (Uni.Lu) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ import os import string @@ -101,3 +101,10 @@ def is_valid_module_name(mod_name): return False _log.debug("Module name %s validated" % mod_name) return True + + +def det_hidden_modname(modname): + """Determine the hidden equivalent of the specified module name.""" + moddir = os.path.dirname(modname) + modfile = os.path.basename(modname) + return os.path.join(moddir, '.%s' % modfile).lstrip(os.path.sep) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 9139e5f1d6..b670a4078f 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1,5 +1,5 @@ -# # -# Copyright 2009-2014 Ghent University +## +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -21,7 +21,7 @@ # # You should have received a copy of the GNU General Public License # along with EasyBuild. If not, see . -# # +## """ This python module implements the environment modules functionality: - loading modules @@ -38,20 +38,18 @@ import os import re import subprocess -import sys from distutils.version import StrictVersion from subprocess import PIPE from vsc.utils import fancylogger -from vsc.utils.missing import get_subclasses, any +from vsc.utils.missing import get_subclasses from vsc.utils.patterns import Singleton from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, get_modules_tool, install_path -from easybuild.tools.environment import modify_env -from easybuild.tools.filetools import convert_name, mkdir, read_file, which +from easybuild.tools.environment import ORIG_OS_ENVIRON, restore_env +from easybuild.tools.filetools import convert_name, mkdir, read_file, path_matches, which from easybuild.tools.module_naming_scheme import DEVEL_MODULE_SUFFIX from easybuild.tools.run import run_cmd -from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME, DUMMY_TOOLCHAIN_VERSION from vsc.utils.missing import nub # software root/version environment variable name prefixes @@ -59,9 +57,9 @@ VERSION_ENV_VAR_NAME_PREFIX = "EBVERSION" DEVEL_ENV_VAR_NAME_PREFIX = "EBDEVEL" -# keep track of original LD_LIBRARY_PATH, because we can change it by loading modules and break modulecmd +# environment variables to reset/restore when running a module command (to avoid breaking it) # see e.g., https://bugzilla.redhat.com/show_bug.cgi?id=719785 -LD_LIBRARY_PATH = os.getenv('LD_LIBRARY_PATH', '') +LD_ENV_VAR_KEYS = ['LD_LIBRARY_PATH', 'LD_PRELOAD'] output_matchers = { # matches whitespace and module-listing headers @@ -124,7 +122,8 @@ class ModulesTool(object): TERSE_OPTION = (0, '--terse') # module command to use COMMAND = None - # environment variable to determine the module command (instead of COMMAND) + # environment variable to determine path to module command; + # used as fallback in case command is not available in $PATH COMMAND_ENVIRONMENT = None # run module command explicitly using this shell COMMAND_SHELL = None @@ -134,15 +133,17 @@ class ModulesTool(object): REQ_VERSION = None # the regexp, should have a "version" group (multiline search) VERSION_REGEXP = None + # modules tool user cache directory + USER_CACHE_DIR = None - __metaclass__ = Singleton - - def __init__(self, mod_paths=None): + def __init__(self, mod_paths=None, testing=False): """ Create a ModulesTool object @param mod_paths: A list of paths where the modules can be located @type mod_paths: list """ + # this can/should be set to True during testing + self.testing = testing self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) self.mod_paths = None @@ -154,12 +155,21 @@ def __init__(self, mod_paths=None): # actual module command (i.e., not the 'module' wrapper function, but the binary) self.cmd = self.COMMAND - if self.COMMAND_ENVIRONMENT is not None and self.COMMAND_ENVIRONMENT in os.environ: - self.log.debug('Set command via environment variable %s' % self.COMMAND_ENVIRONMENT) - self.cmd = os.environ[self.COMMAND_ENVIRONMENT] + env_cmd_path = os.environ.get(self.COMMAND_ENVIRONMENT) + + # only use command path in environment variable if command in not available in $PATH + if which(self.cmd) is None and env_cmd_path is not None: + self.log.debug('Set command via environment variable %s: %s', self.COMMAND_ENVIRONMENT, self.cmd) + self.cmd = env_cmd_path + # check whether paths obtained via $PATH and $LMOD_CMD are different + elif which(self.cmd) != env_cmd_path: + self.log.debug("Different paths found for module command '%s' via which/$PATH and $%s: %s vs %s", + self.COMMAND, self.COMMAND_ENVIRONMENT, self.cmd, env_cmd_path) + + # make sure the module command was found if self.cmd is None: - self.log.error('No command set.') + raise EasyBuildError("No command set.") else: self.log.debug('Using command %s' % self.cmd) @@ -172,23 +182,19 @@ def __init__(self, mod_paths=None): self.check_module_function(allow_mismatch=build_option('allow_modules_tool_mismatch')) self.set_and_check_version() - # this can/should be set to True during testing - self.testing = False - def buildstats(self): """Return tuple with data to be included in buildstats""" return (self.__class__.__name__, self.cmd, self.version) @property def modules(self): - """Property providing access to deprecated 'modules' class variable.""" - self.log.deprecated("'modules' class variable is deprecated, just use load([])", '2.0') - return self._modules + """(NO LONGER SUPPORTED!) Property providing access to 'modules' class variable""" + self.log.nosupport("'modules' class variable is not supported anymore, use load([]) instead", '2.0') def set_and_check_version(self): """Get the module version, and check any requirements""" if self.VERSION_REGEXP is None: - self.log.error('No VERSION_REGEXP defined') + raise EasyBuildError("No VERSION_REGEXP defined") try: txt = self.run_module(self.VERSION_OPTION, return_output=True) @@ -198,18 +204,26 @@ def set_and_check_version(self): if res: self.version = res.group('version') self.log.info("Found version %s" % self.version) + + # make sure version is a valid StrictVersion (e.g., 5.7.3.1 is invalid), + # and replace 'rc' by 'b', to make StrictVersion treat it as a beta-release + self.version = self.version.replace('rc', 'b') + if len(self.version.split('.')) > 3: + self.version = '.'.join(self.version.split('.')[:3]) + + self.log.info("Converted actual version to '%s'" % self.version) else: - self.log.error("Failed to determine version from option '%s' output: %s" % (self.VERSION_OPTION, txt)) + raise EasyBuildError("Failed to determine version from option '%s' output: %s", + self.VERSION_OPTION, txt) except (OSError), err: - self.log.error("Failed to check version: %s" % err) + raise EasyBuildError("Failed to check version: %s", err) if self.REQ_VERSION is None: - self.log.debug('No version requirement defined.') + self.log.debug("No version requirement defined.") else: - # replace 'rc' by 'b', to make StrictVersion treat it as a beta-release - if StrictVersion(self.version.replace('rc', 'b')) < StrictVersion(self.REQ_VERSION): - msg = "EasyBuild requires v%s >= v%s (no rc), found v%s" - self.log.error(msg % (self.__class__.__name__, self.REQ_VERSION, self.version)) + if StrictVersion(self.version) < StrictVersion(self.REQ_VERSION): + raise EasyBuildError("EasyBuild requires v%s >= v%s (no rc), found v%s", + self.__class__.__name__, self.REQ_VERSION, self.version) else: self.log.debug('Version %s matches requirement %s' % (self.version, self.REQ_VERSION)) @@ -221,15 +235,24 @@ def check_cmd_avail(self): self.log.info("Full path for module command is %s, so using it" % self.cmd) else: mod_tool = self.__class__.__name__ - self.log.error("%s modules tool can not be used, '%s' command is not available." % (mod_tool, self.cmd)) + raise EasyBuildError("%s modules tool can not be used, '%s' command is not available.", mod_tool, self.cmd) def check_module_function(self, allow_mismatch=False, regex=None): """Check whether selected module tool matches 'module' function definition.""" - out, ec = run_cmd("type module", simple=False, log_ok=False, log_all=False) + if self.testing: + # grab 'module' function definition from environment if it's there; only during testing + if 'module' in os.environ: + out, ec = os.environ['module'], 0 + else: + out, ec = None, 1 + else: + out, ec = run_cmd("type module", simple=False, log_ok=False, log_all=False) + if regex is None: regex = r".*%s" % os.path.basename(self.cmd) mod_cmd_re = re.compile(regex, re.M) mod_details = "pattern '%s' (%s)" % (mod_cmd_re.pattern, self.__class__.__name__) + if ec == 0: if mod_cmd_re.search(out): self.log.debug("Found pattern '%s' in defined 'module' function." % mod_cmd_re.pattern) @@ -243,7 +266,7 @@ def check_module_function(self, allow_mismatch=False, regex=None): else: msg += "Or alternatively, use --allow-modules-tool-mismatch to stop treating this as an error. " msg += "Obtained definition of 'module' function: %s" % out - self.log.error(msg) + raise EasyBuildError(msg) else: # module function may not be defined (weird, but fine) self.log.warning("No 'module' function defined, can't check if it matches %s." % mod_details) @@ -311,16 +334,19 @@ def check_module_path(self): self.use(mod_path) self.log.info("$MODULEPATH set based on list of module paths (via 'module use'): %s" % os.environ['MODULEPATH']) - def available(self, mod_name=None): + def available(self, mod_name=None, extra_args=None): """ Return a list of available modules for the given (partial) module name; use None to obtain a list of all available modules. @param mod_name: a (partial) module name for filtering (default: None) """ + if extra_args is None: + extra_args = [] if mod_name is None: mod_name = '' - mods = self.run_module('avail', mod_name) + args = ['avail'] + extra_args + [mod_name] + mods = self.run_module(*args) # sort list of modules in alphabetical order mods.sort(key=lambda m: m['mod_name']) @@ -329,20 +355,40 @@ def available(self, mod_name=None): self.log.debug("'module available %s' gave %d answers: %s" % (mod_name, len(ans), ans)) return ans - def exists(self, mod_name): + def exist(self, mod_names, mod_exists_regex_template=r'^\s*\S*/%s:\s*$'): """ - Check if module with specified name exists. + Check if modules with specified names exists. """ - return mod_name in self.available(mod_name) + avail_mod_names = self.available() + # differentiate between hidden and visible modules + mod_names = [(mod_name, not os.path.basename(mod_name).startswith('.')) for mod_name in mod_names] + + mods_exist = [] + for (mod_name, visible) in mod_names: + if visible: + mods_exist.append(mod_name in avail_mod_names) + else: + # hidden modules are not visible in 'avail', need to use 'show' instead + modtype = ('hidden', 'visible (not hidden)')[visible] + self.log.debug("checking whether %s module %s exists via 'show'..." % (modtype, mod_name)) + txt = self.show(mod_name) + mod_exists_regex = re.compile(mod_exists_regex_template % re.escape(mod_name), re.M) + mods_exist.append(bool(mod_exists_regex.search(txt))) + + return mods_exist + + def exists(self, mod_name): + """NO LONGER SUPPORTED: use exist method instead""" + self.log.nosupport("exists() is not supported anymore, use exist([]) instead", '2.0') - def load(self, modules, mod_paths=None, purge=False, orig_env=None): + def load(self, modules, mod_paths=None, purge=False, init_env=None): """ Load all requested modules. @param modules: list of modules to load @param mod_paths: list of module paths to activate before loading @param purge: whether or not a 'module purge' should be run before loading - @param orig_env: original environment to restore after running 'module purge' + @param init_env: original environment to restore after running 'module purge' """ if mod_paths is None: mod_paths = [] @@ -350,9 +396,9 @@ def load(self, modules, mod_paths=None, purge=False, orig_env=None): # purge all loaded modules if desired if purge: self.purge() - # restore original environment if provided - if orig_env is not None: - modify_env(os.environ, orig_env) + # restore initial environment if provided + if init_env is not None: + restore_env(init_env) # make sure $MODULEPATH is set correctly after purging self.check_module_path() @@ -369,8 +415,7 @@ def unload(self, modules=None): Unload all requested modules. """ if modules is None: - self.log.deprecated("Unloading modules listed in _modules class variable", '2.0') - modules = self._modules[:] + self.log.nosupport("Unloading modules listed in _modules class variable", '2.0') for mod in modules: self.run_module('unload', mod) @@ -395,16 +440,17 @@ def get_value_from_modulefile(self, mod_name, regex): @param mod_name: module name @param regex: (compiled) regular expression, with one group """ - if self.exists(mod_name): + if self.exist([mod_name])[0]: modinfo = self.show(mod_name) self.log.debug("modinfo: %s" % modinfo) res = regex.search(modinfo) if res: return res.group(1) else: - self.log.error("Failed to determine value from 'show' (pattern: '%s') in %s" % (regex.pattern, modinfo)) + raise EasyBuildError("Failed to determine value from 'show' (pattern: '%s') in %s", + regex.pattern, modinfo) else: - raise EasyBuildError("Can't get module file path for non-existing module %s" % mod_name) + raise EasyBuildError("Can't get value from a non-existing module %s", mod_name) def modulefile_path(self, mod_name): """Get the path of the module file for the specified module.""" @@ -413,13 +459,9 @@ def modulefile_path(self, mod_name): modpath_re = re.compile('^\s*(?P[^/\n]*/[^ ]+):$', re.M) return self.get_value_from_modulefile(mod_name, modpath_re) - def module_software_name(self, mod_name): - """Get the software name for a given module name.""" - raise NotImplementedError - - def set_ld_library_path(self, ld_library_paths): - """Set $LD_LIBRARY_PATH to the given list of paths.""" - os.environ['LD_LIBRARY_PATH'] = ':'.join(ld_library_paths) + def set_path_env_var(self, key, paths): + """Set path environment variable to the given list of paths.""" + os.environ[key] = os.pathsep.join(paths) def run_module(self, *args, **kwargs): """ @@ -435,31 +477,28 @@ def run_module(self, *args, **kwargs): args.insert(*self.TERSE_OPTION) module_path_key = None - original_module_path = None if 'mod_paths' in kwargs: module_path_key = 'mod_paths' elif 'modulePath' in kwargs: module_path_key = 'modulePath' if module_path_key is not None: - original_module_path = os.environ['MODULEPATH'] - os.environ['MODULEPATH'] = kwargs[module_path_key] - self.log.deprecated("Use of '%s' named argument in 'run_module'" % module_path_key, '2.0') + self.log.nosupport("Use of '%s' named argument in 'run_module'" % module_path_key, '2.0') self.log.debug('Current MODULEPATH: %s' % os.environ.get('MODULEPATH', '')) - # change our LD_LIBRARY_PATH here + # restore selected original environment variables before running module command environ = os.environ.copy() - environ['LD_LIBRARY_PATH'] = LD_LIBRARY_PATH - cur_ld_library_path = os.environ.get('LD_LIBRARY_PATH', '') - new_ld_library_path = environ['LD_LIBRARY_PATH'] - self.log.debug("Adjusted LD_LIBRARY_PATH from '%s' to '%s'" % (cur_ld_library_path, new_ld_library_path)) + for key in LD_ENV_VAR_KEYS: + environ[key] = ORIG_OS_ENVIRON.get(key, '') + self.log.debug("Changing %s from '%s' to '%s' in environment for module command", + key, os.environ.get(key, ''), environ[key]) # prefix if a particular shell is specified, using shell argument to Popen doesn't work (no output produced (?)) cmdlist = [self.cmd, 'python'] if self.COMMAND_SHELL is not None: if not isinstance(self.COMMAND_SHELL, (list, tuple)): - msg = 'COMMAND_SHELL needs to be list or tuple, now %s (value %s)' - self.log.error(msg % (type(self.COMMAND_SHELL), self.COMMAND_SHELL)) + raise EasyBuildError("COMMAND_SHELL needs to be list or tuple, now %s (value %s)", + type(self.COMMAND_SHELL), self.COMMAND_SHELL) cmdlist = self.COMMAND_SHELL + cmdlist full_cmd = ' '.join(cmdlist + args) @@ -469,18 +508,16 @@ def run_module(self, *args, **kwargs): # stderr will contain text (just like the normal module command) (stdout, stderr) = proc.communicate() self.log.debug("Output of module command '%s': stdout: %s; stderr: %s" % (full_cmd, stdout, stderr)) - if original_module_path is not None: - os.environ['MODULEPATH'] = original_module_path - self.log.deprecated("Restoring $MODULEPATH back to what it was before running module command/.", '2.0') if kwargs.get('return_output', False): return stdout + stderr else: - # the module command was run with an outdated LD_LIBRARY_PATH, which will be adjusted on loading a module + # the module command was run with an outdated selected environment variables (see LD_ENV_VAR_KEYS list) + # which will be adjusted on loading a module; # this needs to be taken into account when updating the environment via produced output, see below - # keep track of current LD_LIBRARY_PATH, so we can correct the adjusted LD_LIBRARY_PATH below - prev_ld_library_path = os.environ.get('LD_LIBRARY_PATH', '').split(':')[::-1] + # keep track of current values of select env vars, so we can correct the adjusted values below + prev_ld_values = dict([(key, os.environ.get(key, '').split(os.pathsep)[::-1]) for key in LD_ENV_VAR_KEYS]) # Change the environment try: @@ -490,16 +527,16 @@ def run_module(self, *args, **kwargs): exec stdout except Exception, err: out = "stdout: %s, stderr: %s" % (stdout, stderr) - raise EasyBuildError("Changing environment as dictated by module failed: %s (%s)" % (err, out)) + raise EasyBuildError("Changing environment as dictated by module failed: %s (%s)", err, out) - # correct LD_LIBRARY_PATH as yielded by the adjustments made + # correct values of selected environment variables as yielded by the adjustments made # make sure we get the order right (reverse lists with [::-1]) - curr_ld_library_path = os.environ.get('LD_LIBRARY_PATH', '').split(':') - new_ld_library_path = [x for x in nub(prev_ld_library_path + curr_ld_library_path[::-1]) if len(x)][::-1] + for key in LD_ENV_VAR_KEYS: + curr_ld_val = os.environ.get(key, '').split(os.pathsep) + new_ld_val = [x for x in nub(prev_ld_values[key] + curr_ld_val[::-1]) if x][::-1] - self.log.debug("Correcting paths in LD_LIBRARY_PATH from %s to %s" % - (curr_ld_library_path, new_ld_library_path)) - self.set_ld_library_path(new_ld_library_path) + self.log.debug("Correcting paths in $%s from %s to %s" % (key, curr_ld_val, new_ld_val)) + self.set_path_env_var(key, new_ld_val) # Process stderr result = [] @@ -509,7 +546,6 @@ def run_module(self, *args, **kwargs): error = output_matchers['error'].search(line) if error: - self.log.error(line) raise EasyBuildError(line) modules = output_matchers['available'].finditer(line) @@ -531,33 +567,126 @@ def loaded_modules(self): return loaded_modules - # depth=sys.maxint should be equivalent to infinite recursion depth - def dependencies_for(self, mod_name, depth=sys.maxint): + def read_module_file(self, mod_name): """ - Obtain a list of dependencies for the given module, determined recursively, up to a specified depth (optionally) + Read module file with specified name. """ modfilepath = self.modulefile_path(mod_name) self.log.debug("modulefile path %s: %s" % (mod_name, modfilepath)) - modtxt = read_file(modfilepath) + return read_file(modfilepath) - loadregex = re.compile(r"^\s+module load\s+(.*)$", re.M) - mods = loadregex.findall(modtxt) + def modpath_extensions_for(self, mod_names): + """ + Determine dictionary with $MODULEPATH extensions for specified modules. + Modules with an empty list of $MODULEPATH extensions are included. + """ + self.log.debug("Determining $MODULEPATH extensions for modules %s" % mod_names) - if depth > 0: - # recursively determine dependencies for these dependency modules, until depth is non-positive - moddeps = [self.dependencies_for(mod, depth=depth - 1) for mod in mods] - else: - # ignore any deeper dependencies - moddeps = [] + # copy environment so we can restore it + env = os.environ.copy() - # add dependencies of dependency modules only if they're not there yet - for moddepdeps in moddeps: - for dep in moddepdeps: - if not dep in mods: - mods.append(dep) + modpath_exts = {} + for mod_name in mod_names: + modtxt = self.read_module_file(mod_name) + useregex = re.compile(r"^\s*module\s+use\s+(\S+)", re.M) + exts = useregex.findall(modtxt) - return mods + self.log.debug("Found $MODULEPATH extensions for %s: %s" % (mod_name, exts)) + modpath_exts.update({mod_name: exts}) + + if exts: + # load this module, since it may extend $MODULEPATH to make other modules available + # this is required to obtain the list of $MODULEPATH extensions they make (via 'module show') + self.load([mod_name]) + + # restore environment (modules may have been loaded above) + restore_env(env) + + return modpath_exts + + def path_to_top_of_module_tree(self, top_paths, mod_name, full_mod_subdir, deps, modpath_exts=None): + """ + Recursively determine path to the top of the module tree, + for given module, module subdir and list of $MODULEPATH extensions per dependency module. + + For example, when to determine the path to the top of the module tree for the HPL/2.1 module being + installed with a goolf/1.5.14 toolchain in a Core/Compiler/MPI hierarchy (HierarchicalMNS): + + * starting point: + top_paths = ['', '/Core'] + mod_name = 'HPL/2.1' + full_mod_subdir = '/MPI/Compiler/GCC/4.8.2/OpenMPI/1.6.5' + deps = ['GCC/4.8.2', 'OpenMPI/1.6.5', 'OpenBLAS/0.2.8-LAPACK-3.5.0', 'FFTW/3.3.4', 'ScaLAPACK/...'] + + * 1st iteration: find module that extends $MODULEPATH with '/MPI/Compiler/GCC/4.8.2/OpenMPI/1.6.5', + => OpenMPI/1.6.5 (in '/Compiler/GCC/4.8.2' subdir); + recurse with mod_name = 'OpenMPI/1.6.5' and full_mod_subdir = '/Compiler/GCC/4.8.2' + + * 2nd iteration: find module that extends $MODULEPATH with '/Compiler/GCC/4.8.2' + => GCC/4.8.2 (in '/Core' subdir); + recurse with mod_name = 'GCC/4.8.2' and full_mod_subdir = '/Core' + + * 3rd iteration: try to find module that extends $MODULEPATH with '/Core' + => '/Core' is in top_paths, so stop recursion + + @param top_paths: list of potentation 'top of module tree' (absolute) paths + @param mod_name: (short) module name for starting point (only used in log messages) + @param full_mod_subdir: absolute path to module subdirectory for starting point + @param deps: list of dependency modules for module at starting point + @param modpath_exts: list of module path extensions for each of the dependency modules + """ + # copy environment so we can restore it + env = os.environ.copy() + + if path_matches(full_mod_subdir, top_paths): + self.log.debug("Top of module tree reached with %s (module subdir: %s)" % (mod_name, full_mod_subdir)) + return [] + + self.log.debug("Checking for dependency that extends $MODULEPATH with %s" % full_mod_subdir) + + if modpath_exts is None: + # only retain dependencies that have a non-empty lists of $MODULEPATH extensions + modpath_exts = dict([(k, v) for k, v in self.modpath_extensions_for(deps).items() if v]) + self.log.debug("Non-empty lists of module path extensions for dependencies: %s" % modpath_exts) + + mods_to_top = [] + full_mod_subdirs = [] + for dep in modpath_exts: + # if a $MODULEPATH extension is identical to where this module will be installed, we have a hit + # use os.path.samefile when comparing paths to avoid issues with resolved symlinks + full_modpath_exts = modpath_exts[dep] + if path_matches(full_mod_subdir, full_modpath_exts): + # full path to module subdir of dependency is simply path to module file without (short) module name + dep_full_mod_subdir = self.modulefile_path(dep)[:-len(dep)-1] + full_mod_subdirs.append(dep_full_mod_subdir) + + mods_to_top.append(dep) + self.log.debug("Found module to top of module tree: %s (subdir: %s, modpath extensions %s)", + dep, dep_full_mod_subdir, full_modpath_exts) + + if full_modpath_exts: + # load module for this dependency, since it may extend $MODULEPATH to make dependencies available + # this is required to obtain the corresponding module file paths (via 'module show') + self.load([dep]) + + # restore original environment (modules may have been loaded above) + restore_env(env) + + path = mods_to_top[:] + if mods_to_top: + # remove retained dependencies from the list, since we're climbing up the module tree + remaining_modpath_exts = dict([m for m in modpath_exts.items() if not m[0] in mods_to_top]) + + self.log.debug("Path to top from %s extended to %s, so recursing to find way to the top" % (mod_name, mods_to_top)) + for mod_name, full_mod_subdir in zip(mods_to_top, full_mod_subdirs): + path.extend(self.path_to_top_of_module_tree(top_paths, mod_name, full_mod_subdir, None, + modpath_exts=remaining_modpath_exts)) + else: + self.log.debug("Path not extended, we must have reached the top of the module tree") + + self.log.debug("Path to top of module tree from %s: %s" % (mod_name, path)) + return path def update(self): """Update after new modules were added.""" @@ -570,12 +699,6 @@ class EnvironmentModulesC(ModulesTool): REQ_VERSION = '3.2.10' VERSION_REGEXP = r'^\s*(VERSION\s*=\s*)?(?P\d\S*)\s*' - def module_software_name(self, mod_name): - """Get the software name for a given module name.""" - # line that specified conflict contains software name - name_re = re.compile('^conflict\s*(?P\S+).*$', re.M) - return self.get_value_from_modulefile(mod_name, name_re) - def update(self): """Update after new modules were added.""" pass @@ -593,11 +716,11 @@ class EnvironmentModulesTcl(EnvironmentModulesC): REQ_VERSION = None VERSION_REGEXP = r'^Modules\s+Release\s+Tcl\s+(?P\d\S*)\s' - def set_ld_library_path(self, ld_library_paths): - """Set $LD_LIBRARY_PATH to the given list of paths.""" - super(EnvironmentModulesTcl, self).set_ld_library_path(ld_library_paths) + def set_path_env_var(self, key, paths): + """Set environment variable with given name to the given list of paths.""" + super(EnvironmentModulesTcl, self).set_path_env_var(key, paths) # for Tcl environment modules, we need to make sure the _modshare env var is kept in sync - os.environ['LD_LIBRARY_PATH_modshare'] = ':1:'.join(ld_library_paths) + os.environ['%s_modshare' % key] = ':1:'.join(paths) def run_module(self, *args, **kwargs): """ @@ -655,6 +778,7 @@ class Lmod(ModulesTool): # we need at least Lmod v5.6.3 (and it can't be a release candidate) REQ_VERSION = '5.6.3' VERSION_REGEXP = r"^Modules\s+based\s+on\s+Lua:\s+Version\s+(?P\d\S*)\s" + USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache') def __init__(self, *args, **kwargs): """Constructor, set lmod-specific class variable values.""" @@ -678,7 +802,13 @@ def available(self, mod_name=None): @param name: a (partial) module name for filtering (default: None) """ - mods = super(Lmod, self).available(mod_name=mod_name) + extra_args = [] + if StrictVersion(self.version) >= StrictVersion('5.7.5'): + # make hidden modules visible for recent version of Lmod + extra_args = ['--show_hidden'] + + mods = super(Lmod, self).available(mod_name=mod_name, extra_args=extra_args) + # only retain actual modules, exclude module directories (which end with a '/') real_mods = [mod for mod in mods if not mod.endswith('/')] @@ -690,72 +820,72 @@ def available(self, mod_name=None): def update(self): """Update after new modules were added.""" - spider_cmd = os.path.join(os.path.dirname(self.cmd), 'spider') - cmd = [spider_cmd, '-o', 'moduleT', os.environ['MODULEPATH']] - self.log.debug("Running command '%s'..." % ' '.join(cmd)) + if build_option('update_modules_tool_cache'): + spider_cmd = os.path.join(os.path.dirname(self.cmd), 'spider') + cmd = [spider_cmd, '-o', 'moduleT', os.environ['MODULEPATH']] + self.log.debug("Running command '%s'..." % ' '.join(cmd)) - proc = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE, env=os.environ) - (stdout, stderr) = proc.communicate() + proc = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE, env=os.environ) + (stdout, stderr) = proc.communicate() - if stderr: - self.log.error("An error occured when running '%s': %s" % (' '.join(cmd), stderr)) + if stderr: + raise EasyBuildError("An error occured when running '%s': %s", ' '.join(cmd), stderr) - if self.testing: - # don't actually update local cache when testing, just return the cache contents - return stdout - else: - try: - cache_filefn = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache', 'moduleT.lua') - self.log.debug("Updating Lmod spider cache %s with output from '%s'" % (cache_filefn, ' '.join(cmd))) - cache_dir = os.path.dirname(cache_filefn) - if not os.path.exists(cache_dir): - mkdir(cache_dir, parents=True) - cache_file = open(cache_filefn, 'w') - cache_file.write(stdout) - cache_file.close() - except (IOError, OSError), err: - self.log.error("Failed to update Lmod spider cache %s: %s" % (cache_filefn, err)) - - def module_software_name(self, mod_name): - """Get the software name for a given module name.""" - # line that specified conflict contains software name - name_re = re.compile('^conflict\("*(?P[^ "]+)"\).*$', re.M) - return self.get_value_from_modulefile(mod_name, name_re) + if self.testing: + # don't actually update local cache when testing, just return the cache contents + return stdout + else: + try: + cache_fp = os.path.join(self.USER_CACHE_DIR, 'moduleT.lua') + self.log.debug("Updating Lmod spider cache %s with output from '%s'" % (cache_fp, ' '.join(cmd))) + cache_dir = os.path.dirname(cache_fp) + if not os.path.exists(cache_dir): + mkdir(cache_dir, parents=True) + cache_file = open(cache_fp, 'w') + cache_file.write(stdout) + cache_file.close() + except (IOError, OSError), err: + raise EasyBuildError("Failed to update Lmod spider cache %s: %s", cache_fp, err) def prepend_module_path(self, path): # Lmod pushes a path to the front on 'module use' self.use(path) self.set_mod_paths() + def exist(self, mod_names): + """Check if modules with specified names exists.""" + # module file may be either in Tcl syntax (no file extension) or Lua sytax (.lua extension); + # the current configuration for matters little, since the module may have been installed with a different cfg; + # Lmod may pick up both Tcl and Lua module files, regardless of the EasyBuild configuration + return super(Lmod, self).exist(mod_names, r'^\s*\S*/%s(.lua)?:\s*$') + def get_software_root_env_var_name(name): """Return name of environment variable for software root.""" newname = convert_name(name, upper=True) - return ''.join([ROOT_ENV_VAR_NAME_PREFIX, newname]) + return ROOT_ENV_VAR_NAME_PREFIX + newname def get_software_root(name, with_env_var=False): """ Return the software root set for a particular software name. """ - environment_key = get_software_root_env_var_name(name) - newname = convert_name(name, upper=True) - legacy_key = "SOFTROOT%s" % newname + env_var = get_software_root_env_var_name(name) + legacy_key = "SOFTROOT%s" % convert_name(name, upper=True) - # keep on supporting legacy installations - if environment_key in os.environ: - env_var = environment_key - else: - env_var = legacy_key - if legacy_key in os.environ: - _log.deprecated("Legacy env var %s is being relied on!" % legacy_key, "2.0") + root = None + if env_var in os.environ: + root = os.getenv(env_var) - root = os.getenv(env_var) + elif legacy_key in os.environ: + _log.nosupport("Legacy env var %s is being relied on!" % legacy_key, "2.0") if with_env_var: - return (root, env_var) + res = (root, env_var) else: - return root + res = root + + return res def get_software_libdir(name, only_one=True, fs=None): @@ -784,7 +914,8 @@ def get_software_libdir(name, only_one=True, fs=None): if len(res) == 1: res = res[0] else: - _log.error("Multiple library subdirectories found for %s in %s: %s" % (name, root, ', '.join(res))) + raise EasyBuildError("Multiple library subdirectories found for %s in %s: %s", + name, root, ', '.join(res)) return res else: # return None if software package root could not be determined @@ -794,25 +925,23 @@ def get_software_libdir(name, only_one=True, fs=None): def get_software_version_env_var_name(name): """Return name of environment variable for software root.""" newname = convert_name(name, upper=True) - return ''.join([VERSION_ENV_VAR_NAME_PREFIX, newname]) + return VERSION_ENV_VAR_NAME_PREFIX + newname def get_software_version(name): """ Return the software version set for a particular software name. """ - environment_key = get_software_version_env_var_name(name) - newname = convert_name(name, upper=True) - legacy_key = "SOFTVERSION%s" % newname + env_var = get_software_version_env_var_name(name) + legacy_key = "SOFTVERSION%s" % convert_name(name, upper=True) - # keep on supporting legacy installations - if environment_key in os.environ: - return os.getenv(environment_key) - else: - if legacy_key in os.environ: - _log.deprecated("Legacy env var %s is being relied on!" % legacy_key, "2.0") - return os.getenv(legacy_key) + version = None + if env_var in os.environ: + version = os.getenv(env_var) + elif legacy_key in os.environ: + _log.nosupport("Legacy env var %s is being relied on!" % legacy_key, "2.0") + return version def curr_module_paths(): """ @@ -839,7 +968,7 @@ def avail_modules_tools(): return class_dict -def modules_tool(mod_paths=None): +def modules_tool(mod_paths=None, testing=False): """ Return interface to modules tool (environment modules (C, Tcl), or Lmod) """ @@ -847,15 +976,12 @@ def modules_tool(mod_paths=None): modules_tool = get_modules_tool() if modules_tool is not None: modules_tool_class = avail_modules_tools().get(modules_tool) - return modules_tool_class(mod_paths=mod_paths) + return modules_tool_class(mod_paths=mod_paths, testing=testing) else: return None -# provide Modules class for backward compatibility (e.g., in easyblocks) class Modules(EnvironmentModulesC): - """Deprecated interface to modules tool.""" - + """NO LONGER SUPPORTED: interface to modules tool, use modules_tool from easybuild.tools.modules instead""" def __init__(self, *args, **kwargs): - _log.deprecated("modules.Modules class is now an abstract interface, use modules.modules_tool instead", "2.0") - super(Modules, self).__init__(*args, **kwargs) + _log.nosupport("modules.Modules class is now an abstract interface, use modules.modules_tool instead", '2.0') diff --git a/easybuild/tools/multidiff.py b/easybuild/tools/multidiff.py new file mode 100644 index 0000000000..925fac805a --- /dev/null +++ b/easybuild/tools/multidiff.py @@ -0,0 +1,291 @@ +# # +# Copyright 2014-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Module which allows the diffing of multiple files + +@author: Toon Willems (Ghent University) +@author: Kenneth Hoste (Ghent University) +""" + +import difflib +import math +import os +from vsc.utils import fancylogger + +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import read_file +from easybuild.tools.systemtools import det_terminal_size + + +SEP_WIDTH = 5 + +# text colors +PURPLE = "\033[0;35m" +# background colors +GREEN_BACK = "\033[0;42m" +RED_BACK = "\033[0;41m" +# end character for colorized text +END_COLOR = "\033[0m" + +# meaning characters in diff context +HAT = '^' +MINUS = '-' +PLUS = '+' +SPACE = ' ' +QUESTIONMARK = '?' + +END_LONG_LINE = '...' + +# restrict displaying of differences to limited number of groups +MAX_DIFF_GROUPS = 3 + + +_log = fancylogger.getLogger('multidiff', fname=False) + + +class MultiDiff(object): + """ + Class representing a multi-diff. + """ + def __init__(self, base_fn, base_lines, files, colored=True): + """ + MultiDiff constructor + @param base: base to compare with + @param files: list of files to compare with base + @param colored: boolean indicating whether a colored multi-diff should be generated + """ + self.base_fn = base_fn + self.base_lines = base_lines + self.files = files + self.colored = colored + self.diff_info = {} + + def parse_line(self, line_no, diff_line, meta, squigly_line): + """ + Register a diff line + @param line_no: line number + @param diff_line: diff line generated by difflib + @param meta: meta information (e.g., filename) + @param squigly_line: squigly line indicating which characters changed + """ + # register (diff_line, meta, squigly_line) tuple for specified line number and determined key + key = diff_line[0] + if not key in [MINUS, PLUS]: + raise EasyBuildError("diff line starts with unexpected character: %s", diff_line) + line_key_tuples = self.diff_info.setdefault(line_no, {}).setdefault(key, []) + line_key_tuples.append((diff_line, meta, squigly_line)) + + def color_line(self, line, color): + """Create colored version of given line, with given color, if color mode is enabled.""" + if self.colored: + line = ''.join([color, line, END_COLOR]) + return line + + def merge_squigly(self, squigly1, squigly2): + """Combine two squigly lines into a single squigly line.""" + sq1 = list(squigly1) + sq2 = list(squigly2) + # longest line is base + base, other = (sq1, sq2) if len(sq1) > len(sq2) else (sq2, sq1) + + for i, char in enumerate(other): + if base[i] in [HAT, SPACE] and base[i] != char: + base[i] = char + + return ''.join(base) + + def colorize(self, line, squigly): + """Add colors to the diff line based on the squigly line.""" + if not self.colored: + return line + + # must be a list so we can insert stuff + chars = list(line) + flag = ' ' + offset = 0 + color_map = { + HAT: GREEN_BACK if line.startswith(PLUS) else RED_BACK, + MINUS: RED_BACK, + PLUS: GREEN_BACK, + } + if squigly: + for i, squigly_char in enumerate(squigly): + if squigly_char != flag: + chars.insert(i + offset, END_COLOR) + offset += 1 + if squigly_char in [HAT, MINUS, PLUS]: + chars.insert(i + offset, color_map[squigly_char]) + offset += 1 + flag = squigly_char + chars.insert(len(squigly) + offset, END_COLOR) + else: + chars.insert(0, color_map.get(line[0], '')) + chars.append(END_COLOR) + + return ''.join(chars) + + def get_line(self, line_no): + """ + Return the line information for a specific line + @param line_no: line number to obtain information for + @return: list with text lines providing line information + """ + output = [] + diff_dict = self.diff_info.get(line_no, {}) + for key in [MINUS, PLUS]: + lines, changes_dict, squigly_dict = set(), {}, {} + + # obtain relevant diff lines + if key in diff_dict: + for (diff_line, meta, squigly_line) in diff_dict[key]: + if squigly_line: + # merge squigly lines + if diff_line in squigly_dict: + squigly_line = self.merge_squigly(squigly_line, squigly_dict[diff_line]) + squigly_dict[diff_line] = squigly_line + lines.add(diff_line) + # track meta info (which filenames are relevant) + changes_dict.setdefault(diff_line, set()).add(meta) + + # sort: lines with most changes last, limit number to MAX_DIFF_GROUPS + lines = sorted(lines, key=lambda line: len(changes_dict[line]))[:MAX_DIFF_GROUPS] + + for diff_line in lines: + squigly_line = squigly_dict.get(diff_line, '') + line = ['%s %s' % (line_no, self.colorize(diff_line, squigly_line))] + + # mention to how may files this diff applies + files = changes_dict[diff_line] + num_files = len(self.files) + line.append("(%d/%d)" % (len(files), num_files)) + + # list files to which this diff applies (don't list all files) + if len(files) != num_files: + line.append(', '.join(files)) + + output.append(' '.join(line)) + + # prepend spaces to match line number length in non-color mode + if not self.colored and squigly_line: + prepend = ' ' * (2 + int(math.log10(line_no))) + output.append(''.join([prepend, squigly_line])) + + # print seperator only if needed + if diff_dict and not self.diff_info.get(line_no + 1, {}): + output.extend([' ', '-' * SEP_WIDTH, ' ']) + + return output + + def __str__(self): + """ + Create a string representation of this multi-diff + """ + def limit(text, length): + """Limit text to specified length, terminate color mode and add END_LONG_LINE if trimmed.""" + if len(text) > length: + maxlen = length - len(END_LONG_LINE) + res = text[:maxlen] + if self.colored: + res += END_COLOR + return res + END_LONG_LINE + else: + return text + + _, term_width = det_terminal_size() + + base = self.color_line(self.base_fn, PURPLE) + filenames = ', '.join(map(os.path.basename, self.files)) + output = [ + "Comparing %s with %s" % (base, filenames), + '=' * SEP_WIDTH, + ] + + diff = False + for i in range(len(self.base_lines)): + lines = filter(None, self.get_line(i)) + if lines: + output.append('\n'.join([limit(line, term_width) for line in lines])) + diff = True + + if not diff: + output.append("(no diff)") + + output.append('=' * SEP_WIDTH) + + return '\n'.join(output) + + +def multidiff(base, files, colored=True): + """ + Generate a diff for multiple files, all compared to base. + @param base: base to compare with + @param files: list of files to compare with base + @param colored: boolean indicating whether a colored multi-diff should be generated + @return: text with multidiff overview + """ + differ = difflib.Differ() + base_lines = read_file(base).split('\n') + mdiff = MultiDiff(os.path.basename(base), base_lines, files, colored=colored) + + # use the MultiDiff class to store the information + for filepath in files: + lines = read_file(filepath).split('\n') + diff = differ.compare(lines, base_lines) + filename = os.path.basename(filepath) + + # contruct map of line number to diff lines and mapping between diff lines + # example partial diff: + # + # - toolchain = {'name': 'goolfc', 'version': '2.6.10'} + # ? - ^ ^ + # + # + toolchain = {'name': 'goolf', 'version': '1.6.20'} + # ? ^ ^ + # + local_diff = {} + squigly_dict = {} + last_added = None + offset = 1 + for (i, line) in enumerate(diff): + # diff line indicating changed characters on line above, a.k.a. a 'squigly' line + if line.startswith(QUESTIONMARK): + squigly_dict[last_added] = line + offset -= 1 + # diff line indicating addition change + elif line.startswith(PLUS): + local_diff.setdefault(i + offset, []).append((line, filename)) + last_added = line + # diff line indicated removal change + elif line.startswith(MINUS): + local_diff.setdefault(i + offset, []).append((line, filename)) + last_added = line + offset -= 1 + + # construct the multi-diff based on the constructed dict + for line_no in local_diff: + for (line, filename) in local_diff[line_no]: + mdiff.parse_line(line_no, line.rstrip(), filename, squigly_dict.get(line, '').rstrip()) + + return str(mdiff) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index d1f2173033..b4b729ab0c 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1,5 +1,5 @@ -# # -# Copyright 2009-2014 Ghent University +## +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -21,7 +21,7 @@ # # You should have received a copy of the GNU General Public License # along with EasyBuild. If not, see . -# # +## """ Command line options for eb @@ -33,36 +33,55 @@ @author: Toon Willems (Ghent University) @author: Ward Poelmans (Ghent University) """ +import glob import os import re +import shutil import sys from distutils.version import LooseVersion +from vsc.utils.missing import nub -from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.easyblock import MODULE_ONLY_STEPS, SOURCE_STEP, EasyBlock +from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR from easybuild.framework.easyconfig.constants import constant_documentation -from easybuild.framework.easyconfig.default import convert_to_help -from easybuild.framework.easyconfig.easyconfig import get_easyblock_class +from easybuild.framework.easyconfig.easyconfig import HAVE_AUTOPEP8 from easybuild.framework.easyconfig.format.pyheaderconfigobj import build_easyconfig_constants_dict from easybuild.framework.easyconfig.licenses import license_documentation from easybuild.framework.easyconfig.templates import template_documentation from easybuild.framework.easyconfig.tools import get_paths_for from easybuild.framework.extension import Extension -from easybuild.tools import build_log, config, run # @UnusedImport make sure config is always initialized! -from easybuild.tools.build_log import print_warning -from easybuild.tools.config import get_default_configfiles, get_pretend_installpath -from easybuild.tools.config import get_default_oldstyle_configfile_defaults, DEFAULT_MODULECLASSES -from easybuild.tools.convert import ListOfStrings +from easybuild.tools import build_log, run # build_log should always stay there, to ensure EasyBuildLog +from easybuild.tools.build_log import EasyBuildError, raise_easybuilderror +from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX +from easybuild.tools.config import DEFAULT_MODULES_TOOL, DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS +from easybuild.tools.config import DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL, DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX +from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_STRICT +from easybuild.tools.config import get_pretend_installpath, mk_full_default_path, set_tmpdir +from easybuild.tools.configobj import ConfigObj, ConfigObjError +from easybuild.tools.docs import FORMAT_RST, FORMAT_TXT, avail_easyconfig_params from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, fetch_github_token +from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains +from easybuild.tools.job.backend import avail_job_backends from easybuild.tools.modules import avail_modules_tools +from easybuild.tools.module_generator import ModuleGeneratorLua, avail_module_generators from easybuild.tools.module_naming_scheme import GENERAL_CLASS from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes +from easybuild.tools.modules import Lmod from easybuild.tools.ordereddict import OrderedDict +from easybuild.tools.package.utilities import avail_package_naming_schemes from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.repository.repository import avail_repositories from easybuild.tools.version import this_is_easybuild from vsc.utils import fancylogger from vsc.utils.generaloption import GeneralOption -from vsc.utils.missing import any + + +CONFIG_ENV_VAR_PREFIX = 'EASYBUILD' + +XDG_CONFIG_HOME = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), ".config")) +XDG_CONFIG_DIRS = os.environ.get('XDG_CONFIG_DIRS', '/etc').split(os.pathsep) +DEFAULT_SYS_CFGFILES = [f for d in XDG_CONFIG_DIRS for f in sorted(glob.glob(os.path.join(d, 'easybuild.d', '*.cfg')))] +DEFAULT_USER_CFGFILE = os.path.join(XDG_CONFIG_HOME, 'easybuild', 'config.cfg') class EasyBuildOptions(GeneralOption): @@ -70,21 +89,41 @@ class EasyBuildOptions(GeneralOption): VERSION = this_is_easybuild() DEFAULT_LOGLEVEL = 'INFO' - DEFAULT_CONFIGFILES = get_default_configfiles() + DEFAULT_CONFIGFILES = DEFAULT_SYS_CFGFILES[:] + if os.path.exists(DEFAULT_USER_CFGFILE): + DEFAULT_CONFIGFILES.append(DEFAULT_USER_CFGFILE) ALLOPTSMANDATORY = False # allow more than one argument + CONFIGFILES_RAISE_MISSING = True # don't allow non-existing config files to be specified + + def __init__(self, *args, **kwargs): + """Constructor.""" + + self.default_repositorypath = [mk_full_default_path('repositorypath')] + self.default_robot_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None) or [] + + # set up constants to seed into config files parser, by section + self.go_cfg_constants = { + self.DEFAULTSECT: { + 'DEFAULT_REPOSITORYPATH': (self.default_repositorypath[0], "Default easyconfigs repository path"), + 'DEFAULT_ROBOT_PATHS': (os.pathsep.join(self.default_robot_paths), + "List of default robot paths ('%s'-separated)" % os.pathsep), + } + } + + # update or define go_configfiles_initenv in named arguments to pass to parent constructor + go_cfg_initenv = kwargs.setdefault('go_configfiles_initenv', {}) + for section, constants in self.go_cfg_constants.items(): + constants = dict([(name, value) for (name, (value, _)) in constants.items()]) + go_cfg_initenv.setdefault(section, {}).update(constants) + + super(EasyBuildOptions, self).__init__(*args, **kwargs) def basic_options(self): """basic runtime options""" all_stops = [x[0] for x in EasyBlock.get_steps()] strictness_options = [run.IGNORE, run.WARN, run.ERROR] - try: - default_robot_path = get_paths_for("easyconfigs", robot_path=None)[0] - except: - self.log.warning("basic_options: unable to determine default easyconfig path") - default_robot_path = False # False as opposed to None, since None is used for indicating that --robot was not used - descr = ("Basic options", "Basic runtime options for EasyBuild.") opts = OrderedDict({ @@ -95,12 +134,15 @@ def basic_options(self): 'job': ("Submit the build as a job", None, 'store_true', False), 'logtostdout': ("Redirect main log to stdout", None, 'store_true', False, 'l'), 'only-blocks': ("Only build listed blocks", None, 'extend', None, 'b', {'metavar': 'BLOCKS'}), - 'robot': ("Path(s) to search for easyconfigs for missing dependencies (colon-separated)" , - None, 'store_or_None', default_robot_path, 'r', {'metavar': 'PATH'}), + 'robot': ("Enable dependency resolution, using easyconfigs in specified paths", + 'pathlist', 'store_or_None', [], 'r', {'metavar': 'PATH[%sPATH]' % os.pathsep}), + 'robot-paths': ("Additional paths to consider by robot for easyconfigs (--robot paths get priority)", + 'pathlist', 'add_flex', self.default_robot_paths, {'metavar': 'PATH[%sPATH]' % os.pathsep}), 'skip': ("Skip existing software (useful for installing additional packages)", None, 'store_true', False, 'k'), - 'stop': ("Stop the installation after certain step", 'choice', 'store_or_None', 'source', 's', all_stops), - 'strict': ("Set strictness level", 'choice', 'store', run.WARN, strictness_options), + 'stop': ("Stop the installation after certain step", + 'choice', 'store_or_None', SOURCE_STEP, 's', all_stops), + 'strict': ("Set strictness level", 'choice', 'store', DEFAULT_STRICT, strictness_options), }) self.log.debug("basic_options: descr %s opts %s" % (descr, opts)) @@ -117,18 +159,20 @@ def software_options(self): ) opts = OrderedDict({ - 'amend':(("Specify additional search and build parameters (can be used multiple times); " - "for example: versionprefix=foo or patches=one.patch,two.patch)"), + 'amend': (("Specify additional search and build parameters (can be used multiple times); " + "for example: versionprefix=foo or patches=one.patch,two.patch)"), None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), - 'software-name': ("Search and build software with name", + 'software': ("Search and build software with given name and version", + None, 'extend', None, {'metavar': 'NAME,VERSION'}), + 'software-name': ("Search and build software with given name", None, 'store', None, {'metavar': 'NAME'}), - 'software-version': ("Search and build software with version", + 'software-version': ("Search and build software with given version", None, 'store', None, {'metavar': 'VERSION'}), - 'toolchain': ("Search and build with toolchain (name and version)", + 'toolchain': ("Search and build with given toolchain (name and version)", None, 'extend', None, {'metavar': 'NAME,VERSION'}), - 'toolchain-name': ("Search and build with toolchain name", + 'toolchain-name': ("Search and build with given toolchain name", None, 'store', None, {'metavar': 'NAME'}), - 'toolchain-version': ("Search and build with toolchain version", + 'toolchain-version': ("Search and build with given toolchain version", None, 'store', None, {'metavar': 'VERSION'}), }) @@ -154,28 +198,43 @@ def override_options(self): 'allow-modules-tool-mismatch': ("Allow mismatch of modules tool and definition of 'module' function", None, 'store_true', False), 'cleanup-builddir': ("Cleanup build dir after successful installation.", None, 'store_true', True), + 'cleanup-tmpdir': ("Cleanup tmp dir after successful run.", None, 'store_true', True), + 'color': ("Allow color output", None, 'store_true', True), 'deprecated': ("Run pretending to be (future) version, to test removal of deprecated code.", None, 'store', None), + 'download-timeout': ("Timeout for initiating downloads (in seconds)", float, 'store', None), + 'dump-autopep8': ("Reformat easyconfigs using autopep8 when dumping them", None, 'store_true', False), 'easyblock': ("easyblock to use for processing the spec file or dumping the options", None, 'store', None, 'e', {'metavar': 'CLASS'}), - 'experimental': ("Allow experimental code (with behaviour that can be changed or removed at any given time).", + 'experimental': ("Allow experimental code (with behaviour that can be changed/removed at any given time).", None, 'store_true', False), 'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None), + 'group-writable-installdir': ("Enable group write permissions on installation directory after installation", + None, 'store_true', False), + 'hidden': ("Install 'hidden' module file(s) by prefixing their name with '.'", None, 'store_true', False), 'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False), 'filter-deps': ("Comma separated list of dependencies that you DON'T want to install with EasyBuild, " "because equivalent OS packages are installed. (e.g. --filter-deps=zlib,ncurses)", - str, 'extend', None), - 'oldstyleconfig': ("Look for and use the oldstyle configuration file.", - None, 'store_true', True), + 'strlist', 'extend', None), + 'hide-deps': ("Comma separated list of dependencies that you want automatically hidden, " + "(e.g. --hide-deps=zlib,ncurses)", 'strlist', 'extend', None), + 'module-only': ("Only generate module file(s); skip all steps except for %s" % ', '.join(MODULE_ONLY_STEPS), + None, 'store_true', False), + 'optarch': ("Set architecture optimization, overriding native architecture optimizations", + None, 'store', None), + 'parallel': ("Specify (maximum) level of parallellism used during build procedure", + 'int', 'store', None), 'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"), - None, 'store_true', False, 'p'), + None, 'store_true', False, 'p'), + 'read-only-installdir': ("Set read-only permissions on installation directory after installation", + None, 'store_true', False), 'set-gid-bit': ("Set group ID bit on newly created directories", None, 'store_true', False), 'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), 'umask': ("umask to use (e.g. '022'); non-user write permissions on install directories are removed", None, 'store', None), - 'optarch': ("Set architecture optimization, overriding native architecture optimizations", - None, 'store', None), + 'update-modules-tool-cache': ("Update modules tool cache file(s) after generating module file", + None, 'store_true', False), }) self.log.debug("override_options: descr %s opts %s" % (descr, opts)) @@ -185,60 +244,70 @@ def config_options(self): # config options descr = ("Configuration options", "Configure EasyBuild behavior.") - oldstyle_defaults = get_default_oldstyle_configfile_defaults() - opts = OrderedDict({ 'avail-module-naming-schemes': ("Show all supported module naming schemes", None, 'store_true', False,), 'avail-modules-tools': ("Show all supported module tools", None, "store_true", False,), 'avail-repositories': ("Show all repository types (incl. non-usable)", - None, "store_true", False,), - 'buildpath': ("Temporary build path", None, 'store', oldstyle_defaults['buildpath']), + None, "store_true", False,), + 'buildpath': ("Temporary build path", None, 'store', mk_full_default_path('buildpath')), + 'external-modules-metadata': ("List of files specifying metadata for external modules (INI format)", + 'strlist', 'store', []), 'ignore-dirs': ("Directory names to ignore when searching for files/dirs", 'strlist', 'store', ['.git', '.svn']), - 'installpath': ("Install path for software and modules", None, 'store', oldstyle_defaults['installpath']), - 'config': ("Path to EasyBuild config file", - None, 'store', oldstyle_defaults['config'], 'C'), + 'include-easyblocks': ("Location(s) of extra or customized easyblocks", 'strlist', 'store', []), + 'include-module-naming-schemes': ("Location(s) of extra or customized module naming schemes", + 'strlist', 'store', []), + 'include-toolchains': ("Location(s) of extra or customized toolchains or toolchain components", + 'strlist', 'store', []), + 'installpath': ("Install path for software and modules", + None, 'store', mk_full_default_path('installpath')), + 'installpath-modules': ("Install path for modules (if None, combine --installpath and --subdir-modules)", + None, 'store', None), + 'installpath-software': ("Install path for software (if None, combine --installpath and --subdir-software)", + None, 'store', None), + 'job-backend': ("Backend to use for submitting jobs", 'choice', 'store', + DEFAULT_JOB_BACKEND, sorted(avail_job_backends().keys())), + # purposely take a copy for the default logfile format 'logfile-format': ("Directory name and format of the log file", - 'strtuple', 'store', oldstyle_defaults['logfile_format'], {'metavar': 'DIR,FORMAT'}), - 'module-naming-scheme': ("Module naming scheme", - 'choice', 'store', oldstyle_defaults['module_naming_scheme'], - sorted(avail_module_naming_schemes().keys())), + 'strtuple', 'store', DEFAULT_LOGFILE_FORMAT[:], {'metavar': 'DIR,FORMAT'}), + 'module-naming-scheme': ("Module naming scheme to use", None, 'store', DEFAULT_MNS), + 'module-syntax': ("Syntax to be used for module files", 'choice', 'store', DEFAULT_MODULE_SYNTAX, + sorted(avail_module_generators().keys())), 'moduleclasses': (("Extend supported module classes " "(For more info on the default classes, use --show-default-moduleclasses)"), - None, 'extend', oldstyle_defaults['moduleclasses']), + None, 'extend', [x[0] for x in DEFAULT_MODULECLASSES]), 'modules-footer': ("Path to file containing footer to be added to all generated module files", None, 'store_or_None', None, {'metavar': "PATH"}), 'modules-tool': ("Modules tool to use", - 'choice', 'store', oldstyle_defaults['modules_tool'], - sorted(avail_modules_tools().keys())), + 'choice', 'store', DEFAULT_MODULES_TOOL, sorted(avail_modules_tools().keys())), + 'packagepath': ("The destination path for the packages built by package-tool", + None, 'store', mk_full_default_path('packagepath')), + 'package-naming-scheme': ("Packaging naming scheme choice", + 'choice', 'store', DEFAULT_PNS, sorted(avail_package_naming_schemes().keys())), 'prefix': (("Change prefix for buildpath, installpath, sourcepath and repositorypath " - "(repositorypath prefix is only relevant in case of FileRepository repository) " - "(used prefix for defaults %s)" % oldstyle_defaults['prefix']), - None, 'store', None), + "(used prefix for defaults %s)" % DEFAULT_PREFIX), + None, 'store', None), 'recursive-module-unload': ("Enable generating of modules that unload recursively.", None, 'store_true', False), 'repository': ("Repository type, using repositorypath", - 'choice', 'store', oldstyle_defaults['repository'], sorted(avail_repositories().keys())), + 'choice', 'store', DEFAULT_REPOSITORY, sorted(avail_repositories().keys())), 'repositorypath': (("Repository path, used by repository " "(is passed as list of arguments to create the repository instance). " "For more info, use --avail-repositories."), - 'strlist', 'store', - oldstyle_defaults['repositorypath'][oldstyle_defaults['repository']]), - 'show-default-moduleclasses': ("Show default module classes with description", - None, 'store_true', False), + 'strlist', 'store', self.default_repositorypath), 'sourcepath': ("Path(s) to where sources should be downloaded (string, colon-separated)", - None, 'store', oldstyle_defaults['sourcepath']), - 'subdir-modules': ("Installpath subdir for modules", None, 'store', oldstyle_defaults['subdir_modules']), - 'subdir-software': ("Installpath subdir for software", None, 'store', oldstyle_defaults['subdir_software']), + None, 'store', mk_full_default_path('sourcepath')), + 'subdir-modules': ("Installpath subdir for modules", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_modules']), + 'subdir-software': ("Installpath subdir for software", + None, 'store', DEFAULT_PATH_SUBDIRS['subdir_software']), 'suffix-modules-path': ("Suffix for module files install path", None, 'store', GENERAL_CLASS), # this one is sort of an exception, it's something jobscripts can set, # has no real meaning for regular eb usage 'testoutput': ("Path to where a job should place the output (to be set within jobscript)", - None, 'store', None), - 'tmp-logdir': ("Log directory where temporary log files are stored", - None, 'store', oldstyle_defaults['tmp_logdir']), + None, 'store', None), + 'tmp-logdir': ("Log directory where temporary log files are stored", None, 'store', None), 'tmpdir': ('Directory to use for temporary storage', None, 'store', None), }) @@ -250,16 +319,18 @@ def informative_options(self): descr = ("Informative options", "Obtain information about EasyBuild.") opts = OrderedDict({ + 'avail-cfgfile-constants': ("Show all constants that can be used in configuration files", + None, 'store_true', False), 'avail-easyconfig-constants': ("Show all constants that can be used in easyconfigs", None, 'store_true', False), 'avail-easyconfig-licenses': ("Show all license constants that can be used in easyconfigs", None, 'store_true', False), 'avail-easyconfig-params': (("Show all easyconfig parameters (include " "easyblock-specific ones by using -e)"), - None, "store_true", False, 'a'), + 'choice', 'store_or_None', FORMAT_TXT, [FORMAT_RST, FORMAT_TXT], 'a'), 'avail-easyconfig-templates': (("Show all template names and template constants " "that can be used in easyconfigs"), - None, 'store_true', False), + None, 'store_true', False), 'dep-graph': ("Create dependency graph", None, "store", None, {'metavar': 'depgraph.'}), 'list-easyblocks': ("Show list of available easyblocks", @@ -270,6 +341,9 @@ def informative_options(self): None, 'store', None, {'metavar': 'STR'}), 'search-short': ("Search for easyconfig files in the robot directory, print short paths", None, 'store', None, 'S', {'metavar': 'STR'}), + 'show-default-configfiles': ("Show list of default config files", None, 'store_true', False), + 'show-default-moduleclasses': ("Show default module classes with description", + None, 'store_true', False), }) self.log.debug("informative_options: descr %s opts %s" % (descr, opts)) @@ -288,9 +362,10 @@ def regtest_options(self): None, 'store_true', False), 'regtest-output-dir': ("Set output directory for test-run", None, 'store', None, {'metavar': 'DIR'}), + 'review-pr': ("Review specified pull request", int, 'store', None, {'metavar': 'PR#'}), 'sequential': ("Specify this option if you want to prevent parallel build", None, 'store_true', False), - 'upload-test-report': ("Upload full test report as a gist on GitHub", None, 'store_true', None), + 'upload-test-report': ("Upload full test report as a gist on GitHub", None, 'store_true', False), 'test-report-env-filter': ("Regex used to filter out variables in environment dump of test report", None, 'regex', None), }) @@ -298,6 +373,20 @@ def regtest_options(self): self.log.debug("regtest_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) + def package_options(self): + # package-related options + descr = ("Package options", "Control packaging performed by EasyBuild.") + + opts = OrderedDict({ + 'package': ("Enabling packaging", None, 'store_true', False), + 'package-tool': ("Packaging tool to use", None, 'store', DEFAULT_PKG_TOOL), + 'package-type': ("Type of package to generate", None, 'store', DEFAULT_PKG_TYPE), + 'package-release': ("Package release iteration number", None, 'store', DEFAULT_PKG_RELEASE), + }) + + self.log.debug("package_options: descr %s opts %s" % (descr, opts)) + self.add_group_parser(opts, descr) + def easyconfig_options(self): # easyconfig options (to be passed to easyconfig instance) descr = ("Options for Easyconfigs", "Options to be passed to all Easyconfig.") @@ -306,6 +395,22 @@ def easyconfig_options(self): self.log.debug("easyconfig_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr, prefix='easyconfig') + def job_options(self): + """Option related to --job.""" + descr = ("Options for job backend", "Options for job backend (only relevant when --job is used)") + + opts = OrderedDict({ + 'backend-config': ("Configuration file for job backend", None, 'store', None), + 'cores': ("Number of cores to request per job", 'int', 'store', None), + 'max-walltime': ("Maximum walltime for jobs (in hours)", 'int', 'store', 24), + 'output-dir': ("Output directory for jobs (default: current directory)", None, 'store', os.getcwd()), + 'polling-interval': ("Interval between polls for status of jobs (in seconds)", float, 'store', 30.0), + 'target-resource': ("Target resource for jobs", None, 'store', None), + }) + + self.log.debug("job_options: descr %s opts %s", descr, opts) + self.add_group_parser(opts, descr, prefix='job') + def easyblock_options(self): # easyblock options (to be passed to easyblock instance) descr = ("Options for Easyblocks", "Options to be passed to all Easyblocks.") @@ -327,31 +432,41 @@ def unittest_options(self): def validate(self): """Additional validation of options""" - stop_msg = [] + error_msgs = [] - if self.options.toolchain and not len(self.options.toolchain) == 2: - stop_msg.append('--toolchain requires NAME,VERSION (given %s)' % - (','.join(self.options.toolchain))) - if self.options.try_toolchain and not len(self.options.try_toolchain) == 2: - stop_msg.append('--try-toolchain requires NAME,VERSION (given %s)' % - (','.join(self.options.try_toolchain))) + for opt in ['software', 'try-software', 'toolchain', 'try-toolchain']: + val = getattr(self.options, opt.replace('-', '_')) + if val and len(val) != 2: + msg = "--%s requires NAME,VERSION (given %s)" % (opt, ','.join(val)) + error_msgs.append(msg) if self.options.umask: umask_regex = re.compile('^[0-7]{3}$') if not umask_regex.match(self.options.umask): - stop_msg.append("--umask value should be 3 digits (0-7) (regex pattern '%s')" % umask_regex.pattern) - - if len(stop_msg) > 0: - indent = " "*2 - stop_msg = ['%s%s' % (indent, x) for x in stop_msg] - stop_msg.insert(0, 'ERROR: Found %s problems validating the options:' % len(stop_msg)) - print "\n".join(stop_msg) - sys.exit(1) + msg = "--umask value should be 3 digits (0-7) (regex pattern '%s')" % umask_regex.pattern + error_msgs.append(msg) + + # subdir options must be relative + for typ in ['modules', 'software']: + subdir_opt = 'subdir_%s' % typ + val = getattr(self.options, subdir_opt) + if os.path.isabs(getattr(self.options, subdir_opt)): + msg = "Configuration option '%s' must specify a *relative* path (use 'installpath-%s' instead?): '%s'" + msg = msg % (subdir_opt, typ, val) + error_msgs.append(msg) + + # specified module naming scheme must be a known one + avail_mnss = avail_module_naming_schemes() + if self.options.module_naming_scheme and self.options.module_naming_scheme not in avail_mnss: + msg = "Selected module naming scheme '%s' is unknown: %s" % (self.options.module_naming_scheme, avail_mnss) + error_msgs.append(msg) + + if error_msgs: + raise EasyBuildError("Found problems validating the options: %s", '\n'.join(error_msgs)) def postprocess(self): """Do some postprocessing, in particular print stuff""" build_log.EXPERIMENTAL = self.options.experimental - config.SUPPORT_OLDSTYLE = self.options.oldstyleconfig # set strictness of run module if self.options.strict: @@ -365,65 +480,135 @@ def postprocess(self): if self.options.unittest_file: fancylogger.logToFile(self.options.unittest_file) + # set tmpdir + self.tmpdir = set_tmpdir(self.options.tmpdir) + + # take --include options into account + self._postprocess_include() + # prepare for --list/--avail if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates, - self.options.list_easyblocks, self.options.list_toolchains, + self.options.list_easyblocks, self.options.list_toolchains, self.options.avail_cfgfile_constants, self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses, self.options.avail_repositories, self.options.show_default_moduleclasses, self.options.avail_modules_tools, self.options.avail_module_naming_schemes, - ]): + self.options.show_default_configfiles, + ]): build_easyconfig_constants_dict() # runs the easyconfig constants sanity check self._postprocess_list_avail() # fail early if required dependencies for functionality requiring using GitHub API are not available: if self.options.from_pr or self.options.upload_test_report: if not HAVE_GITHUB_API: - self.log.error("Required support for using GitHub API is not available (see warnings).") + raise EasyBuildError("Required support for using GitHub API is not available (see warnings).") + + if self.options.module_syntax == ModuleGeneratorLua.SYNTAX and self.options.modules_tool != Lmod.__name__: + raise EasyBuildError("Generating Lua module files requires Lmod as modules tool.") # make sure a GitHub token is available when it's required if self.options.upload_test_report: if not HAVE_KEYRING: - self.log.error("Python 'keyring' module required for obtaining GitHub token is not available.") + raise EasyBuildError("Python 'keyring' module required for obtaining GitHub token is not available.") if self.options.github_user is None: - self.log.error("No GitHub user name provided, required for fetching GitHub token.") + raise EasyBuildError("No GitHub user name provided, required for fetching GitHub token.") token = fetch_github_token(self.options.github_user) if token is None: - self.log.error("Failed to obtain required GitHub token for user '%s'" % self.options.github_user) + raise EasyBuildError("Failed to obtain required GitHub token for user '%s'", self.options.github_user) + + # make sure autopep8 is available when it needs to be + if self.options.dump_autopep8: + if not HAVE_AUTOPEP8: + raise EasyBuildError("Python 'autopep8' module required to reformat dumped easyconfigs as requested") + + self._postprocess_external_modules_metadata() self._postprocess_config() + def _postprocess_external_modules_metadata(self): + """Parse file(s) specifying metadata for external modules.""" + # leave external_modules_metadata untouched if no files are provided + if not self.options.external_modules_metadata: + self.log.debug("No metadata provided for external modules.") + return + + parsed_external_modules_metadata = ConfigObj() + for path in self.options.external_modules_metadata: + if os.path.exists(path): + self.log.debug("Parsing %s with external modules metadata", path) + try: + parsed_external_modules_metadata.merge(ConfigObj(path)) + except ConfigObjError, err: + raise EasyBuildError("Failed to parse %s with external modules metadata: %s", path, err) + else: + raise EasyBuildError("Specified path for file with external modules metadata does not exist: %s", path) + + # make sure name/version values are always lists, make sure they're equal length + for mod, entry in parsed_external_modules_metadata.items(): + for key in ['name', 'version']: + if isinstance(entry.get(key), basestring): + entry[key] = [entry[key]] + self.log.debug("Transformed external module metadata value %s for %s into a single-value list: %s", + key, mod, entry[key]) + + # if both names and versions are available, lists must be of same length + names, versions = entry.get('name'), entry.get('version') + if names is not None and versions is not None and len(names) != len(versions): + raise EasyBuildError("Different length for lists of names/versions in metadata for external module %s: " + "names: %s; versions: %s", mod, names, versions) + + self.options.external_modules_metadata = parsed_external_modules_metadata + self.log.debug("External modules metadata: %s", self.options.external_modules_metadata) + + def _postprocess_include(self): + """Postprocess --include options.""" + # set up included easyblocks, module naming schemes and toolchains/toolchain components + if self.options.include_easyblocks: + include_easyblocks(self.tmpdir, self.options.include_easyblocks) + + if self.options.include_module_naming_schemes: + include_module_naming_schemes(self.tmpdir, self.options.include_module_naming_schemes) + + if self.options.include_toolchains: + include_toolchains(self.tmpdir, self.options.include_toolchains) + def _postprocess_config(self): """Postprocessing of configuration options""" if self.options.prefix is not None: - changed_defaults = get_default_oldstyle_configfile_defaults(self.options.prefix) - for dest in ['installpath', 'buildpath', 'sourcepath', 'repositorypath']: + # prefix applies to all paths, and repository has to be reinitialised to take new repositorypath in account + # in the legacy-style configuration, repository is initialised in configuration file itself + for dest in ['installpath', 'buildpath', 'sourcepath', 'repository', 'repositorypath', 'packagepath']: if not self.options._action_taken.get(dest, False): - new_def = changed_defaults[dest] - if dest == 'repositorypath': - setattr(self.options, dest, new_def[changed_defaults['repository']]) + if dest == 'repository': + setattr(self.options, dest, DEFAULT_REPOSITORY) + elif dest == 'repositorypath': + repositorypath = [mk_full_default_path(dest, prefix=self.options.prefix)] + setattr(self.options, dest, repositorypath) + self.go_cfg_constants[self.DEFAULTSECT]['DEFAULT_REPOSITORYPATH'] = repositorypath else: - setattr(self.options, dest, new_def) - # LEGACY this line is here for oldstyle reasons - self.log.deprecated('Fake action taken to distinguish from default', '2.0') + setattr(self.options, dest, mk_full_default_path(dest, prefix=self.options.prefix)) + # LEGACY this line is here for oldstyle config reasons self.options._action_taken[dest] = True if self.options.pretend: self.options.installpath = get_pretend_installpath() - # split supplied list of robot paths to obtain a list - if self.options.robot: - class RobotPath(ListOfStrings): - SEPARATOR_LIST = os.pathsep - # explicit definition of __str__ is required for unknown reason related to the way Wrapper is defined - __str__ = ListOfStrings.__str__ - self.options.robot = RobotPath(self.options.robot) + if self.options.robot is not None: + # paths specified to --robot have preference over --robot-paths + # keep both values in sync if robot is enabled, which implies enabling dependency resolver + self.options.robot_paths = self.options.robot + self.options.robot_paths + self.options.robot = self.options.robot_paths def _postprocess_list_avail(self): """Create all the additional info that can be requested (exit at the end)""" msg = '' + + # dump supported configuration file constants + if self.options.avail_cfgfile_constants: + msg += self.avail_cfgfile_constants() + # dump possible easyconfig params if self.options.avail_easyconfig_params: - msg += self.avail_easyconfig_params() + msg += avail_easyconfig_params(self.options.easyblock, self.options.avail_easyconfig_params) # dump easyconfig template options if self.options.avail_easyconfig_templates: @@ -457,6 +642,10 @@ def _postprocess_list_avail(self): if self.options.avail_module_naming_schemes: msg += self.avail_list('module naming schemes', avail_module_naming_schemes()) + # dump default list of config files that are considered + if self.options.show_default_configfiles: + msg += self.show_default_configfiles() + # dump default moduleclasses with description if self.options.show_default_moduleclasses: msg += self.show_default_moduleclasses() @@ -465,48 +654,47 @@ def _postprocess_list_avail(self): self.log.info(msg) else: print msg + + # cleanup tmpdir + try: + shutil.rmtree(self.tmpdir) + except OSError as err: + raise EasyBuildError("Failed to clean up temporary directory %s: %s", self.tmpdir, err) + sys.exit(0) - def avail_easyconfig_params(self): + def avail_cfgfile_constants(self): """ - Print the available easyconfig parameters, for the given easyblock. + Return overview of constants supported in configuration files. """ - app = get_easyblock_class(self.options.easyblock) - extra = app.extra_options() - mapping = convert_to_help(extra, has_default=False) - if len(extra) > 0: - ebb_msg = " (* indicates specific for the %s EasyBlock)" % app.__name__ - extra_names = [x[0] for x in extra] - else: - ebb_msg = '' - extra_names = [] - txt = ["Available easyconfig parameters%s" % ebb_msg] - params = [(k, v) for (k, v) in mapping.items() if k.upper() not in ['HIDDEN']] - for key, values in params: - txt.append("%s" % key.upper()) - txt.append('-' * len(key)) - for name, value in values: - tabs = "\t" * (3 - (len(name) + 1) / 8) - if name in extra_names: - starred = '(*)' - else: - starred = '' - txt.append("%s%s:%s%s" % (name, starred, tabs, value)) - txt.append('') - - return "\n".join(txt) - - def avail_classes_tree(self, classes, classNames, detailed, depth=0): + lines = [ + "Constants available (only) in configuration files:", + "syntax: %(CONSTANT_NAME)s", + ] + for section in self.go_cfg_constants: + lines.append('') + if section != self.DEFAULTSECT: + section_title = "only in '%s' section:" % section + lines.append(section_title) + for cst_name, (cst_value, cst_help) in sorted(self.go_cfg_constants[section].items()): + lines.append("* %s: %s [value: %s]" % (cst_name, cst_help, cst_value)) + return '\n'.join(lines) + + def avail_classes_tree(self, classes, class_names, locations, detailed, depth=0): """Print list of classes as a tree.""" txt = [] - for className in classNames: - classInfo = classes[className] + for class_name in class_names: + class_info = classes[class_name] if detailed: - txt.append("%s|-- %s (%s)" % ("| " * depth, className, classInfo['module'])) + mod = class_info['module'] + loc = '' + if mod in locations: + loc = '@ %s' % locations[mod] + txt.append("%s|-- %s (%s %s)" % ("| " * depth, class_name, mod, loc)) else: - txt.append("%s|-- %s" % ("| " * depth, className)) - if 'children' in classInfo: - txt.extend(self.avail_classes_tree(classes, classInfo['children'], detailed, depth + 1)) + txt.append("%s|-- %s" % ("| " * depth, class_name)) + if 'children' in class_info: + txt.extend(self.avail_classes_tree(classes, class_info['children'], locations, detailed, depth + 1)) return txt def avail_easyblocks(self): @@ -517,6 +705,7 @@ def avail_easyblocks(self): # finish initialisation of the toolchain module (ie set the TC_CONSTANT constants) search_toolchain('') + locations = {} for package in ["easybuild.easyblocks", "easybuild.easyblocks.generic"]: __import__(package) @@ -529,16 +718,21 @@ def avail_easyblocks(self): for f in os.listdir(path): res = module_regexp.match(f) if res: - __import__("%s.%s" % (package, res.group(1))) + easyblock = '%s.%s' % (package, res.group(1)) + if easyblock not in locations: + __import__(easyblock) + locations.update({easyblock: os.path.join(path, f)}) + else: + self.log.debug("%s already imported from %s, ignoring %s", + easyblock, locations[easyblock], path) def add_class(classes, cls): """Add a new class, and all of its subclasses.""" children = cls.__subclasses__() classes.update({cls.__name__: { - 'module': cls.__module__, - 'children': [x.__name__ for x in children] - } - }) + 'module': cls.__module__, + 'children': [x.__name__ for x in children] + }}) for child in children: add_class(classes, child) @@ -553,14 +747,18 @@ def add_class(classes, cls): for root in roots: root = root.__name__ if detailed: - txt.append("%s (%s)" % (root, classes[root]['module'])) + mod = classes[root]['module'] + loc = '' + if mod in locations: + loc = ' @ %s' % locations[mod] + txt.append("%s (%s%s)" % (root, mod, loc)) else: txt.append("%s" % root) if 'children' in classes[root]: - txt.extend(self.avail_classes_tree(classes, classes[root]['children'], detailed)) + txt.extend(self.avail_classes_tree(classes, classes[root]['children'], locations, detailed)) txt.append("") - return "\n".join(txt) + return '\n'.join(txt) def avail_toolchains(self): """Show list of known toolchains.""" @@ -572,14 +770,14 @@ def avail_toolchains(self): for (tcname, tcc) in tclist: tc = tcc(version='1.2.3') # version doesn't matter here, but something needs to be there - tc_elems = [e for es in tc.definition().values() for e in es] - txt.append("\t%s: %s" % (tcname, ', '.join(sorted(tc_elems)))) + tc_elems = nub(sorted([e for es in tc.definition().values() for e in es])) + txt.append("\t%s: %s" % (tcname, ', '.join(tc_elems))) return '\n'.join(txt) def avail_repositories(self): """Show list of known repository types.""" - repopath_defaults = get_default_oldstyle_configfile_defaults()['repositorypath'] + repopath_defaults = self.default_repositorypath all_repos = avail_repositories(check_useable=False) usable_repos = avail_repositories(check_useable=True).keys() @@ -605,14 +803,34 @@ def avail_list(self, name, items): """Show list of available values passed by argument.""" return "List of supported %s:\n\t%s" % (name, '\n\t'.join(items)) + def show_default_configfiles(self): + """Show list of default config files.""" + xdg_config_home = os.environ.get('XDG_CONFIG_HOME', '(not set)') + xdg_config_dirs = os.environ.get('XDG_CONFIG_DIRS', '(not set)') + system_cfg_glob_paths = os.path.join('{' + ', '.join(XDG_CONFIG_DIRS) + '}', 'easybuild.d', '*.cfg') + found_cfgfile_cnt = len(self.DEFAULT_CONFIGFILES) + found_cfgfile_list = ', '.join(self.DEFAULT_CONFIGFILES) or '(none)' + lines = [ + "Default list of configuration files:", + '', + "[with $XDG_CONFIG_HOME: %s, $XDG_CONFIG_DIRS: %s]" % (xdg_config_home, xdg_config_dirs), + '', + "* user-level: %s" % os.path.join('${XDG_CONFIG_HOME:-$HOME/.config}', 'easybuild', 'config.cfg'), + " -> %s => %s" % (DEFAULT_USER_CFGFILE, ('not found', 'found')[os.path.exists(DEFAULT_USER_CFGFILE)]), + "* system-level: %s" % os.path.join('${XDG_CONFIG_DIRS:-/etc}', 'easybuild.d', '*.cfg'), + " -> %s => %s" % (system_cfg_glob_paths, ', '.join(DEFAULT_SYS_CFGFILES) or "(no matches)"), + '', + "Default list of existing configuration files (%d): %s" % (found_cfgfile_cnt, found_cfgfile_list), + ] + return '\n'.join(lines) + def show_default_moduleclasses(self): """Show list of default moduleclasses and description.""" - txt = ["Default available moduleclasses"] - indent = " " * 2 + lines = ["Default available module classes:", ''] maxlen = max([len(x[0]) for x in DEFAULT_MODULECLASSES]) + 1 # at least 1 space for name, descr in DEFAULT_MODULECLASSES: - txt.append("%s%s:%s%s" % (indent, name, (" " * (maxlen - len(name))), descr)) - return "\n".join(txt) + lines.append("\t%s:%s%s" % (name, (" " * (maxlen - len(name))), descr)) + return '\n'.join(lines) def parse_options(args=None): @@ -626,7 +844,12 @@ def parse_options(args=None): description = ("Builds software based on easyconfig (or parse a directory).\n" "Provide one or more easyconfigs or directories, use -H or --help more information.") - eb_go = EasyBuildOptions(usage=usage, description=description, prog='eb', envvar_prefix='EASYBUILD', go_args=args) + try: + eb_go = EasyBuildOptions(usage=usage, description=description, prog='eb', envvar_prefix=CONFIG_ENV_VAR_PREFIX, + go_args=args, error_env_options=True, error_env_option_method=raise_easybuilderror) + except Exception as err: + raise EasyBuildError("Failed to parse configuration options: %s" % err) + return eb_go @@ -638,6 +861,7 @@ def process_software_build_specs(options): try_to_generate = False build_specs = {} + logger = fancylogger.getLogger() # regular options: don't try to generate easyconfig, and search opts_map = { @@ -668,17 +892,23 @@ def process_software_build_specs(options): # only when a try option is set do we enable generating easyconfigs try_to_generate = True - # process --toolchain --try-toolchain (sanity check done in tools.options) - tc = options.toolchain or options.try_toolchain - if tc: - if options.toolchain and options.try_toolchain: - print_warning("Ignoring --try-toolchain, only using --toolchain specification.") - elif options.try_toolchain: - try_to_generate = True - build_specs.update({ - 'toolchain_name': tc[0], - 'toolchain_version': tc[1], - }) + # process --(try-)software/toolchain + for opt in ['software', 'toolchain']: + val = getattr(options, opt) + tryval = getattr(options, 'try_%s' % opt) + if val or tryval: + if val and tryval: + logger.warning("Ignoring --try-%(opt)s, only using --%(opt)s specification" % {'opt': opt}) + elif tryval: + try_to_generate = True + val = val or tryval # --try-X value is overridden by --X + key_prefix = '' + if opt == 'toolchain': + key_prefix = 'toolchain_' + build_specs.update({ + '%sname' % key_prefix: val[0], + '%sversion' % key_prefix: val[1], + }) # provide both toolchain and toolchain_name/toolchain_version keys if 'toolchain_name' in build_specs: @@ -694,8 +924,8 @@ def process_software_build_specs(options): if options.amend: amends += options.amend if options.try_amend: - print_warning("Ignoring options passed via --try-amend, only using those passed via --amend.") - if options.try_amend: + logger.warning("Ignoring options passed via --try-amend, only using those passed via --amend.") + elif options.try_amend: amends += options.try_amend try_to_generate = True diff --git a/easybuild/tools/package/__init__.py b/easybuild/tools/package/__init__.py new file mode 100644 index 0000000000..51a62d3f44 --- /dev/null +++ b/easybuild/tools/package/__init__.py @@ -0,0 +1,38 @@ +## +# Copyright 2009-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +This declares the namespace for the tools.package submodule of EasyBuild, +which contains support for packaging and package naming schemes that can be overriden to cover site customizations. + +@author: Stijn De Weirdt (Ghent University) +@author: Dries Verdegem (Ghent University) +@author: Kenneth Hoste (Ghent University) +@author: Pieter De Baets (Ghent University) +@author: Jens Timmerman (Ghent University) +""" +from pkgutil import extend_path + +# we're not the only ones in this namespace +__path__ = extend_path(__path__, __name__) #@ReservedAssignment diff --git a/easybuild/tools/package/package_naming_scheme/__init__.py b/easybuild/tools/package/package_naming_scheme/__init__.py new file mode 100644 index 0000000000..6af2449881 --- /dev/null +++ b/easybuild/tools/package/package_naming_scheme/__init__.py @@ -0,0 +1,37 @@ +## +# Copyright 2009-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +This declares the namespace for the tools.package.package_naming_scheme submodule of EasyBuild. + +@author: Stijn De Weirdt (Ghent University) +@author: Dries Verdegem (Ghent University) +@author: Kenneth Hoste (Ghent University) +@author: Pieter De Baets (Ghent University) +@author: Jens Timmerman (Ghent University) +""" +from pkgutil import extend_path + +# we're not the only ones in this namespace +__path__ = extend_path(__path__, __name__) #@ReservedAssignment diff --git a/easybuild/tools/package/package_naming_scheme/easybuild_pns.py b/easybuild/tools/package/package_naming_scheme/easybuild_pns.py new file mode 100644 index 0000000000..d6698b551e --- /dev/null +++ b/easybuild/tools/package/package_naming_scheme/easybuild_pns.py @@ -0,0 +1,53 @@ +## +# Copyright 2015-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Implementation of the EasyBuild packaging naming scheme + +@author: Robert Schmidt (Ottawa Hospital Research Institute) +@author: Kenneth Hoste (Ghent University) +""" +from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version +from easybuild.tools.package.package_naming_scheme.pns import PackageNamingScheme +from easybuild.tools.version import VERSION as EASYBUILD_VERSION + + +class EasyBuildPNS(PackageNamingScheme): + """Class implmenting the default EasyBuild packaging naming scheme.""" + + def name(self, ec): + """Determine package name""" + self.log.debug("Easyconfig dict passed to name() looks like: %s ", ec) + return '%s-%s' % (ec['name'], det_full_ec_version(ec)) + + def version(self, ec): + """Determine package version: EasyBuild version used to build & install.""" + ebver = str(EASYBUILD_VERSION) + if ebver.endswith('dev'): + # try and make sure that 'dev' EasyBuild version is not considered newer just because it's longer + # (e.g., 2.2.0 vs 2.2.0dev) + # cfr. http://rpm.org/ticket/56, + # https://debian-handbook.info/browse/stable/sect.manipulating-packages-with-dpkg.html (see box in 5.4.3) + ebver.replace('dev', '~dev') + return 'eb-%s' % ebver diff --git a/easybuild/tools/package/package_naming_scheme/pns.py b/easybuild/tools/package/package_naming_scheme/pns.py new file mode 100644 index 0000000000..d66bce2262 --- /dev/null +++ b/easybuild/tools/package/package_naming_scheme/pns.py @@ -0,0 +1,58 @@ +## +# Copyright 2015-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## + +""" +Abstract implementation of a package naming scheme. + +@author: Robert Schmidt (Ottawa Hospital Research Institute) +@author: Kenneth Hoste (Ghent University) +""" +from abc import ABCMeta, abstractmethod +from vsc.utils import fancylogger + +from easybuild.tools.config import build_option + + +class PackageNamingScheme(object): + """Abstract class for package naming schemes""" + __metaclass__ = ABCMeta + + def __init__(self): + """initialize logger.""" + self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) + + @abstractmethod + def name(self, ec): + """Determine package name""" + pass + + @abstractmethod + def version(self, ec): + """Determine package version.""" + pass + + def release(self, ec=None): + """Determine package release""" + return build_option('package_release') diff --git a/easybuild/tools/package/utilities.py b/easybuild/tools/package/utilities.py new file mode 100644 index 0000000000..2e6977b160 --- /dev/null +++ b/easybuild/tools/package/utilities.py @@ -0,0 +1,193 @@ +## +# Copyright 2015-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## + +""" +Various utilities related to packaging support. + +@author: Marc Litherland (Novartis) +@author: Gianluca Santarossa (Novartis) +@author: Robert Schmidt (Ottawa Hospital Research Institute) +@author: Fotis Georgatos (Uni.Lu, NTUA) +@author: Kenneth Hoste (Ghent University) +""" +import os +import tempfile +import pprint + +from vsc.utils import fancylogger +from vsc.utils.missing import get_subclasses +from vsc.utils.patterns import Singleton + +from easybuild.tools.config import PKG_TOOL_FPM, PKG_TYPE_RPM, build_option, get_package_naming_scheme +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import which +from easybuild.tools.package.package_naming_scheme.pns import PackageNamingScheme +from easybuild.tools.run import run_cmd +from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME +from easybuild.tools.utilities import import_available_modules + + +_log = fancylogger.getLogger('tools.package') + + +def avail_package_naming_schemes(): + """ + Returns the list of valed naming schemes that are in the easybuild.package.package_naming_scheme namespace + """ + import_available_modules('easybuild.tools.package.package_naming_scheme') + class_dict = dict([(x.__name__, x) for x in get_subclasses(PackageNamingScheme)]) + return class_dict + + +def package(easyblock): + """ + Package installed software, according to active packaging configuration settings.""" + pkgtool = build_option('package_tool') + + if pkgtool == PKG_TOOL_FPM: + pkgdir = package_with_fpm(easyblock) + else: + raise EasyBuildError("Unknown packaging tool specified: %s", pkgtool) + + return pkgdir + + +def package_with_fpm(easyblock): + """ + This function will build a package using fpm and return the directory where the packages are + """ + workdir = tempfile.mkdtemp(prefix='eb-pkgs-') + pkgtype = build_option('package_type') + _log.info("Will be creating %s package(s) in %s", pkgtype, workdir) + + try: + origdir = os.getcwd() + os.chdir(workdir) + except OSError, err: + raise EasyBuildError("Failed to chdir into workdir %s: %s", workdir, err) + + package_naming_scheme = ActivePNS() + + pkgname = package_naming_scheme.name(easyblock.cfg) + pkgver = package_naming_scheme.version(easyblock.cfg) + pkgrel = package_naming_scheme.release(easyblock.cfg) + + _log.debug("Got the PNS values for (name, version, release): (%s, %s, %s)", pkgname, pkgver, pkgrel) + deps = [] + if easyblock.toolchain.name != DUMMY_TOOLCHAIN_NAME: + toolchain_dict = easyblock.toolchain.as_dict() + deps.extend([toolchain_dict]) + + deps.extend(easyblock.cfg.dependencies()) + + _log.debug("The dependencies to be added to the package are: %s", + pprint.pformat([easyblock.toolchain.as_dict()] + easyblock.cfg.dependencies())) + depstring = '' + for dep in deps: + _log.debug("The dep added looks like %s ", dep) + dep_pkgname = package_naming_scheme.name(dep) + depstring += " --depends '%s'" % dep_pkgname + + cmdlist = [ + PKG_TOOL_FPM, + '--workdir', workdir, + '--name', pkgname, + '--provides', pkgname, + '-t', pkgtype, # target + '-s', 'dir', # source + '--version', pkgver, + '--iteration', pkgrel, + depstring, + easyblock.installdir, + easyblock.module_generator.get_module_filepath(), + ] + cmd = ' '.join(cmdlist) + _log.debug("The flattened cmdlist looks like: %s", cmd) + run_cmd(cmd, log_all=True, simple=True) + + _log.info("Created %s package(s) in %s", pkgtype, workdir) + + try: + os.chdir(origdir) + except OSError, err: + raise EasyBuildError("Failed to chdir back to %s: %s", origdir, err) + + return workdir + + +def check_pkg_support(): + """Check whether packaging is supported, i.e. whether the required dependencies are available.""" + # packaging support is considered experimental for now (requires using --experimental) + _log.experimental("Support for packaging installed software.") + + pkgtool = build_option('package_tool') + pkgtool_path = which(pkgtool) + if pkgtool_path: + _log.info("Selected packaging tool '%s' found at %s", pkgtool, pkgtool_path) + + # rpmbuild is required for generating RPMs with FPM + if pkgtool == PKG_TOOL_FPM and build_option('package_type') == PKG_TYPE_RPM: + rpmbuild_path = which('rpmbuild') + if rpmbuild_path: + _log.info("Required tool 'rpmbuild' found at %s", rpmbuild_path) + else: + raise EasyBuildError("rpmbuild is required when generating RPM packages with FPM, but was not found") + + else: + raise EasyBuildError("Selected packaging tool '%s' not found", pkgtool) + + +class ActivePNS(object): + """ + The wrapper class for Package Naming Schemes. + """ + __metaclass__ = Singleton + + def __init__(self): + """Initialize logger and find available PNSes to load""" + self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) + + avail_pns = avail_package_naming_schemes() + sel_pns = get_package_naming_scheme() + if sel_pns in avail_pns: + self.pns = avail_pns[sel_pns]() + else: + raise EasyBuildError("Selected package naming scheme %s could not be found in %s", + sel_pns, avail_pns.keys()) + + def name(self, ec): + """Determine package name""" + name = self.pns.name(ec) + return name + + def version(self, ec): + """Determine package version""" + version = self.pns.version(ec) + return version + + def release(self, ec): + """Determine package release""" + release = self.pns.release() + return release diff --git a/easybuild/tools/parallelbuild.py b/easybuild/tools/parallelbuild.py index 92967dae82..e30b9203d2 100644 --- a/easybuild/tools/parallelbuild.py +++ b/easybuild/tools/parallelbuild.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,6 +34,7 @@ """ import math import os +import subprocess import easybuild.tools.config as config from easybuild.framework.easyblock import get_easyblock_instance @@ -41,108 +42,125 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import get_repository, get_repositorypath from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version -from easybuild.tools.pbs_job import PbsJob, connect_to_server, disconnect_from_server, get_ppn +from easybuild.tools.job.backend import job_backend from easybuild.tools.repository.repository import init_repository from vsc.utils import fancylogger _log = fancylogger.getLogger('parallelbuild', fname=False) +def _to_key(dep): + """Determine key for specified dependency.""" + return ActiveMNS().det_full_module_name(dep) -def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir=None): +def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir='easybuild-build', prepare_first=True): """ - easyconfigs is a list of easyconfigs which can be built (e.g. they have no unresolved dependencies) - this function will build them in parallel by submitting jobs + Build easyconfigs in parallel by submitting jobs to a batch-queuing system. + Return list of jobs submitted. + + Argument `easyconfigs` is a list of easyconfigs which can be + built: e.g. they have no unresolved dependencies. This function + will build them in parallel by submitting jobs. + @param build_command: build command to use @param easyconfigs: list of easyconfig files @param output_dir: output directory - returns the jobs + @param prepare_first: prepare by runnning fetch step first for each easyconfig """ _log.info("going to build these easyconfigs in parallel: %s", easyconfigs) - job_ids = {} + + active_job_backend = job_backend() + if active_job_backend is None: + raise EasyBuildError("Can not use --job if no job backend is available.") + + try: + active_job_backend.init() + except RuntimeError as err: + raise EasyBuildError("connection to server failed (%s: %s), can't submit jobs.", err.__class__.__name__, err) + # dependencies have already been resolved, # so one can linearly walk over the list and use previous job id's jobs = [] - # create a single connection, and reuse it - conn = connect_to_server() - if conn is None: - _log.error("connect_to_server returned %s, can't submit jobs." % (conn)) - - # determine ppn once, and pass is to each job being created - # this avoids having to figure out ppn over and over again, every time creating a temp connection to the server - ppn = get_ppn() + # keep track of which job builds which module + module_to_job = {} - def tokey(dep): - """Determine key for specified dependency.""" - return ActiveMNS().det_full_module_name(dep) - - for ec in easyconfigs: - # This is very important, otherwise we might have race conditions + for easyconfig in easyconfigs: + # this is very important, otherwise we might have race conditions # e.g. GCC-4.5.3 finds cloog.tar.gz but it was incorrectly downloaded by GCC-4.6.3 # running this step here, prevents this - prepare_easyconfig(ec) + if prepare_first: + prepare_easyconfig(easyconfig) # the new job will only depend on already submitted jobs - _log.info("creating job for ec: %s" % str(ec)) - new_job = create_job(build_command, ec, output_dir=output_dir, conn=conn, ppn=ppn) + _log.info("creating job for ec: %s" % easyconfig['ec']) + new_job = create_job(active_job_backend, build_command, easyconfig, output_dir=output_dir) - # sometimes unresolved_deps will contain things, not needed to be build - job_deps = [job_ids[dep] for dep in map(tokey, ec['unresolved_deps']) if dep in job_ids] - new_job.add_dependencies(job_deps) + # filter out dependencies marked as external modules + deps = [d for d in easyconfig['ec'].all_dependencies if not d.get('external_module', False)] - # place user hold on job to prevent it from starting too quickly, - # we might still need it in the queue to set it as a dependency for another job; - # only set hold for job without dependencies, other jobs have a dependency hold set anyway - with_hold = False - if not job_deps: - with_hold = True + dep_mod_names = map(ActiveMNS().det_full_module_name, deps) + job_deps = [module_to_job[dep] for dep in dep_mod_names if dep in module_to_job] # actually (try to) submit job - new_job.submit(with_hold) - _log.info("job for module %s has been submitted (job id: %s)" % (new_job.module, new_job.jobid)) + active_job_backend.queue(new_job, job_deps) + _log.info("job %s for module %s has been submitted", new_job, new_job.module) # update dictionary - job_ids[new_job.module] = new_job.jobid - new_job.cleanup() + module_to_job[new_job.module] = new_job jobs.append(new_job) - # release all user holds on jobs after submission is completed - for job in jobs: - if job.has_holds(): - _log.info("releasing hold on job %s" % job.jobid) - job.release_hold() - - disconnect_from_server(conn) + active_job_backend.complete() return jobs -def create_job(build_command, easyconfig, output_dir=None, conn=None, ppn=None): +def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True): + """ + Submit jobs. + @param ordered_ecs: list of easyconfigs, in the order they should be processed + @param cmd_line_opts: list of command line options (in 'longopt=value' form) + @param testing: If `True`, skip actual job submission + @param prepare_first: prepare by runnning fetch step first for each easyconfig """ - Creates a job, to build a *single* easyconfig + curdir = os.getcwd() + + # the options to ignore (help options can't reach here) + ignore_opts = ['robot', 'job'] + + # generate_cmd_line returns the options in form --longopt=value + opts = [x for x in cmd_line_opts if not x.split('=')[0] in ['--%s' % y for y in ignore_opts]] + + # compose string with command line options, properly quoted and with '%' characters escaped + opts_str = subprocess.list2cmdline(opts).replace('%', '%%') + + command = "unset TMPDIR && cd %s && eb %%(spec)s %s %%(add_opts)s --testoutput=%%(output_dir)s" % (curdir, opts_str) + _log.info("Command template for jobs: %s" % command) + job_info_lines = [] + if testing: + _log.debug("Skipping actual submission of jobs since testing mode is enabled") + else: + return build_easyconfigs_in_parallel(command, ordered_ecs, prepare_first=prepare_first) + + +def create_job(job_backend, build_command, easyconfig, output_dir='easybuild-build'): + """ + Creates a job to build a *single* easyconfig. + + @param job_backend: A factory object for querying server parameters and creating actual job objects @param build_command: format string for command, full path to an easyconfig file will be substituted in it @param easyconfig: easyconfig as processed by process_easyconfig - @param output_dir: optional output path; $EASYBUILDTESTOUTPUT will be set inside the job with this variable - @param conn: open connection to PBS server - @param ppn: ppn setting to use (# 'processors' (cores) per node to use) + @param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable + returns the job """ - if output_dir is None: - output_dir = 'easybuild-build' - - # create command based on build_command template - command = build_command % {'spec': easyconfig['spec']} - # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD easybuild_vars = {} for name in os.environ: if name.startswith("EASYBUILD"): easybuild_vars[name] = os.environ[name] - others = ["PYTHONPATH", "MODULEPATH"] - - for env_var in others: + for env_var in ["PYTHONPATH", "MODULEPATH"]: if env_var in os.environ: easybuild_vars[env_var] = os.environ[env_var] @@ -152,18 +170,27 @@ def create_job(build_command, easyconfig, output_dir=None, conn=None, ppn=None): ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec'])) name = '-'.join(ec_tuple) - var = config.OLDSTYLE_ENVIRONMENT_VARIABLES['test_output_path'] - easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name) + # determine whether additional options need to be passed to the 'eb' command + add_opts = '' + if easyconfig['hidden']: + add_opts += ' --hidden' + + # create command based on build_command template + command = build_command % { + 'add_opts': add_opts, + 'output_dir': os.path.join(os.path.abspath(output_dir), name), + 'spec': easyconfig['spec'], + } # just use latest build stats repo = init_repository(get_repository(), get_repositorypath()) buildstats = repo.get_buildstats(*ec_tuple) - resources = {} + extra = {} if buildstats: previous_time = buildstats[-1]['build_time'] - resources['hours'] = int(math.ceil(previous_time * 2 / 60)) + extra['hours'] = int(math.ceil(previous_time * 2 / 60)) - job = PbsJob(command, name, easybuild_vars, resources=resources, conn=conn, ppn=ppn) + job = job_backend.make_job(command, name, easybuild_vars, **extra) job.module = easyconfig['ec'].full_mod_name return job @@ -182,4 +209,4 @@ def prepare_easyconfig(ec): easyblock_instance.close_log() os.remove(easyblock_instance.logfile) except (OSError, EasyBuildError), err: - _log.error("An error occured while preparing %s: %s" % (ec, err)) + raise EasyBuildError("An error occured while preparing %s: %s", ec, err) diff --git a/easybuild/tools/repository/__init__.py b/easybuild/tools/repository/__init__.py index 25f9cc1ca8..ebdc3c10b6 100644 --- a/easybuild/tools/repository/__init__.py +++ b/easybuild/tools/repository/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2014 Ghent University +# Copyright 2011-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/filerepo.py b/easybuild/tools/repository/filerepo.py index 06e67f76bc..4cd314c08e 100644 --- a/easybuild/tools/repository/filerepo.py +++ b/easybuild/tools/repository/filerepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,7 +34,7 @@ @author: Jens Timmerman (Ghent University) @author: Toon Willems (Ghent University) @author: Ward Poelmans (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ import os import time @@ -78,7 +78,7 @@ def add_easyconfig(self, cfg, name, version, stats, previous): mkdir(full_path, parents=True) # destination - dest = os.path.join(full_path, "%s.eb" % version) + dest = os.path.join(full_path, "%s-%s.eb" % (name, version)) txt = "# Built with EasyBuild version %s on %s\n" % (VERBOSE_VERSION, time.strftime("%Y-%m-%d_%H-%M-%S")) diff --git a/easybuild/tools/repository/gitrepo.py b/easybuild/tools/repository/gitrepo.py index 8117856d85..c74cff6e54 100644 --- a/easybuild/tools/repository/gitrepo.py +++ b/easybuild/tools/repository/gitrepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,7 +34,7 @@ @author: Jens Timmerman (Ghent University) @author: Toon Willems (Ghent University) @author: Ward Poelmans (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ import getpass import os @@ -43,8 +43,11 @@ import time from vsc.utils import fancylogger +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import rmtree2 from easybuild.tools.repository.filerepo import FileRepository +from easybuild.tools.utilities import only_if_module_is_available +from easybuild.tools.version import VERSION _log = fancylogger.getLogger('gitrepo', fname=False) @@ -52,7 +55,7 @@ # failing imports are just ignored # a NameError should be catched where these are used -# GitPython +# GitPython (http://gitorious.org/git-python) try: import git from git import GitCommandError @@ -72,10 +75,11 @@ class GitRepository(FileRepository): USABLE = HAVE_GIT + @only_if_module_is_available('git', pkgname='GitPython') def __init__(self, *args): """ Initialize git client to None (will be set later) - All the real logic is in the setup_repo and create_wroking_copy methods + All the real logic is in the setup_repo and create_working_copy methods """ self.client = None FileRepository.__init__(self, *args) @@ -84,9 +88,6 @@ def setup_repo(self): """ Set up git repository. """ - if not HAVE_GIT: - self.log.error("It seems like GitPython is not available, which is required for Git support.") - self.wc = tempfile.mkdtemp(prefix='git-wc-') def create_working_copy(self): @@ -103,7 +104,7 @@ def create_working_copy(self): self.log.debug("rep name is %s" % reponame) except (git.GitCommandError, OSError), err: # it might already have existed - self.log.warning("Git local repo initialization failed, it might already exist: %s" % err) + self.log.warning("Git local repo initialization failed, it might already exist: %s", err) # local repo should now exist, let's connect to it again try: @@ -111,14 +112,14 @@ def create_working_copy(self): self.log.debug("connectiong to git repo in %s" % self.wc) self.client = git.Git(self.wc) except (git.GitCommandError, OSError), err: - self.log.error("Could not create a local git repo in wc %s: %s" % (self.wc, err)) + raise EasyBuildError("Could not create a local git repo in wc %s: %s", self.wc, err) # try to get the remote data in the local repo try: res = self.client.pull() self.log.debug("pulled succesfully to %s in %s" % (res, self.wc)) except (git.GitCommandError, OSError), err: - self.log.error("pull in working copy %s went wrong: %s" % (self.wc, err)) + raise EasyBuildError("pull in working copy %s went wrong: %s", self.wc, err) def add_easyconfig(self, cfg, name, version, stats, append): """ @@ -136,22 +137,24 @@ def commit(self, msg=None): """ Commit working copy to git repository """ - self.log.debug("committing in git: %s" % msg) - tup = (socket.gethostname(), time.strftime("%Y-%m-%d_%H-%M-%S"), getpass.getuser(), msg) - completemsg = "EasyBuild-commit from %s (time: %s, user: %s) \n%s" % tup + host = socket.gethostname() + timestamp = time.strftime("%Y-%m-%d_%H-%M-%S") + user = getpass.getuser() + completemsg = "%s with EasyBuild v%s @ %s (time: %s, user: %s)" % (msg, VERSION, host, timestamp, user) + self.log.debug("committing in git with message: %s" % msg) self.log.debug("git status: %s" % self.client.status()) try: - self.client.commit('-am "%s"' % completemsg) - self.log.debug("succesfull commit") + self.client.commit('-am %s' % completemsg) + self.log.debug("succesfull commit: %s", self.client.log('HEAD^!')) except GitCommandError, err: - self.log.warning("Commit from working copy %s (msg: %s) failed, empty commit?\n%s" % (self.wc, msg, err)) + self.log.warning("Commit from working copy %s failed, empty commit? (msg: %s): %s", self.wc, msg, err) try: info = self.client.push() - self.log.debug("push info: %s " % info) + self.log.debug("push info: %s ", info) except GitCommandError, err: - tup = (self.wc, self.repo, msg, err) - self.log.warning("Push from working copy %s to remote %s (msg: %s) failed: %s" % tup) + self.log.warning("Push from working copy %s to remote %s failed (msg: %s): %s", + self.wc, self.repo, msg, err) def cleanup(self): """ @@ -161,4 +164,4 @@ def cleanup(self): self.wc = os.path.dirname(self.wc) rmtree2(self.wc) except IOError, err: - self.log.error("Can't remove working copy %s: %s" % (self.wc, err)) + raise EasyBuildError("Can't remove working copy %s: %s", self.wc, err) diff --git a/easybuild/tools/repository/hgrepo.py b/easybuild/tools/repository/hgrepo.py index 571377a552..1e764d535b 100644 --- a/easybuild/tools/repository/hgrepo.py +++ b/easybuild/tools/repository/hgrepo.py @@ -43,6 +43,7 @@ import time from vsc.utils import fancylogger +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import rmtree2 from easybuild.tools.repository.filerepo import FileRepository @@ -88,7 +89,7 @@ def setup_repo(self): Set up mercurial repository. """ if not HAVE_HG: - self.log.error("The python-hglib Python module is not available, which is required for Mercurial support.") + raise EasyBuildError("python-hglib is not available, which is required for Mercurial support.") self.wc = tempfile.mkdtemp(prefix='hg-wc-') @@ -110,18 +111,18 @@ def create_working_copy(self): self.log.debug("connection to mercurial repo in %s" % self.wc) self.client = hglib.open(self.wc) except HgServerError, err: - self.log.error("Could not connect to local mercurial repo: %s" % err) + raise EasyBuildError("Could not connect to local mercurial repo: %s", err) except (HgCapabilityError, HgResponseError), err: - self.log.error("Server response: %s", err) + raise EasyBuildError("Server response: %s", err) except (OSError, ValueError), err: - self.log.error("Could not create a local mercurial repo in wc %s: %s" % (self.wc, err)) + raise EasyBuildError("Could not create a local mercurial repo in wc %s: %s", self.wc, err) # try to get the remote data in the local repo try: self.client.pull() self.log.debug("pulled succesfully in %s" % self.wc) except (HgCommandError, HgServerError, HgResponseError, OSError, ValueError), err: - self.log.error("pull in working copy %s went wrong: %s" % (self.wc, err)) + raise EasyBuildError("pull in working copy %s went wrong: %s", self.wc, err) def add_easyconfig(self, cfg, name, version, stats, append): """ @@ -167,4 +168,4 @@ def cleanup(self): try: rmtree2(self.wc) except IOError, err: - self.log.error("Can't remove working copy %s: %s" % (self.wc, err)) + raise EasyBuildError("Can't remove working copy %s: %s", self.wc, err) diff --git a/easybuild/tools/repository/repository.py b/easybuild/tools/repository/repository.py index b5d22c33a0..eea4247ffb 100644 --- a/easybuild/tools/repository/repository.py +++ b/easybuild/tools/repository/repository.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -32,11 +32,12 @@ @author: Jens Timmerman (Ghent University) @author: Toon Willems (Ghent University) @author: Ward Poelmans (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ from vsc.utils import fancylogger from vsc.utils.missing import get_subclasses +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.utilities import import_available_modules _log = fancylogger.getLogger('repository', fname=False) @@ -126,7 +127,7 @@ def avail_repositories(check_useable=True): class_dict = dict([(x.__name__, x) for x in get_subclasses(Repository) if x.USABLE or not check_useable]) if not 'FileRepository' in class_dict: - _log.error('avail_repositories: FileRepository missing from list of repositories') + raise EasyBuildError("avail_repositories: FileRepository missing from list of repositories") return class_dict @@ -144,13 +145,13 @@ def init_repository(repository, repository_path): elif isinstance(repository_path, (tuple, list)) and len(repository_path) <= 2: inited_repo = repo(*repository_path) else: - _log.error('repository_path should be a string or list/tuple of maximum 2 elements (current: %s, type %s)' % - (repository_path, type(repository_path))) + raise EasyBuildError("repository_path should be a string or list/tuple of maximum 2 elements " + "(current: %s, type %s)", repository_path, type(repository_path)) except Exception, err: - _log.error('Failed to create a repository instance for %s (class %s) with args %s (msg: %s)' % - (repository, repo.__name__, repository_path, err)) + raise EasyBuildError("Failed to create a repository instance for %s (class %s) with args %s (msg: %s)", + repository, repo.__name__, repository_path, err) else: - _log.error('Unknown typo of repository spec: %s (type %s)' % (repo, type(repo))) + raise EasyBuildError("Unknown typo of repository spec: %s (type %s)", repo, type(repo)) inited_repo.init() return inited_repo diff --git a/easybuild/tools/repository/svnrepo.py b/easybuild/tools/repository/svnrepo.py index 9058fb74f2..555981aa56 100644 --- a/easybuild/tools/repository/svnrepo.py +++ b/easybuild/tools/repository/svnrepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,7 +34,7 @@ @author: Jens Timmerman (Ghent University) @author: Toon Willems (Ghent University) @author: Ward Poelmans (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ import getpass import os @@ -43,14 +43,17 @@ import time from vsc.utils import fancylogger +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import rmtree2 from easybuild.tools.repository.filerepo import FileRepository +from easybuild.tools.utilities import only_if_module_is_available + _log = fancylogger.getLogger('svnrepo', fname=False) + # optional Python packages, these might be missing # failing imports are just ignored -# a NameError should be catched where these are used # PySVN try: @@ -58,7 +61,7 @@ from pysvn import ClientError # IGNORE:E0611 pysvn fails to recognize ClientError is available HAVE_PYSVN = True except ImportError: - _log.debug('Failed to import pysvn module') + _log.debug("Failed to import pysvn module") HAVE_PYSVN = False @@ -73,6 +76,7 @@ class SvnRepository(FileRepository): USABLE = HAVE_PYSVN + @only_if_module_is_available('pysvn', url='http://pysvn.tigris.org/') def __init__(self, *args): """ Set self.client to None. Real logic is in setup_repo and create_working_copy @@ -85,9 +89,6 @@ def setup_repo(self): Set up SVN repository. """ self.repo = os.path.join(self.repo, self.subdir) - if not HAVE_PYSVN: - self.log.error("pysvn not available. Make sure it is installed properly. " + - "Run 'python -c \"import pysvn\"' to test.") # try to connect to the repository self.log.debug("Try to connect to repository %s" % self.repo) @@ -95,13 +96,13 @@ def setup_repo(self): self.client = pysvn.Client() self.client.exception_style = 0 except ClientError: - self.log.error("Svn Client initialization failed.") + raise EasyBuildError("Svn Client initialization failed.") try: if not self.client.is_url(self.repo): - self.log.error("Provided repository %s is not a valid svn url" % self.repo) + raise EasyBuildError("Provided repository %s is not a valid svn url", self.repo) except ClientError: - self.log.error("Can't connect to svn repository %s" % self.repo) + raise EasyBuildError("Can't connect to svn repository %s", self.repo) def create_working_copy(self): """ @@ -114,16 +115,16 @@ def create_working_copy(self): try: self.client.info2(self.repo, recurse=False) except ClientError: - self.log.error("Getting info from %s failed." % self.wc) + raise EasyBuildError("Getting info from %s failed.", self.wc) try: res = self.client.update(self.wc) self.log.debug("Updated to revision %s in %s" % (res, self.wc)) except ClientError: - self.log.error("Update in wc %s went wrong" % self.wc) + raise EasyBuildError("Update in wc %s went wrong", self.wc) if len(res) == 0: - self.log.error("Update returned empy list (working copy: %s)" % (self.wc)) + raise EasyBuildError("Update returned empy list (working copy: %s)", self.wc) if res[0].number == -1: # revision number of update is -1 @@ -132,7 +133,7 @@ def create_working_copy(self): res = self.client.checkout(self.repo, self.wc) self.log.debug("Checked out revision %s in %s" % (res.number, self.wc)) except ClientError, err: - self.log.error("Checkout of path / in working copy %s went wrong: %s" % (self.wc, err)) + raise EasyBuildError("Checkout of path / in working copy %s went wrong: %s", self.wc, err) def add_easyconfig(self, cfg, name, version, stats, append): """ @@ -158,7 +159,7 @@ def commit(self, msg=None): try: self.client.checkin(self.wc, completemsg, recurse=True) except ClientError, err: - self.log.error("Commit from working copy %s (msg: %s) failed: %s" % (self.wc, msg, err)) + raise EasyBuildError("Commit from working copy %s (msg: %s) failed: %s", self.wc, msg, err) def cleanup(self): """ @@ -167,4 +168,4 @@ def cleanup(self): try: rmtree2(self.wc) except OSError, err: - self.log.error("Can't remove working copy %s: %s" % (self.wc, err)) + raise EasyBuildError("Can't remove working copy %s: %s", self.wc, err) diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py new file mode 100644 index 0000000000..9b82878931 --- /dev/null +++ b/easybuild/tools/robot.py @@ -0,0 +1,255 @@ +# # +# Copyright 2009-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Dependency resolution functionality, a.k.a. robot. + +@author: Stijn De Weirdt (Ghent University) +@author: Dries Verdegem (Ghent University) +@author: Kenneth Hoste (Ghent University) +@author: Pieter De Baets (Ghent University) +@author: Jens Timmerman (Ghent University) +@author: Toon Willems (Ghent University) +@author: Ward Poelmans (Ghent University) +""" +import os +from vsc.utils import fancylogger + +from easybuild.framework.easyconfig.easyconfig import ActiveMNS, process_easyconfig, robot_find_easyconfig +from easybuild.framework.easyconfig.tools import find_resolved_modules, skip_available +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option +from easybuild.tools.filetools import det_common_path_prefix, search_file +from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS +from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version +from easybuild.tools.modules import modules_tool + + +_log = fancylogger.getLogger('tools.robot', fname=False) + + +def det_robot_path(robot_paths_option, tweaked_ecs_path, pr_path, auto_robot=False): + """Determine robot path.""" + robot_path = robot_paths_option[:] + _log.info("Using robot path(s): %s" % robot_path) + + # paths to tweaked easyconfigs or easyconfigs downloaded from a PR have priority + if tweaked_ecs_path is not None: + robot_path.insert(0, tweaked_ecs_path) + _log.info("Prepended list of robot search paths with %s: %s" % (tweaked_ecs_path, robot_path)) + if pr_path is not None: + robot_path.insert(0, pr_path) + _log.info("Prepended list of robot search paths with %s: %s" % (pr_path, robot_path)) + + return robot_path + + +def dry_run(easyconfigs, short=False, build_specs=None): + """ + Compose dry run overview for supplied easyconfigs ([ ] for unavailable, [x] for available, [F] for forced) + @param easyconfigs: list of parsed easyconfigs (EasyConfig instances) + @param short: use short format for overview: use a variable for common prefixes + @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) + """ + lines = [] + if build_option('robot_path') is None: + lines.append("Dry run: printing build status of easyconfigs") + all_specs = easyconfigs + else: + lines.append("Dry run: printing build status of easyconfigs and dependencies") + all_specs = resolve_dependencies(easyconfigs, build_specs=build_specs, retain_all_deps=True) + + unbuilt_specs = skip_available(all_specs) + dry_run_fmt = " * [%1s] %s (module: %s)" # markdown compatible (list of items with checkboxes in front) + + listed_ec_paths = [spec['spec'] for spec in easyconfigs] + + var_name = 'CFGS' + common_prefix = det_common_path_prefix([spec['spec'] for spec in all_specs]) + # only allow short if common prefix is long enough + short = short and common_prefix is not None and len(common_prefix) > len(var_name) * 2 + for spec in all_specs: + if spec in unbuilt_specs: + ans = ' ' + elif build_option('force') and spec['spec'] in listed_ec_paths: + ans = 'F' + else: + ans = 'x' + + if spec['ec'].short_mod_name != spec['ec'].full_mod_name: + mod = "%s | %s" % (spec['ec'].mod_subdir, spec['ec'].short_mod_name) + else: + mod = spec['ec'].full_mod_name + + if short: + item = os.path.join('$%s' % var_name, spec['spec'][len(common_prefix) + 1:]) + else: + item = spec['spec'] + lines.append(dry_run_fmt % (ans, item, mod)) + + if short: + # insert after 'Dry run:' message + lines.insert(1, "%s=%s" % (var_name, common_prefix)) + return '\n'.join(lines) + + +def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False): + """ + Work through the list of easyconfigs to determine an optimal order + @param unprocessed: list of easyconfigs + @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) + @param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability; + retain all deps when True, check matching build option when False + """ + + robot = build_option('robot_path') + # retain all dependencies if specified by either the resp. build option or the dedicated named argument + retain_all_deps = build_option('retain_all_deps') or retain_all_deps + + if retain_all_deps: + # assume that no modules are available when forced, to retain all dependencies + avail_modules = [] + _log.info("Forcing all dependencies to be retained.") + else: + # Get a list of all available modules (format: [(name, installversion), ...]) + avail_modules = modules_tool().available() + + if len(avail_modules) == 0: + _log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH')) + + ordered_ecs = [] + # all available modules can be used for resolving dependencies except those that will be installed + being_installed = [p['full_mod_name'] for p in unprocessed] + avail_modules = [m for m in avail_modules if not m in being_installed] + + _log.debug('unprocessed before resolving deps: %s' % unprocessed) + + # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though) + irresolvable = [] + loopcnt = 0 + maxloopcnt = 10000 + while unprocessed: + # make sure this stops, we really don't want to get stuck in an infinite loop + loopcnt += 1 + if loopcnt > maxloopcnt: + raise EasyBuildError("Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)", + maxloopcnt, unprocessed, irresolvable) + + # first try resolving dependencies without using external dependencies + last_processed_count = -1 + while len(avail_modules) > last_processed_count: + last_processed_count = len(avail_modules) + res = find_resolved_modules(unprocessed, avail_modules, retain_all_deps=retain_all_deps) + more_ecs, unprocessed, avail_modules = res + for ec in more_ecs: + if not ec['full_mod_name'] in [x['full_mod_name'] for x in ordered_ecs]: + ordered_ecs.append(ec) + + # dependencies marked as external modules should be resolved via available modules at this point + missing_external_modules = [d['full_mod_name'] for ec in unprocessed for d in ec['dependencies'] + if d.get('external_module', False)] + if missing_external_modules: + raise EasyBuildError("Missing modules for one or more dependencies marked as external modules: %s", + missing_external_modules) + + # robot: look for existing dependencies, add them + if robot and unprocessed: + + # rely on EasyBuild module naming scheme when resolving dependencies, since we know that will + # generate sensible module names that include the necessary information for the resolution to work + # (name, version, toolchain, versionsuffix) + being_installed = [EasyBuildMNS().det_full_module_name(p['ec']) for p in unprocessed] + + additional = [] + for entry in unprocessed: + # do not choose an entry that is being installed in the current run + # if they depend, you probably want to rebuild them using the new dependency + deps = entry['dependencies'] + candidates = [d for d in deps if not EasyBuildMNS().det_full_module_name(d) in being_installed] + if candidates: + cand_dep = candidates[0] + # find easyconfig, might not find any + _log.debug("Looking for easyconfig for %s" % str(cand_dep)) + # note: robot_find_easyconfig may return None + path = robot_find_easyconfig(cand_dep['name'], det_full_ec_version(cand_dep)) + + if path is None: + # no easyconfig found for dependency, add to list of irresolvable dependencies + if cand_dep not in irresolvable: + _log.debug("Irresolvable dependency found: %s" % cand_dep) + irresolvable.append(cand_dep) + # remove irresolvable dependency from list of dependencies so we can continue + entry['dependencies'].remove(cand_dep) + else: + _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) + # build specs should not be passed down to resolved dependencies, + # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself + hidden = cand_dep.get('hidden', False) + processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden) + + # ensure that selected easyconfig provides required dependency + mods = [spec['ec'].full_mod_name for spec in processed_ecs] + dep_mod_name = ActiveMNS().det_full_module_name(cand_dep) + if not dep_mod_name in mods: + raise EasyBuildError("easyconfig file %s does not contain module %s (mods: %s)", + path, dep_mod_name, mods) + + for ec in processed_ecs: + if not ec in unprocessed + additional: + additional.append(ec) + _log.debug("Added %s as dependency of %s" % (ec, entry)) + else: + mod_name = EasyBuildMNS().det_full_module_name(entry['ec']) + _log.debug("No more candidate dependencies to resolve for %s" % mod_name) + + # add additional (new) easyconfigs to list of stuff to process + unprocessed.extend(additional) + _log.debug("Unprocessed dependencies: %s", unprocessed) + + elif not robot: + # no use in continuing if robot is not enabled, dependencies won't be resolved anyway + irresolvable = [dep for x in unprocessed for dep in x['dependencies']] + break + + if irresolvable: + _log.warning("Irresolvable dependencies (details): %s" % irresolvable) + irresolvable_mods_eb = [EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable] + _log.warning("Irresolvable dependencies (EasyBuild module names): %s" % ', '.join(irresolvable_mods_eb)) + irresolvable_mods = [ActiveMNS().det_full_module_name(dep) for dep in irresolvable] + raise EasyBuildError("Irresolvable dependencies encountered: %s", ', '.join(irresolvable_mods)) + + _log.info("Dependency resolution complete, building as follows: %s" % ordered_ecs) + return ordered_ecs + + +def search_easyconfigs(query, short=False): + """Search for easyconfigs, if a query is provided.""" + robot_path = build_option('robot_path') + if robot_path: + search_path = robot_path + else: + search_path = [os.getcwd()] + ignore_dirs = build_option('ignore_dirs') + silent = build_option('silent') + search_file(search_path, query, short=short, ignore_dirs=ignore_dirs, silent=silent) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 9b42b159fe..fe55d9d6a4 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -43,7 +43,7 @@ from vsc.utils import fancylogger from easybuild.tools.asyncprocess import PIPE, STDOUT, Popen, recv_some, send_all -import easybuild.tools.build_log # this import is required to obtain a correct (EasyBuild) logger! +from easybuild.tools.build_log import EasyBuildError _log = fancylogger.getLogger('run', fname=False) @@ -60,35 +60,12 @@ strictness = WARN -def adjust_cmd(func): - """Make adjustments to given command, if required.""" - - def inner(cmd, *args, **kwargs): - # SuSE hack - # - profile is not resourced, and functions (e.g. module) is not inherited - if 'PROFILEREAD' in os.environ and (len(os.environ['PROFILEREAD']) > 0): - filepaths = ['/etc/profile.d/modules.sh'] - extra = '' - for fp in filepaths: - if os.path.exists(fp): - extra = ". %s &&%s" % (fp, extra) - else: - _log.warning("Can't find file %s" % fp) - - cmd = "%s %s" % (extra, cmd) - - return func(cmd, *args, **kwargs) - - return inner - - -@adjust_cmd def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None): """ Executes a command cmd - returns exitcode and stdout+stderr (mixed) - no input though stdin - - if log_ok or log_all are set -> will log.error if non-zero exit-code + - if log_ok or log_all are set -> will raise EasyBuildError if non-zero exit-code - if simple is True -> instead of returning a tuple (output, ec) it will just return True or False signifying succes - inp is the input given to the command - regexp -> Regex used to check the output for errors. If True will use default (see parselogForError) @@ -119,7 +96,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, close_fds=True, executable="/bin/bash") except OSError, err: - _log.error("run_cmd init cmd %s failed:%s" % (cmd, err)) + raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err) if inp: p.stdin.write(inp) p.stdin.close() @@ -148,13 +125,12 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True try: os.chdir(cwd) except OSError, err: - _log.error("Failed to return to %s after executing command: %s" % (cwd, err)) + raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err) return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp) -@adjust_cmd -def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None): +def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None, maxhits=50): """ Executes a command cmd - looks for questions and tries to answer based on qa dictionary @@ -200,15 +176,15 @@ def process_QA(q, a_s): if regQ.search(q): return (a_s, regQ) else: - _log.error("runqanda: Question %s converted in %s does not match itself" % (q, regQtxt)) + raise EasyBuildError("runqanda: Question %s converted in %s does not match itself", q, regQtxt) def check_answers_list(answers): """Make sure we have a list of answers (as strings).""" if isinstance(answers, basestring): answers = [answers] elif not isinstance(answers, list): - msg = "Invalid type for answer on %s, no string or list: %s (%s)" % (question, type(answers), answers) - _log.error(msg) + raise EasyBuildError("Invalid type for answer on %s, no string or list: %s (%s)", + question, type(answers), answers) # list is manipulated when answering matching question, so return a copy return answers[:] @@ -247,16 +223,14 @@ def check_answers_list(answers): _log.debug('run_cmd_qa: Command output will be logged to %s' % runLog.name) runLog.write(cmd + "\n\n") except IOError, err: - _log.error("Opening log file for Q&A failed: %s" % err) + raise EasyBuildError("Opening log file for Q&A failed: %s", err) else: runLog = None - maxHitCount = 50 - try: p = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT, stdin=PIPE, close_fds=True, executable="/bin/bash") except OSError, err: - _log.error("run_cmd_qa init cmd %s failed:%s" % (cmd, err)) + raise EasyBuildError("run_cmd_qa init cmd %s failed:%s", cmd, err) ec = p.poll() stdoutErr = '' @@ -320,7 +294,7 @@ def check_answers_list(answers): else: hitCount = 0 - if hitCount > maxHitCount: + if hitCount > maxhits: # explicitly kill the child process before exiting try: os.killpg(p.pid, signal.SIGKILL) @@ -328,8 +302,8 @@ def check_answers_list(answers): except OSError, err: _log.debug("run_cmd_qa exception caught when killing child process: %s" % err) _log.debug("run_cmd_qa: full stdouterr: %s" % stdoutErr) - _log.error("run_cmd_qa: cmd %s : Max nohits %s reached: end of output %s" % - (cmd, maxHitCount, stdoutErr[-500:])) + raise EasyBuildError("run_cmd_qa: cmd %s : Max nohits %s reached: end of output %s", + cmd, maxhits, stdoutErr[-500:]) # the sleep below is required to avoid exiting on unknown 'questions' too early (see above) time.sleep(1) @@ -351,7 +325,7 @@ def check_answers_list(answers): try: os.chdir(cwd) except OSError, err: - _log.error("Failed to return to %s after executing command: %s" % (cwd, err)) + raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err) return parse_cmd_output(cmd, stdoutErr, ec, simple, log_all, log_ok, regexp) @@ -370,24 +344,23 @@ def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp): check_ec = True use_regexp = True else: - _log.error("invalid strictness setting: %s" % strictness) + raise EasyBuildError("invalid strictness setting: %s", strictness) # allow for overriding the regexp setting if not regexp: use_regexp = False + _log.debug('cmd "%s" exited with exitcode %s and output:\n%s' % (cmd, ec, stdouterr)) + if ec and (log_all or log_ok): # We don't want to error if the user doesn't care if check_ec: - _log.error('cmd "%s" exited with exitcode %s and output:\n%s' % (cmd, ec, stdouterr)) + raise EasyBuildError('cmd "%s" exited with exitcode %s and output:\n%s', cmd, ec, stdouterr) else: _log.warn('cmd "%s" exited with exitcode %s and output:\n%s' % (cmd, ec, stdouterr)) - - if not ec: + elif not ec: if log_all: _log.info('cmd "%s" exited with exitcode %s and output:\n%s' % (cmd, ec, stdouterr)) - else: - _log.debug('cmd "%s" exited with exitcode %s and output:\n%s' % (cmd, ec, stdouterr)) # parse the stdout/stderr for errors when strictness dictates this or when regexp is passed in if use_regexp or regexp: @@ -395,7 +368,7 @@ def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp): if len(res) > 0: message = "Found %s errors in command output (output: %s)" % (len(res), ", ".join([r[0] for r in res])) if use_regexp: - _log.error(message) + raise EasyBuildError(message) else: _log.warn(message) @@ -425,7 +398,7 @@ def parse_log_for_error(txt, regExp=None, stdout=True, msg=None): elif type(regExp) == str: pass else: - _log.error("parse_log_for_error no valid regExp used: %s" % regExp) + raise EasyBuildError("parse_log_for_error no valid regExp used: %s", regExp) reg = re.compile(regExp, re.I) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 7cd2babc04..8f3c5891d0 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2014 Ghent University +# Copyright 2011-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,20 +28,20 @@ @author: Jens Timmerman (Ghent University) @auther: Ward Poelmans (Ghent University) """ +import fcntl import grp # @UnresolvedImport import os import platform import pwd import re +import struct import sys +import termios from socket import gethostname from vsc.utils import fancylogger -try: - # this import fails with Python 2.4 because it requires the ctypes module (only in Python 2.5+) - from vsc.utils.affinity import sched_getaffinity -except ImportError: - pass +from vsc.utils.affinity import sched_getaffinity +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file, which from easybuild.tools.run import run_cmd @@ -51,13 +51,26 @@ # constants AMD = 'AMD' ARM = 'ARM' +IBM = 'IBM' INTEL = 'Intel' +POWER = 'POWER' LINUX = 'Linux' DARWIN = 'Darwin' UNKNOWN = 'UNKNOWN' +MAX_FREQ_FP = '/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq' +PROC_CPUINFO_FP = '/proc/cpuinfo' + +CPU_FAMILIES = [ARM, AMD, INTEL, POWER] +VENDORS = { + 'ARM': ARM, + 'AuthenticAMD': AMD, + 'GenuineIntel': INTEL, + 'IBM': IBM, +} + class SystemToolsException(Exception): """raised when systemtools fails""" @@ -67,146 +80,127 @@ def get_avail_core_count(): """ Returns the number of available CPUs, according to cgroups and taskssets limits """ - # tiny inner function to help figure out number of available cores in a cpuset - def count_bits(n): - """Count the number of set bits for a given integer.""" - bit_cnt = 0 - while n > 0: - n &= n - 1 - bit_cnt += 1 - return bit_cnt - + core_cnt = None os_type = get_os_type() - if os_type == LINUX: - try: - # the preferred approach is via sched_getaffinity (yields a long, so cast it down to int) - num_cores = int(sum(sched_getaffinity().cpus)) - return num_cores - except NameError: - pass - # in case sched_getaffinity isn't available, fall back to relying on /proc/cpuinfo - - # determine total number of cores via /proc/cpuinfo - try: - txt = read_file('/proc/cpuinfo', log_error=False) - # sometimes this is uppercase - max_num_cores = txt.lower().count('processor\t:') - except IOError, err: - raise SystemToolsException("An error occured while determining total core count: %s" % err) - - # determine cpuset we're in (if any) - mypid = os.getpid() - try: - f = open("/proc/%s/status" % mypid, 'r') - txt = f.read() - f.close() - cpuset = re.search("^Cpus_allowed:\s*([0-9,a-f]+)", txt, re.M | re.I) - except IOError: - cpuset = None - - if cpuset is not None: - # use cpuset mask to determine actual number of available cores - mask_as_int = long(cpuset.group(1).replace(',', ''), 16) - num_cores_in_cpuset = count_bits((2**max_num_cores - 1) & mask_as_int) - _log.info("In cpuset with %s CPUs" % num_cores_in_cpuset) - return num_cores_in_cpuset - else: - _log.debug("No list of allowed CPUs found, not in a cpuset.") - return max_num_cores + if os_type == LINUX: + # simple use available sched_getaffinity() function (yields a long, so cast it down to int) + core_cnt = int(sum(sched_getaffinity().cpus)) else: - # BSD + # BSD-type systems + out, _ = run_cmd('sysctl -n hw.ncpu') try: - out, _ = run_cmd('sysctl -n hw.ncpu') - num_cores = int(out) - if num_cores > 0: - return num_cores + if int(out) > 0: + core_cnt = int(out) except ValueError: pass - raise SystemToolsException('Can not determine number of cores on this system') + if core_cnt is None: + raise SystemToolsException('Can not determine number of cores on this system') + else: + return core_cnt def get_core_count(): - """ - Try to detect the number of virtual or physical CPUs on this system - (DEPRECATED, use get_avail_core_count instead) - """ - _log.deprecated("get_core_count() is deprecated, use get_avail_core_count() instead", '2.0') - return get_avail_core_count() + """NO LONGER SUPPORTED: use get_avail_core_count() instead""" + _log.nosupport("get_core_count() is replaced by get_avail_core_count()", '2.0') def get_cpu_vendor(): - """Try to detect the cpu identifier + """ + Try to detect the CPU vendor - will return INTEL, ARM or AMD constant + @return: a value from the VENDORS dict """ - regexp = re.compile(r"^vendor_id\s+:\s*(?P\S+)\s*$", re.M) - VENDORS = { - 'GenuineIntel': INTEL, - 'AuthenticAMD': AMD, - } + vendor = None os_type = get_os_type() - if os_type == LINUX: - try: - txt = read_file('/proc/cpuinfo', log_error=False) - arch = UNKNOWN - # vendor_id might not be in the /proc/cpuinfo, so this might fail - res = regexp.search(txt) - if res: - arch = res.groupdict().get('vendorid', UNKNOWN) - if arch in VENDORS: - return VENDORS[arch] + if os_type == LINUX and os.path.exists(PROC_CPUINFO_FP): + txt = read_file(PROC_CPUINFO_FP) + arch = UNKNOWN - # some embeded linux on arm behaves differently (e.g. raspbian) - regexp = re.compile(r"^Processor\s+:\s*(?PARM\S+)\s*", re.M) - res = regexp.search(txt) - if res: - arch = res.groupdict().get('vendorid', UNKNOWN) - if ARM in arch: - return ARM - except IOError, err: - raise SystemToolsException("An error occured while determining CPU vendor since: %s" % err) + vendor_regex = re.compile(r"(vendor_id.*?)?\s*:\s*(?P(?(1)\S+|(?:IBM|ARM)))") + res = vendor_regex.search(txt) + if res: + arch = res.group('vendor') + if arch in VENDORS: + vendor = VENDORS[arch] + _log.debug("Determined CPU vendor on Linux as being '%s' via regex '%s' in %s", + vendor, vendor_regex.pattern, PROC_CPUINFO_FP) elif os_type == DARWIN: - out, exitcode = run_cmd("sysctl -n machdep.cpu.vendor") + cmd = "sysctl -n machdep.cpu.vendor" + out, ec = run_cmd(cmd) out = out.strip() - if not exitcode and out and out in VENDORS: - return VENDORS[out] + if ec == 0 and out in VENDORS: + vendor = VENDORS[out] + _log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd)) + + if vendor is None: + vendor = UNKNOWN + _log.warning("Could not determine CPU vendor on %s, returning %s" % (os_type, vendor)) + + return vendor + + +def get_cpu_family(): + """ + Determine CPU family. + @return: a value from the CPU_FAMILIES list + """ + family = None + vendor = get_cpu_vendor() + if vendor in CPU_FAMILIES: + family = vendor + _log.debug("Using vendor as CPU family: %s" % family) else: - # BSD - out, exitcode = run_cmd("sysctl -n hw.model") - out = out.strip() - if not exitcode and out: - return out.split(' ')[0] + # POWER family needs to be determined indirectly via 'cpu' in /proc/cpuinfo + if os.path.exists(PROC_CPUINFO_FP): + cpuinfo_txt = read_file(PROC_CPUINFO_FP) + power_regex = re.compile(r"^cpu\s+:\s*POWER.*", re.M) + if power_regex.search(cpuinfo_txt): + family = POWER + _log.debug("Determined CPU family using regex '%s' in %s: %s", + power_regex.pattern, PROC_CPUINFO_FP, family) + + if family is None: + family = UNKNOWN + _log.warning("Failed to determine CPU family, returning %s" % family) - return UNKNOWN + return family def get_cpu_model(): """ - returns cpu model - f.ex Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz + Determine CPU model, e.g., Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz """ + model = None os_type = get_os_type() - if os_type == LINUX: - regexp = re.compile(r"^model name\s+:\s*(?P.+)\s*$", re.M) - try: - txt = read_file('/proc/cpuinfo', log_error=False) - if txt is not None: - return regexp.search(txt).groupdict()['modelname'].strip() - except IOError, err: - raise SystemToolsException("An error occured when determining CPU model: %s" % err) + + if os_type == LINUX and os.path.exists(PROC_CPUINFO_FP): + # we need 'model name' on Linux/x86, but 'model' is there first with different info + # 'model name' is not there for Linux/POWER, but 'model' has the right info + model_regex = re.compile(r"^model(?:\s+name)?\s+:\s*(?P.*[A-Za-z].+)\s*$", re.M) + txt = read_file(PROC_CPUINFO_FP) + res = model_regex.search(txt) + if res is not None: + model = res.group('model').strip() + _log.debug("Determined CPU model on Linux using regex '%s' in %s: %s", + model_regex.pattern, PROC_CPUINFO_FP, model) elif os_type == DARWIN: - out, exitcode = run_cmd("sysctl -n machdep.cpu.brand_string") - out = out.strip() - if not exitcode: - return out + cmd = "sysctl -n machdep.cpu.brand_string" + out, ec = run_cmd(cmd) + if ec == 0: + model = out.strip() + _log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model)) - return UNKNOWN + if model is None: + model = UNKNOWN + _log.warning("Failed to determine CPU model, returning %s" % model) + + return model def get_cpu_speed(): @@ -214,61 +208,48 @@ def get_cpu_speed(): Returns the (maximum) cpu speed in MHz, as a float value. In case of throttling, the highest cpu speed is returns. """ + cpu_freq = None os_type = get_os_type() - if os_type == LINUX: - try: - # Linux with cpu scaling - max_freq_fp = '/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq' - try: - f = open(max_freq_fp, 'r') - cpu_freq = float(f.read())/1000 - f.close() - return cpu_freq - except IOError, err: - _log.warning("Failed to read %s to determine max. CPU clock frequency with CPU scaling: %s" % (max_freq_fp, err)) - - # Linux without cpu scaling - cpuinfo_fp = '/proc/cpuinfo' - try: - cpu_freq = None - f = open(cpuinfo_fp, 'r') - for line in f: - cpu_freq = re.match("^cpu MHz\s*:\s*([0-9.]+)", line) - if cpu_freq is not None: - break - f.close() - if cpu_freq is None: - raise SystemToolsException("Failed to determine CPU frequency from %s" % cpuinfo_fp) - else: - return float(cpu_freq.group(1)) - except IOError, err: - _log.warning("Failed to read %s to determine CPU clock frequency: %s" % (cpuinfo_fp, err)) - except (IOError, OSError), err: - raise SystemToolsException("Determining CPU speed failed, exception occured: %s" % err) + if os_type == LINUX: + # Linux with cpu scaling + if os.path.exists(MAX_FREQ_FP): + _log.debug("Trying to determine CPU frequency on Linux via %s" % MAX_FREQ_FP) + txt = read_file(MAX_FREQ_FP) + cpu_freq = float(txt)/1000 + + # Linux without cpu scaling + elif os.path.exists(PROC_CPUINFO_FP): + _log.debug("Trying to determine CPU frequency on Linux via %s" % PROC_CPUINFO_FP) + cpuinfo_txt = read_file(PROC_CPUINFO_FP) + # 'cpu MHz' on Linux/x86 (& more), 'clock' on Linux/POWER + cpu_freq_regex = re.compile(r"^(?:cpu MHz|clock)\s*:\s*(?P\d+(?:\.\d+)?)", re.M) + res = cpu_freq_regex.search(cpuinfo_txt) + if res: + cpu_freq = float(res.group('cpu_freq')) + _log.debug("Found CPU frequency using regex '%s': %s" % (cpu_freq_regex.pattern, cpu_freq)) + else: + _log.debug("Failed to determine CPU frequency from %s", PROC_CPUINFO_FP) + else: + _log.debug("%s not found to determine max. CPU clock frequency without CPU scaling: %s" % PROC_CPUINFO_FP) elif os_type == DARWIN: - # OS X - out, ec = run_cmd("sysctl -n hw.cpufrequency_max") - # returns clock frequency in cycles/sec, but we want MHz - mhz = float(out.strip())/(1000**2) + cmd = "sysctl -n hw.cpufrequency_max" + _log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" % cmd) + out, ec = run_cmd(cmd) if ec == 0: - return mhz + # returns clock frequency in cycles/sec, but we want MHz + cpu_freq = float(out.strip())/(1000**2) - raise SystemToolsException("Could not determine CPU clock frequency (OS: %s)." % os_type) + else: + raise SystemToolsException("Could not determine CPU clock frequency (OS: %s)." % os_type) + return cpu_freq -def get_kernel_name(): - """Try to determine kernel name - e.g., 'Linux', 'Darwin', ... - """ - _log.deprecated("get_kernel_name() (replaced by get_os_type())", "2.0") - try: - kernel_name = os.uname()[0] - return kernel_name - except OSError, err: - raise SystemToolsException("Failed to determine kernel name: %s" % err) +def get_kernel_name(): + """NO LONGER SUPPORTED: use get_os_type() instead""" + _log.nosupport("get_kernel_name() is replaced by get_os_type()", '2.0') def get_os_type(): @@ -326,15 +307,9 @@ def get_os_name(): Determine system name, e.g., 'redhat' (generic), 'centos', 'debian', 'fedora', 'suse', 'ubuntu', 'red hat enterprise linux server', 'SL' (Scientific Linux), 'opensuse', ... """ - try: - # platform.linux_distribution is more useful, but only available since Python 2.6 - # this allows to differentiate between Fedora, CentOS, RHEL and Scientific Linux (Rocks is just CentOS) - os_name = platform.linux_distribution()[0].strip().lower() - except AttributeError: - # platform.dist can be used as a fallback - # CentOS, RHEL, Rocks and Scientific Linux may all appear as 'redhat' (especially if Python version is pre v2.6) - os_name = platform.dist()[0].strip().lower() - _log.deprecated("platform.dist as fallback for platform.linux_distribution", "2.0") + # platform.linux_distribution is more useful, but only available since Python 2.6 + # this allows to differentiate between Fedora, CentOS, RHEL and Scientific Linux (Rocks is just CentOS) + os_name = platform.linux_distribution()[0].strip().lower() os_name_map = { 'red hat enterprise linux server': 'RHEL', @@ -377,7 +352,7 @@ def get_os_version(): if not known_sp: suff = '_UNKNOWN_SP' else: - _log.error("Don't know how to determine subversions for SLES %s" % os_version) + raise EasyBuildError("Don't know how to determine subversions for SLES %s", os_version) return os_version else: @@ -391,27 +366,24 @@ def check_os_dependency(dep): # - uses rpm -q and dpkg -s --> can be run as non-root!! # - fallback on which # - should be extended to files later? + found = None cmd = None - if get_os_name() in ['debian', 'ubuntu']: - if which('dpkg'): - cmd = "dpkg -s %s" % dep - else: - # OK for get_os_name() == redhat, fedora, RHEL, SL, centos - if which('rpm'): - cmd = "rpm -q %s" % dep + if which('rpm'): + cmd = "rpm -q %s" % dep + found = run_cmd(cmd, simple=True, log_all=False, log_ok=False) - found = None - if cmd is not None: + if not found and which('dpkg'): + cmd = "dpkg -s %s" % dep found = run_cmd(cmd, simple=True, log_all=False, log_ok=False) - if found is None: + if cmd is None: # fallback for when os-dependency is a binary/library found = which(dep) - # try locate if it's available - if found is None and which('locate'): - cmd = 'locate --regexp "/%s$"' % dep - found = run_cmd(cmd, simple=True, log_all=False, log_ok=False) + # try locate if it's available + if not found and which('locate'): + cmd = 'locate --regexp "/%s$"' % dep + found = run_cmd(cmd, simple=True, log_all=False, log_ok=False) return found @@ -445,8 +417,8 @@ def get_glibc_version(): _log.debug("Found glibc version %s" % glibc_version) return glibc_version else: - tup = (glibc_ver_str, glibc_ver_regex.pattern) - _log.error("Failed to determine version from '%s' using pattern '%s'." % tup) + raise EasyBuildError("Failed to determine glibc version from '%s' using pattern '%s'.", + glibc_ver_str, glibc_ver_regex.pattern) else: # no glibc on OS X standard _log.debug("No glibc on a non-Linux system, so can't determine version.") @@ -464,7 +436,6 @@ def get_system_info(): 'gcc_version': get_tool_version('gcc', version_option='-v'), 'hostname': gethostname(), 'glibc_version': get_glibc_version(), - 'kernel_name': get_kernel_name(), 'os_name': get_os_name(), 'os_type': get_os_type(), 'os_version': get_os_version(), @@ -480,7 +451,7 @@ def use_group(group_name): try: group_id = grp.getgrnam(group_name).gr_gid except KeyError, err: - _log.error("Failed to get group ID for '%s', group does not exist (err: %s)" % (group_name, err)) + raise EasyBuildError("Failed to get group ID for '%s', group does not exist (err: %s)", group_name, err) group = (group_name, group_id) try: @@ -493,13 +464,13 @@ def use_group(group_name): err_msg += "change the primary group before using EasyBuild, using 'newgrp %s'." % group_name else: err_msg += "current user '%s' is not in group %s (members: %s)" % (user, group, grp_members) - _log.error(err_msg) + raise EasyBuildError(err_msg) _log.info("Using group '%s' (gid: %s)" % group) return group -def det_parallelism(par, maxpar): +def det_parallelism(par=None, maxpar=None): """ Determine level of parallelism that should be used. Default: educated guess based on # cores and 'ulimit -u' setting: min(# cores, ((ulimit -u) - 15) / 6) @@ -509,7 +480,7 @@ def det_parallelism(par, maxpar): try: par = int(par) except ValueError, err: - _log.error("Specified level of parallelism '%s' is not an integer value: %s" % (par, err)) + raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err) else: par = get_avail_core_count() # check ulimit -u @@ -524,10 +495,29 @@ def det_parallelism(par, maxpar): par = par_guess _log.info("Limit parallel builds to %s because max user processes is %s" % (par, out)) except ValueError, err: - _log.exception("Failed to determine max user processes (%s, %s): %s" % (ec, out, err)) + raise EasyBuildError("Failed to determine max user processes (%s, %s): %s", ec, out, err) if maxpar is not None and maxpar < par: _log.info("Limiting parallellism from %s to %s" % (par, maxpar)) par = min(par, maxpar) return par + + +def det_terminal_size(): + """ + Determine the current size of the terminal window. + @return: tuple with terminal width and height + """ + # see http://stackoverflow.com/questions/566746/how-to-get-console-window-width-in-python + try: + height, width, _, _ = struct.unpack('HHHH', fcntl.ioctl(0, termios.TIOCGWINSZ, struct.pack('HHHH', 0, 0, 0, 0))) + except Exception as err: + _log.warning("First attempt to determine terminal size failed: %s", err) + try: + height, width = [int(x) for x in os.popen("stty size").read().strip().split()] + except Exception as err: + _log.warning("Second attempt to determine terminal size failed, going to return defaults: %s", err) + height, width = 25, 80 + + return height, width diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index b572921f7e..c9fb689266 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -42,15 +42,16 @@ import easybuild.tools.config as config from easybuild.framework.easyblock import build_easyconfigs -from easybuild.framework.easyconfig.tools import process_easyconfig, resolve_dependencies +from easybuild.framework.easyconfig.tools import process_easyconfig from easybuild.framework.easyconfig.tools import skip_available from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option -from easybuild.tools.filetools import find_easyconfigs, mkdir, read_file +from easybuild.tools.filetools import find_easyconfigs, mkdir, read_file, write_file from easybuild.tools.github import create_gist, post_comment_in_issue from easybuild.tools.jenkins import aggregate_xml_in_dirs from easybuild.tools.modules import modules_tool from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel +from easybuild.tools.robot import resolve_dependencies from easybuild.tools.systemtools import get_system_info from easybuild.tools.version import FRAMEWORK_VERSION, EASYBLOCKS_VERSION from vsc.utils import fancylogger @@ -75,19 +76,17 @@ def regtest(easyconfig_paths, build_specs=None): _log.info("aggregated xml files inside %s, output written to: %s" % (aggregate_regtest, output_file)) sys.exit(0) - # create base directory, which is used to place - # all log files and the test output as xml - basename = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S") - var = config.OLDSTYLE_ENVIRONMENT_VARIABLES['test_output_path'] - + # create base directory, which is used to place all log files and the test output as xml regtest_output_dir = build_option('regtest_output_dir') + testoutput = build_option('testoutput') if regtest_output_dir is not None: output_dir = regtest_output_dir - elif var in os.environ: - output_dir = os.path.abspath(os.environ[var]) + elif testoutput is not None: + output_dir = os.path.abspath(testoutput) else: # default: current dir + easybuild-test-[timestamp] - output_dir = os.path.join(cur_dir, basename) + dirname = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S") + output_dir = os.path.join(cur_dir, dirname) mkdir(output_dir, parents=True) @@ -97,7 +96,7 @@ def regtest(easyconfig_paths, build_specs=None): for path in easyconfig_paths: ecfiles += find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs')) else: - _log.error("No easyconfig paths specified.") + raise EasyBuildError("No easyconfig paths specified.") test_results = [] @@ -120,29 +119,13 @@ def regtest(easyconfig_paths, build_specs=None): else: resolved = resolve_dependencies(easyconfigs, build_specs=build_specs) - cmd = "eb %(spec)s --regtest --sequential -ld" + cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s" command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd) # retry twice in case of failure, to avoid fluke errors command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {'cmd': cmd} - jobs = build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir) - - print "List of submitted jobs:" - for job in jobs: - print "%s: %s" % (job.name, job.jobid) - print "(%d jobs submitted)" % len(jobs) - - # determine leaf nodes in dependency graph, and report them - all_deps = set() - for job in jobs: - all_deps = all_deps.union(job.deps) + build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir) - leaf_nodes = [] - for job in jobs: - if job.jobid not in all_deps: - leaf_nodes.append(str(job.jobid).split('.')[0]) - - _log.info("Job ids of leaf nodes in dep. graph: %s" % ','.join(leaf_nodes)) _log.info("Submitted regression test as jobs, results in %s" % output_dir) return True # success @@ -157,9 +140,9 @@ def session_state(): } -def session_module_list(): +def session_module_list(testing=False): """Get list of loaded modules ('module list').""" - modtool = modules_tool() + modtool = modules_tool(testing=testing) return modtool.list() @@ -185,7 +168,7 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_l build_overview = [] for (ec, ec_res) in ecs_with_res: test_log = '' - if ec_res['success']: + if ec_res.get('success', False): test_result = 'SUCCESS' else: # compose test result string @@ -299,3 +282,38 @@ def post_easyconfigs_pr_test_report(pr_nr, test_report, msg, init_session_state, msg = "Test report uploaded to %s and mentioned in a comment in easyconfigs PR#%s" % (gist_url, pr_nr) return msg + + +def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state): + """ + Upload/dump overall test report + @param ecs_with_res: processed easyconfigs with build result (success/failure) + @param orig_cnt: number of original easyconfig paths + @param success: boolean indicating whether all builds were successful + @param msg: message to be included in test report + @param init_session_state: initial session state info to include in test report + """ + dump_path = build_option('dump_test_report') + pr_nr = build_option('from_pr') + upload = build_option('upload_test_report') + + if upload: + msg = msg + " (%d easyconfigs in this PR)" % orig_cnt + test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nr=pr_nr, gist_log=True) + if pr_nr: + # upload test report to gist and issue a comment in the PR to notify + txt = post_easyconfigs_pr_test_report(pr_nr, test_report, msg, init_session_state, success) + else: + # only upload test report as a gist + gist_url = upload_test_report_as_gist(test_report) + txt = "Test report uploaded to %s" % gist_url + else: + test_report = create_test_report(msg, ecs_with_res, init_session_state) + txt = None + _log.debug("Test report: %s" % test_report) + + if dump_path is not None: + write_file(dump_path, test_report) + _log.info("Test report dumped to %s" % dump_path) + + return txt diff --git a/easybuild/tools/toolchain/__init__.py b/easybuild/tools/toolchain/__init__.py index fb2a99459a..4c2abb3e18 100644 --- a/easybuild/tools/toolchain/__init__.py +++ b/easybuild/tools/toolchain/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 9b72f621af..12b5631e59 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,10 +28,8 @@ @author: Stijn De Weirdt (Ghent University) @author: Kenneth Hoste (Ghent University) """ - -import os - from easybuild.tools import systemtools +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option from easybuild.tools.toolchain.constants import COMPILER_VARIABLES from easybuild.tools.toolchain.toolchain import Toolchain @@ -110,6 +108,7 @@ class Compiler(Toolchain): COMPILER_F77 = None COMPILER_F90 = None + COMPILER_FC = None COMPILER_F_FLAGS = ['i8', 'r8'] COMPILER_F_UNIQUE_FLAGS = [] @@ -157,10 +156,8 @@ def _set_compiler_toolchainoptions(self): getattr(self, 'COMPILER_%sUNIQUE_OPTS' % infix, None), getattr(self, 'COMPILER_%sUNIQUE_OPTION_MAP' % infix, None), ) - #print "added options for prefix %s" % prefix - # redefine optarch - self._get_optimal_architecture() + self._set_optimal_architecture() def _set_compiler_vars(self): """Set the compiler variables""" @@ -186,7 +183,7 @@ def _set_compiler_vars(self): # only warn if prefix is set, not all languages may be supported (e.g., no Fortran for CUDA) self.log.warn("_set_compiler_vars: %s compiler variable %s undefined" % (prefix, var)) else: - self.log.raiseException("_set_compiler_vars: compiler variable %s undefined" % var) + raise EasyBuildError("_set_compiler_vars: compiler variable %s undefined", var) self.variables[pref_var] = value if is32bit: @@ -241,26 +238,20 @@ def _set_compiler_flags(self): self.variables.nextend('PRECFLAGS', precflags[:1]) # precflags last - self.variables.nappend('CFLAGS', flags) - self.variables.nappend('CFLAGS', cflags) - self.variables.join('CFLAGS', 'OPTFLAGS', 'PRECFLAGS') - - self.variables.nappend('CXXFLAGS', flags) - self.variables.nappend('CXXFLAGS', cflags) - self.variables.join('CXXFLAGS', 'OPTFLAGS', 'PRECFLAGS') - - self.variables.nappend('FFLAGS', flags) - self.variables.nappend('FFLAGS', fflags) - self.variables.join('FFLAGS', 'OPTFLAGS', 'PRECFLAGS') + for var in ['CFLAGS', 'CXXFLAGS']: + self.variables.nappend(var, flags) + self.variables.nappend(var, cflags) + self.variables.join(var, 'OPTFLAGS', 'PRECFLAGS') - self.variables.nappend('F90FLAGS', flags) - self.variables.nappend('F90FLAGS', fflags) - self.variables.join('F90FLAGS', 'OPTFLAGS', 'PRECFLAGS') + for var in ['FCFLAGS', 'FFLAGS', 'F90FLAGS']: + self.variables.nappend(var, flags) + self.variables.nappend(var, fflags) + self.variables.join(var, 'OPTFLAGS', 'PRECFLAGS') - def _get_optimal_architecture(self): + def _set_optimal_architecture(self): """ Get options for the current architecture """ if self.arch is None: - self.arch = systemtools.get_cpu_vendor() + self.arch = systemtools.get_cpu_family() optarch = None if build_option('optarch') is not None: @@ -269,11 +260,11 @@ def _get_optimal_architecture(self): optarch = self.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[self.arch] if optarch is not None: - self.log.info("_get_optimal_architecture: using %s as optarch for %s." % (optarch, self.arch)) + self.log.info("_set_optimal_architecture: using %s as optarch for %s." % (optarch, self.arch)) self.options.options_map['optarch'] = optarch if 'optarch' in self.options.options_map and self.options.options_map.get('optarch', None) is None: - self.log.raiseException("_get_optimal_architecture: don't know how to set optarch for %s." % self.arch) + raise EasyBuildError("_set_optimal_architecture: don't know how to set optarch for %s", self.arch) def comp_family(self, prefix=None): """ @@ -286,4 +277,4 @@ def comp_family(self, prefix=None): if comp_family: return comp_family else: - self.log.raiseException('comp_family: COMPILER_%sFAMILY is undefined.' % infix) + raise EasyBuildError("comp_family: COMPILER_%sFAMILY is undefined", infix) diff --git a/easybuild/tools/toolchain/constants.py b/easybuild/tools/toolchain/constants.py index ddf0f9f4c5..2cdc52a6a0 100644 --- a/easybuild/tools/toolchain/constants.py +++ b/easybuild/tools/toolchain/constants.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -39,12 +39,14 @@ ('CXX', 'C++ compiler'), ('F77', 'Fortran 77 compiler'), ('F90', 'Fortran 90 compiler'), + ('FC', 'Fortran compiler'), ] COMPILER_FLAGS = [ ('CFLAGS', 'C compiler flags'), ('CXXFLAGS', 'C++ compiler flags'), - ('FFLAGS', 'Fortran compiler flags'), + ('FCFLAGS', 'Fortran 77/90 compiler flags'), + ('FFLAGS', 'Fortran 77 compiler flags'), ('F90FLAGS', 'Fortran 90 compiler flags'), ] @@ -70,13 +72,15 @@ CommandFlagList: [ ('CUDA_CC', 'CUDA C compiler command'), ('CUDA_CXX', 'CUDA C++ compiler command'), - ('CUDA_F77', 'CUDA Fortran77 compiler command'), - ('CUDA_F90', 'CUDA Fortran90 compiler command'), + ('CUDA_F77', 'CUDA Fortran 77 compiler command'), + ('CUDA_F90', 'CUDA Fortran 90 compiler command'), + ('CUDA_FC', 'CUDA Fortran 77/90 compiler command'), ], FlagList: [ ('CUDA_CFLAGS', 'CUDA C compiler flags'), ('CUDA_CXXFLAGS', 'CUDA C++ compiler flags'), - ('CUDA_FFLAGS', 'CUDA Fortran compiler flags'), + ('CUDA_FCFLAGS', 'CUDA Fortran 77/90 compiler flags'), + ('CUDA_FFLAGS', 'CUDA Fortran 77 compiler flags'), ('CUDA_F90FLAGS', 'CUDA Fortran 90 compiler flags'), ], } diff --git a/easybuild/tools/toolchain/fft.py b/easybuild/tools/toolchain/fft.py index 192e418d06..f665e35f40 100644 --- a/easybuild/tools/toolchain/fft.py +++ b/easybuild/tools/toolchain/fft.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/linalg.py b/easybuild/tools/toolchain/linalg.py index 7086b0de4c..15b091ebba 100644 --- a/easybuild/tools/toolchain/linalg.py +++ b/easybuild/tools/toolchain/linalg.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,6 +29,7 @@ @author: Kenneth Hoste (Ghent University) """ +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.toolchain import Toolchain @@ -95,7 +96,7 @@ def set_variables(self): def _set_blas_variables(self): """Set BLAS related variables""" if self.BLAS_LIB is None: - self.log.raiseException("_set_blas_variables: BLAS_LIB not set") + raise EasyBuildError("_set_blas_variables: BLAS_LIB not set") self.BLAS_LIB = self.variables.nappend('LIBBLAS', [x % self.BLAS_LIB_MAP for x in self.BLAS_LIB]) self.variables.add_begin_end_linkerflags(self.BLAS_LIB, @@ -143,7 +144,7 @@ def _set_lapack_variables(self): self.variables.join('LAPACK_INC_DIR', 'BLAS_INC_DIR') else: if self.LAPACK_LIB is None: - self.log.raiseException("_set_lapack_variables: LAPACK_LIB not set") + raise EasyBuildError("_set_lapack_variables: LAPACK_LIB not set") self.LAPACK_LIB = self.variables.nappend('LIBLAPACK_ONLY', self.LAPACK_LIB) self.variables.add_begin_end_linkerflags(self.LAPACK_LIB, toggle_startstopgroup=self.LAPACK_LIB_GROUP, @@ -222,7 +223,7 @@ def _set_scalapack_variables(self): """Set ScaLAPACK related variables""" if self.SCALAPACK_LIB is None: - self.log.raiseException("_set_blas_variables: SCALAPACK_LIB not set") + raise EasyBuildError("_set_blas_variables: SCALAPACK_LIB not set") lib_map = {} if hasattr(self, 'BLAS_LIB_MAP') and self.BLAS_LIB_MAP is not None: @@ -268,7 +269,7 @@ def _set_scalapack_variables(self): if getattr(self, 'LIB_MULTITHREAD', None) is not None: self.variables.nappend('LIBSCALAPACK_MT', self.LIB_MULTITHREAD) else: - self.log.raiseException("_set_scalapack_variables: LIBSCALAPACK without SCALAPACK_REQUIRES not implemented") + raise EasyBuildError("_set_scalapack_variables: LIBSCALAPACK without SCALAPACK_REQUIRES not implemented") self.variables.join('SCALAPACK_STATIC_LIBS', 'LIBSCALAPACK') @@ -279,4 +280,3 @@ def _set_scalapack_variables(self): self._add_dependency_variables(self.SCALAPACK_MODULE_NAME, ld=self.SCALAPACK_LIB_DIR, cpp=self.SCALAPACK_INCLUDE_DIR) - diff --git a/easybuild/tools/toolchain/mpi.py b/easybuild/tools/toolchain/mpi.py index a04bb710ef..1f2d5bd924 100644 --- a/easybuild/tools/toolchain/mpi.py +++ b/easybuild/tools/toolchain/mpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,11 +28,12 @@ @author: Stijn De Weirdt (Ghent University) @author: Kenneth Hoste (Ghent University) """ - import os +import tempfile import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import write_file from easybuild.tools.toolchain.constants import COMPILER_VARIABLES, MPI_COMPILER_TEMPLATE, SEQ_COMPILER_TEMPLATE from easybuild.tools.toolchain.toolchain import Toolchain @@ -56,17 +57,19 @@ class Mpi(Toolchain): MPI_UNIQUE_OPTION_MAP = None MPI_SHARED_OPTION_MAP = { - '_opt_MPICC': 'cc=%(CC_base)s', - '_opt_MPICXX':'cxx=%(CXX_base)s', - '_opt_MPIF77':'fc=%(F77_base)s', - '_opt_MPIF90':'f90=%(F90_base)s', - } + '_opt_MPICC': 'cc=%(CC_base)s', + '_opt_MPICXX':'cxx=%(CXX_base)s', + '_opt_MPIF77':'fc=%(F77_base)s', + '_opt_MPIF90':'f90=%(F90_base)s', + '_opt_MPIFC':'fc=%(FC_base)s', + } MPI_COMPILER_MPICC = 'mpicc' MPI_COMPILER_MPICXX = 'mpicxx' MPI_COMPILER_MPIF77 = 'mpif77' MPI_COMPILER_MPIF90 = 'mpif90' + MPI_COMPILER_MPIFC = 'mpifc' MPI_LINK_INFO_OPTION = None @@ -106,7 +109,7 @@ def _set_mpi_compiler_variables(self): value = getattr(self, 'MPI_COMPILER_%s' % var.upper(), None) if value is None: - self.log.raiseException("_set_mpi_compiler_variables: mpi compiler variable %s undefined" % var) + raise EasyBuildError("_set_mpi_compiler_variables: mpi compiler variable %s undefined", var) self.variables.nappend_el(var, value) # complete compiler variable template to produce e.g. 'mpicc -cc=icc -X -Y' from 'mpicc -cc=%(CC_base)' @@ -159,7 +162,7 @@ def mpi_family(self): if self.MPI_FAMILY: return self.MPI_FAMILY else: - self.log.raiseException("mpi_family: MPI_FAMILY is undefined.") + raise EasyBuildError("mpi_family: MPI_FAMILY is undefined.") # FIXME: deprecate this function, use mympirun instead # this requires that either mympirun is packaged together with EasyBuild, or that vsc-tools is a dependency of EasyBuild @@ -167,15 +170,20 @@ def mpi_cmd_for(self, cmd, nr_ranks): """Construct an MPI command for the given command and number of ranks.""" # parameter values for mpirun command - params = {'nr_ranks':nr_ranks, 'cmd':cmd} + params = { + 'nr_ranks': nr_ranks, + 'cmd': cmd, + } # different known mpirun commands + mpirun_n_cmd = "mpirun -n %(nr_ranks)d %(cmd)s" mpi_cmds = { - toolchain.OPENMPI: "mpirun -n %(nr_ranks)d %(cmd)s", # @UndefinedVariable + toolchain.OPENMPI: mpirun_n_cmd, # @UndefinedVariable toolchain.QLOGICMPI: "mpirun -H localhost -np %(nr_ranks)d %(cmd)s", # @UndefinedVariable toolchain.INTELMPI: "mpirun %(mpdbf)s %(nodesfile)s -np %(nr_ranks)d %(cmd)s", # @UndefinedVariable - toolchain.MVAPICH2: "mpirun -n %(nr_ranks)d %(cmd)s", # @UndefinedVariable - toolchain.MPICH2: "mpirun -n %(nr_ranks)d %(cmd)s", # @UndefinedVariable + toolchain.MVAPICH2: mpirun_n_cmd, # @UndefinedVariable + toolchain.MPICH: mpirun_n_cmd, # @UndefinedVariable + toolchain.MPICH2: mpirun_n_cmd, # @UndefinedVariable } mpi_family = self.mpi_family() @@ -184,7 +192,15 @@ def mpi_cmd_for(self, cmd, nr_ranks): if mpi_family == toolchain.INTELMPI: # @UndefinedVariable # set temporary dir for mdp - env.setvar('I_MPI_MPD_TMPDIR', "/tmp") + # note: this needs to be kept *short*, to avoid mpirun failing with "socket.error: AF_UNIX path too long" + # exact limit is unknown, but ~20 characters seems to be OK + env.setvar('I_MPI_MPD_TMPDIR', tempfile.gettempdir()) + mpd_tmpdir = os.environ['I_MPI_MPD_TMPDIR'] + if len(mpd_tmpdir) > 20: + self.log.warning("$I_MPI_MPD_TMPDIR should be (very) short to avoid problems: %s" % mpd_tmpdir) + + # temporary location for mpdboot and nodes files + tmpdir = tempfile.mkdtemp(prefix='mpi_cmd_for-') # set PBS_ENVIRONMENT, so that --file option for mpdboot isn't stripped away env.setvar('PBS_ENVIRONMENT', "PBS_BATCH_MPI") @@ -194,28 +210,28 @@ def mpi_cmd_for(self, cmd, nr_ranks): env.setvar('I_MPI_PROCESS_MANAGER', 'mpd') # create mpdboot file - fn = "/tmp/mpdboot" + fn = os.path.join(tmpdir, 'mpdboot') try: if os.path.exists(fn): os.remove(fn) write_file(fn, "localhost ifhn=localhost") except OSError, err: - self.log.error("Failed to create file %s: %s" % (fn, err)) + raise EasyBuildError("Failed to create file %s: %s", fn, err) - params.update({'mpdbf':"--file=%s" % fn}) + params.update({'mpdbf': "--file=%s" % fn}) # create nodes file - fn = "/tmp/nodes" + fn = os.path.join(tmpdir, 'nodes') try: if os.path.exists(fn): os.remove(fn) write_file(fn, "localhost\n" * nr_ranks) except OSError, err: - self.log.error("Failed to create file %s: %s" % (fn, err)) + raise EasyBuildError("Failed to create file %s: %s", fn, err) - params.update({'nodesfile':"-machinefile %s" % fn}) + params.update({'nodesfile': "-machinefile %s" % fn}) if mpi_family in mpi_cmds.keys(): return mpi_cmds[mpi_family] % params else: - self.log.error("Don't know how to create an MPI command for MPI library of type '%s'." % mpi_family) + raise EasyBuildError("Don't know how to create an MPI command for MPI library of type '%s'.", mpi_family) diff --git a/easybuild/tools/toolchain/options.py b/easybuild/tools/toolchain/options.py index 9b0a5faaa8..32b162151f 100644 --- a/easybuild/tools/toolchain/options.py +++ b/easybuild/tools/toolchain/options.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,6 +37,8 @@ from vsc.utils import fancylogger +from easybuild.tools.build_log import EasyBuildError + class ToolchainOptions(dict): def __init__(self): @@ -62,9 +64,9 @@ def _add_options(self, options): self.log.debug("_add_options: adding options %s" % options) for name, value in options.items(): if not isinstance(value, (list, tuple,)) and len(value) == 2: - self.log.raiseException("_add_options: option name %s has to be 2 element list (%s)" % (name, value)) + raise EasyBuildError("_add_options: option name %s has to be 2 element list (%s)", name, value) if name in self: - self.log.debug("_add_options: redefining previous name %s (previous value %s)" % (name, self.get(name))) + self.log.debug("_add_options: redefining previous name %s (previous value %s)", name, self.get(name)) self.__setitem__(name, value[0]) self.description.__setitem__(name, value[1]) @@ -75,9 +77,9 @@ def _add_options_map(self, options_map): for name in options_map.keys(): if not name in self: if name.startswith('_opt_'): - self.log.debug("_add_options_map: no option with name %s defined, but allowed" % name) + self.log.debug("_add_options_map: no option with name %s defined, but allowed", name) else: - self.log.raiseException("_add_options_map: no option with name %s defined" % name) + raise EasyBuildError("_add_options_map: no option with name %s defined", name) self.options_map.update(options_map) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 939bec0c2a..907bb65734 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,13 +30,16 @@ @author: Stijn De Weirdt (Ghent University) @author: Kenneth Hoste (Ghent University) """ - +import copy import os from vsc.utils import fancylogger +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, install_path from easybuild.tools.environment import setvar -from easybuild.tools.modules import get_software_root, get_software_version, modules_tool +from easybuild.tools.module_generator import dependencies_for +from easybuild.tools.modules import get_software_root, get_software_root_env_var_name +from easybuild.tools.modules import get_software_version, get_software_version_env_var_name, modules_tool from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME, DUMMY_TOOLCHAIN_VERSION from easybuild.tools.toolchain.options import ToolchainOptions from easybuild.tools.toolchain.toolchainvariables import ToolchainVariables @@ -53,6 +56,11 @@ class Toolchain(object): NAME = None VERSION = None + TOOLCHAIN_FAMILY = None + + # list of class 'constants' that should be restored for every new instance of this class + CLASS_CONSTANTS_TO_RESTORE = None + CLASS_CONSTANT_COPIES = {} # class method def _is_toolchain_for(cls, name): @@ -69,41 +77,45 @@ def _is_toolchain_for(cls, name): _is_toolchain_for = classmethod(_is_toolchain_for) - def __init__(self, name=None, version=None, mns=None): + def __init__(self, name=None, version=None, mns=None, class_constants=None): """Toolchain constructor.""" self.base_init() self.dependencies = [] - self.toolchain_dependencies = [] + self.toolchain_dep_mods = [] if name is None: name = self.NAME if name is None: - self.log.raiseException("init: no name provided") + raise EasyBuildError("Toolchain init: no name provided") self.name = name if version is None: version = self.VERSION if version is None: - self.log.raiseException("init: no version provided") + raise EasyBuildError("Toolchain init: no version provided") self.version = version self.vars = None + self._init_class_constants(class_constants) + self.modules_tool = modules_tool() + self.mns = mns self.mod_full_name = None self.mod_short_name = None self.init_modpaths = None if self.name != DUMMY_TOOLCHAIN_NAME: # sometimes no module naming scheme class instance can/will be provided, e.g. with --list-toolchains - if mns is not None: + if self.mns is not None: tc_dict = self.as_dict() - self.mod_full_name = mns.det_full_module_name(tc_dict) - self.mod_short_name = mns.det_short_module_name(tc_dict) - self.init_modpaths = mns.det_init_modulepaths(tc_dict) + self.mod_full_name = self.mns.det_full_module_name(tc_dict) + self.mod_short_name = self.mns.det_short_module_name(tc_dict) + self.init_modpaths = self.mns.det_init_modulepaths(tc_dict) def base_init(self): + """Initialise missing class attributes (log, options, variables).""" if not hasattr(self, 'log'): self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) @@ -117,6 +129,43 @@ def base_init(self): if hasattr(self, 'LINKER_TOGGLE_STATIC_DYNAMIC'): self.variables.LINKER_TOGGLE_STATIC_DYNAMIC = self.LINKER_TOGGLE_STATIC_DYNAMIC + def _init_class_constants(self, class_constants): + """Initialise class 'constants'.""" + # make sure self.CLASS_CONSTANTS_TO_RESTORE is initialised + if class_constants is None: + self.CLASS_CONSTANTS_TO_RESTORE = [] + else: + self.CLASS_CONSTANTS_TO_RESTORE = class_constants[:] + + self._copy_class_constants() + self._restore_class_constants() + + def _copy_class_constants(self): + """Copy class constants that needs to be restored again when a new instance is created.""" + # this only needs to be done the first time (for this class, taking inheritance into account is key) + key = self.__class__ + if key not in self.CLASS_CONSTANT_COPIES: + self.CLASS_CONSTANT_COPIES[key] = {} + for cst in self.CLASS_CONSTANTS_TO_RESTORE: + if hasattr(self, cst): + self.CLASS_CONSTANT_COPIES[key][cst] = copy.deepcopy(getattr(self, cst)) + else: + raise EasyBuildError("Class constant '%s' to be restored does not exist in %s", cst, self) + + self.log.debug("Copied class constants: %s", self.CLASS_CONSTANT_COPIES[key]) + + def _restore_class_constants(self): + """Restored class constants that need to be restored when a new instance is created.""" + key = self.__class__ + for cst in self.CLASS_CONSTANT_COPIES[key]: + newval = copy.deepcopy(self.CLASS_CONSTANT_COPIES[key][cst]) + if hasattr(self, cst): + self.log.debug("Restoring class constant '%s' to %s (was: %s)", cst, newval, getattr(self, cst)) + else: + self.log.debug("Restoring (currently undefined) class constant '%s' to %s", cst, newval) + + setattr(self, cst, newval) + def get_variable(self, name, typ=str): """Get value for specified variable. typ: indicates what type of return value is expected""" @@ -126,7 +175,7 @@ def get_variable(self, name, typ=str): elif typ == list: return self.variables[name].flatten() else: - self.log.raiseException("get_variables: Don't know how to create value of type %s." % typ) + raise EasyBuildError("get_variable: Don't know how to create value of type %s.", typ) def set_variables(self): """Do nothing? Everything should have been set by others @@ -185,20 +234,18 @@ def _get_software_root(self, name): """Try to get the software root for name""" root = get_software_root(name) if root is None: - self.log.raiseException("get_software_root software root for %s was not found in environment" % (name)) + raise EasyBuildError("get_software_root software root for %s was not found in environment", name) else: - self.log.debug("get_software_root software root %s for %s was found in environment" % (root, name)) + self.log.debug("get_software_root software root %s for %s was found in environment", root, name) return root def _get_software_version(self, name): """Try to get the software root for name""" version = get_software_version(name) if version is None: - self.log.raiseException("get_software_version software version for %s was not found in environment" % - (name)) + raise EasyBuildError("get_software_version software version for %s was not found in environment", name) else: - self.log.debug("get_software_version software version %s for %s was found in environment" % - (version, name)) + self.log.debug("get_software_version software version %s for %s was found in environment", version, name) return version @@ -215,12 +262,13 @@ def as_dict(self, name=None, version=None): 'versionsuffix': '', 'dummy': True, 'parsed': True, # pretend this is a parsed easyconfig file, as may be required by det_short_module_name + 'hidden': False, } def det_short_module_name(self): """Determine module name for this toolchain.""" if self.mod_short_name is None: - self.log.error("Toolchain module name was not set yet (using set_module_info).") + raise EasyBuildError("Toolchain module name was not set yet (using set_module_info).") return self.mod_short_name def _toolchain_exists(self): @@ -233,9 +281,9 @@ def _toolchain_exists(self): return True else: if self.mod_short_name is None: - self.log.error("Toolchain module name was not set yet (using set_module_info).") + raise EasyBuildError("Toolchain module name was not set yet (using set_module_info).") # check whether a matching module exists if self.mod_short_name contains a module name - return self.modules_tool.exists(self.mod_full_name) + return self.modules_tool.exist([self.mod_full_name])[0] def set_options(self, options): """ Process toolchain options """ @@ -245,8 +293,8 @@ def set_options(self, options): self.options[opt] = options[opt] else: # used to be warning, but this is a severe error imho - self.log.raiseException("set_options: undefined toolchain option %s specified (possible names %s)" % - (opt, ",".join(self.options.keys()))) + known_opts = ','.join(self.options.keys()) + raise EasyBuildError("Undefined toolchain option %s specified (known options: %s)", opt, known_opts) def get_dependency_version(self, dependency): """ Generate a version string for a dependency on a module using this toolchain """ @@ -266,7 +314,7 @@ def get_dependency_version(self, dependency): if 'version' in dependency: version = "".join([dependency['version'], toolchain, suffix]) - self.log.debug("get_dependency_version: version in dependency return %s" % version) + self.log.debug("get_dependency_version: version in dependency return %s", version) return version else: toolchain_suffix = "".join([toolchain, suffix]) @@ -274,20 +322,21 @@ def get_dependency_version(self, dependency): # Find the most recent (or default) one if len(matches) > 0: version = matches[-1][-1] - self.log.debug("get_dependency_version: version not in dependency return %s" % version) + self.log.debug("get_dependency_version: version not in dependency return %s", version) return else: - self.log.raiseException('get_dependency_version: No toolchain version for dependency '\ - 'name %s (suffix %s) found' % (dependency['name'], toolchain_suffix)) + raise EasyBuildError("No toolchain version for dependency name %s (suffix %s) found", + dependency['name'], toolchain_suffix) def add_dependencies(self, dependencies): """ Verify if the given dependencies exist and add them """ self.log.debug("add_dependencies: adding toolchain dependencies %s" % dependencies) - for dep in dependencies: + dep_mod_names = [dep['full_mod_name'] for dep in dependencies] + deps_exist = self.modules_tool.exist(dep_mod_names) + for dep, dep_mod_name, dep_exists in zip(dependencies, dep_mod_names, deps_exist): self.log.debug("add_dependencies: MODULEPATH: %s" % os.environ['MODULEPATH']) - if not self.modules_tool.exists(dep['full_mod_name']): - tup = (dep['full_mod_name'], dep) - self.log.error("add_dependencies: no module '%s' found for dependency %s" % tup) + if not dep_exists: + raise EasyBuildError("add_dependencies: no module '%s' found for dependency %s", dep_mod_name, dep) else: self.dependencies.append(dep) self.log.debug('add_dependencies: added toolchain dependency %s' % str(dep)) @@ -310,6 +359,49 @@ def definition(self): _log.debug("Toolchain definition for %s: %s" % (self.as_dict(), tc_elems)) return tc_elems + def is_dep_in_toolchain_module(self, name): + """Check whether a specific software name is listed as a dependency in the module for this toolchain.""" + return any(map(lambda m: self.mns.is_short_modname_for(m, name), self.toolchain_dep_mods)) + + def _prepare_dependency_external_module(self, dep): + """Set environment variables picked up by utility functions for dependencies specified as external modules.""" + mod_name = dep['full_mod_name'] + metadata = dep['external_module_metadata'] + self.log.debug("Defining $EB* environment variables for external module %s", mod_name) + + names = metadata.get('name', []) + versions = metadata.get('version', [None]*len(names)) + self.log.debug("Metadata for external module %s: %s", mod_name, metadata) + + for name, version in zip(names, versions): + self.log.debug("Defining $EB* environment variables for external module %s under name %s", mod_name, name) + + # define $EBROOT env var for install prefix, picked up by get_software_root + prefix = metadata.get('prefix') + if prefix is not None: + if prefix in os.environ: + val = os.environ[prefix] + self.log.debug("Using value of $%s as prefix for external module %s: %s", prefix, mod_name, val) + else: + val = prefix + self.log.debug("Using specified prefix for external module %s: %s", mod_name, val) + setvar(get_software_root_env_var_name(name), val) + + # define $EBVERSION env var for software version, picked up by get_software_version + if version is not None: + setvar(get_software_version_env_var_name(name), version) + + def _prepare_dependencies(self): + """Load modules for dependencies, and handle special cases like external modules.""" + # load modules for all dependencies + dep_mods = [dep['short_mod_name'] for dep in self.dependencies] + self.log.debug("Loading modules for dependencies: %s" % dep_mods) + self.modules_tool.load(dep_mods) + + # define $EBROOT* and $EBVERSION* for external modules, if metadata is available + for dep in [d for d in self.dependencies if d['external_module']]: + self._prepare_dependency_external_module(dep) + def prepare(self, onlymod=None): """ Prepare a set of environment parameters based on name/version of toolchain @@ -321,17 +413,17 @@ def prepare(self, onlymod=None): (If string: comma separated list of variables that will be ignored). """ if self.modules_tool is None: - self.log.raiseException("No modules tool defined.") + raise EasyBuildError("No modules tool defined in Toolchain instance.") if not self._toolchain_exists(): - self.log.raiseException("No module found for toolchain name '%s' (%s)" % (self.name, self.version)) + raise EasyBuildError("No module found for toolchain: %s", self.mod_short_name) if self.name == DUMMY_TOOLCHAIN_NAME: if self.version == DUMMY_TOOLCHAIN_VERSION: self.log.info('prepare: toolchain dummy mode, dummy version; not loading dependencies') else: self.log.info('prepare: toolchain dummy mode and loading dependencies') - self.modules_tool.load([dep['short_mod_name'] for dep in self.dependencies]) + self._prepare_dependencies() return # Load the toolchain and dependencies modules @@ -342,33 +434,32 @@ def prepare(self, onlymod=None): for modpath in self.init_modpaths: self.modules_tool.prepend_module_path(os.path.join(install_path('mod'), mod_path_suffix, modpath)) self.modules_tool.load([self.det_short_module_name()]) - self.modules_tool.load([dep['short_mod_name'] for dep in self.dependencies]) + self._prepare_dependencies() # determine direct toolchain dependencies mod_name = self.det_short_module_name() - self.toolchain_dependencies = self.modules_tool.dependencies_for(mod_name, depth=0) - self.log.debug('prepare: list of direct toolchain dependencies: %s' % self.toolchain_dependencies) + self.toolchain_dep_mods = dependencies_for(mod_name, depth=0) + self.log.debug('prepare: list of direct toolchain dependencies: %s' % self.toolchain_dep_mods) - # verify whether elements in toolchain definition match toolchain deps specified by loaded toolchain module - toolchain_module_deps = set([self.modules_tool.module_software_name(d) for d in self.toolchain_dependencies]) # only retain names of toolchain elements, excluding toolchain name toolchain_definition = set([e for es in self.definition().values() for e in es if not e == self.name]) # filter out optional toolchain elements if they're not used in the module - for mod_name in toolchain_definition.copy(): - if not self.is_required(mod_name): - if not mod_name in toolchain_module_deps: - self.log.debug("Removing optional module %s from list of toolchain elements." % mod_name) - toolchain_definition.remove(mod_name) + for elem_name in toolchain_definition.copy(): + if self.is_required(elem_name) or self.is_dep_in_toolchain_module(elem_name): + continue + # not required and missing: remove from toolchain definition + self.log.debug("Removing %s from list of optional toolchain elements." % elem_name) + toolchain_definition.remove(elem_name) - self.log.debug("List of toolchain dependencies from toolchain module: %s" % toolchain_module_deps) + self.log.debug("List of toolchain dependencies from toolchain module: %s" % self.toolchain_dep_mods) self.log.debug("List of toolchain elements from toolchain definition: %s" % toolchain_definition) - if toolchain_module_deps == toolchain_definition: + if all(map(self.is_dep_in_toolchain_module, toolchain_definition)): self.log.info("List of toolchain dependency modules and toolchain definition match!") else: - self.log.error("List of toolchain dependency modules and toolchain definition do not match " \ - "(%s vs %s)" % (toolchain_module_deps, toolchain_definition)) + raise EasyBuildError("List of toolchain dependency modules and toolchain definition do not match " + "(%s vs %s)", self.toolchain_dep_mods, toolchain_definition) # Generate the variables to be set self.set_variables() @@ -409,7 +500,17 @@ def _add_dependency_variables(self, names=None, cpp=None, ld=None): else: deps = [{'name': name} for name in names if name is not None] - for root in self.get_software_root([dep['name'] for dep in deps]): + # collect software install prefixes for dependencies + roots = [] + for dep in deps: + if dep.get('external_module', False): + # for software names provided via external modules, install prefix may be unknown + names = dep['external_module_metadata'].get('name', []) + roots.extend([root for root in self.get_software_root(names) if root is not None]) + else: + roots.extend(self.get_software_root(dep['name'])) + + for root in roots: self.variables.append_subdirs("CPPFLAGS", root, subdirs=cpp_paths) self.variables.append_subdirs("LDFLAGS", root, subdirs=ld_paths) @@ -420,8 +521,7 @@ def _setenv_variables(self, donotset=None): donotsetlist = [] if isinstance(donotset, str): # TODO : more legacy code that should be using proper type - self.log.raiseException("_setenv_variables: using commas-separated list. should be deprecated.") - donotsetlist = donotset.split(',') + raise EasyBuildError("_setenv_variables: using commas-separated list. should be deprecated.") elif isinstance(donotset, list): donotsetlist = donotset @@ -443,44 +543,16 @@ def get_flag(self, name): """Get compiler flag for a certain option.""" return "-%s" % self.options.option(name) + def toolchain_family(self): + """Return toolchain family for this toolchain.""" + return self.TOOLCHAIN_FAMILY + def comp_family(self): """ Return compiler family used in this toolchain (abstract method).""" raise NotImplementedError def mpi_family(self): - """ Return type of MPI library used in this toolchain (abstract method).""" - raise NotImplementedError - - # legacy functions TODO remove AFTER migration - # should search'n'replaced - def get_type(self, name, type_map): - """Determine type of toolchain based on toolchain dependencies.""" - self.log.raiseException("get_type: legacy code. should not be needed anymore.") - - def _set_variables(self, dontset=None): - """ Sets the environment variables """ - self.log.raiseException("_set_variables: legacy code. use _setenv_variables.") - - def _addDependencyVariables(self, names=None): - """ Add LDFLAGS and CPPFLAGS to the self.vars based on the dependencies - names should be a list of strings containing the name of the dependency""" - self.log.raiseException("_addDependencyVaraibles: legacy code. use _add_dependency_variables.") - - def _setVariables(self, dontset=None): - """ Sets the environment variables """ - self.log.raiseException("_setVariables: legacy code. use _set_variables.") - - def _toolkitExists(self, name=None, version=None): - """ - Verify if there exists a toolkit by this name and version + """ Return type of MPI library used in this toolchain or 'None' if MPI is not + supported. """ - self.log.raiseException("_toolkitExists: legacy code. replace use _toolchain_exists.") - - def get_openmp_flag(self): - """Get compiler flag for OpenMP support.""" - self.log.raiseException("get_openmp_flag: legacy code. use options.get_flag('openmp').") - - @property - def opts(self): - """Get value for specified option.""" - self.log.raiseException("opts[x]: legacy code. use options[x].") + return None diff --git a/easybuild/tools/toolchain/toolchainvariables.py b/easybuild/tools/toolchain/toolchainvariables.py index 0c94c8874c..a9c4682152 100644 --- a/easybuild/tools/toolchain/toolchainvariables.py +++ b/easybuild/tools/toolchain/toolchainvariables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/utilities.py b/easybuild/tools/toolchain/utilities.py index 633e10accb..73bf78c7c1 100644 --- a/easybuild/tools/toolchain/utilities.py +++ b/easybuild/tools/toolchain/utilities.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -40,6 +40,7 @@ from vsc.utils.missing import get_subclasses, nub import easybuild.tools.toolchain +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.toolchain import Toolchain from easybuild.tools.utilities import import_available_modules @@ -83,13 +84,13 @@ def search_toolchain(name): if res: tc_const_name = res.group(1) tc_const_value = getattr(mod_class_mod, elem) - tup = (tc_const_name, tc_const_value, mod_class_mod.__name__, package.__name__) - _log.debug("Found constant %s ('%s') in module %s, adding it to %s" % tup) + _log.debug("Found constant %s ('%s') in module %s, adding it to %s", + tc_const_name, tc_const_value, mod_class_mod.__name__, package.__name__) if hasattr(package, tc_const_name): cur_value = getattr(package, tc_const_name) if not tc_const_value == cur_value: - tup = (package.__name__, tc_const_name, cur_value, tc_const_value) - _log.error("Constant %s.%s defined as '%s', can't set it to '%s'." % tup) + raise EasyBuildError("Constant %s.%s defined as '%s', can't set it to '%s'.", + package.__name__, tc_const_name, cur_value, tc_const_value) else: setattr(package, tc_const_name, tc_const_value) @@ -124,7 +125,7 @@ def get_toolchain(tc, tcopts, mns): tc_class, all_tcs = search_toolchain(tc['name']) if not tc_class: all_tcs_names = ",".join([x.NAME for x in all_tcs]) - _log.error("Toolchain %s not found, available toolchains: %s" % (tc['name'], all_tcs_names)) + raise EasyBuildError("Toolchain %s not found, available toolchains: %s", tc['name'], all_tcs_names) tc_inst = tc_class(version=tc['version'], mns=mns) tc_dict = tc_inst.as_dict() _log.debug("Obtained new toolchain instance for %s: %s" % (key, tc_dict)) diff --git a/easybuild/tools/toolchain/variables.py b/easybuild/tools/toolchain/variables.py index b9fb56d00e..cdee3ea41b 100644 --- a/easybuild/tools/toolchain/variables.py +++ b/easybuild/tools/toolchain/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,6 +29,7 @@ @author: Kenneth Hoste (Ghent University) """ +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.variables import StrList, AbsPathList @@ -141,7 +142,7 @@ def _toggle_map(self, toggle_map, name, descr, idx=None): else: self.insert(idx, toggle_map[name]) else: - self.log.raiseException("%s name %s not found in map %s" % (descr, name, toggle_map)) + raise EasyBuildError("%s name %s not found in map %s", descr, name, toggle_map) def toggle_startgroup(self): """Append start group""" diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index 246619e3fd..b826839271 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -32,9 +32,10 @@ import string import sys from vsc.utils import fancylogger -from vsc.utils.missing import any as _any -from vsc.utils.missing import all as _all + import easybuild.tools.environment as env +from easybuild.tools.build_log import EasyBuildError + _log = fancylogger.getLogger('tools.utilities') @@ -45,26 +46,9 @@ UNWANTED_CHARS = ASCII_CHARS.translate(ASCII_CHARS, string.digits + string.ascii_letters + "_") -def any(ls): - """Reimplementation of 'any' function, which is not available in Python 2.4 yet.""" - _log.deprecated("own definition of any", "2.0") - return _any(ls) - - -def all(ls): - """Reimplementation of 'all' function, which is not available in Python 2.4 yet.""" - _log.deprecated("own definition of all", "2.0") - return _all(ls) - - def read_environment(env_vars, strict=False): - """ - Read variables from the environment - @param: env_vars: a dict with key a name, value a environment variable name - @param: strict, boolean, if True enforces that all specified environment variables are found - """ - _log.deprecated("moved read_environment to tools.environment", "2.0") - return env.read_environment(env_vars, strict) + """NO LONGER SUPPORTED: use read_environment from easybuild.tools.environment instead""" + _log.nosupport("read_environment has been moved to easybuild.tools.environment", '2.0') def flatten(lst): @@ -75,7 +59,7 @@ def flatten(lst): return res -def quote_str(x): +def quote_str(val, escape_newline=False, prefer_single_quotes=False): """ Obtain a new value to be used in string replacement context. @@ -84,17 +68,31 @@ def quote_str(x): For string values, it tries to escape the string in quotes, e.g., foo becomes 'foo', foo'bar becomes "foo'bar", foo'bar"baz becomes \"\"\"foo'bar"baz\"\"\", etc. + + @param escape_newline: wrap strings that include a newline in triple quotes """ - if isinstance(x, basestring): - if "'" in x and '"' in x: - return '"""%s"""' % x - elif '"' in x: - return "'%s'" % x + if isinstance(val, basestring): + # forced triple double quotes + if ("'" in val and '"' in val) or (escape_newline and '\n' in val): + return '"""%s"""' % val + # single quotes to escape double quote used in strings + elif '"' in val: + return "'%s'" % val + # if single quotes are preferred, use single quotes; + # unless a space or a single quote are in the string + elif prefer_single_quotes and "'" not in val and ' ' not in val: + return "'%s'" % val + # fallback on double quotes (required in tcl syntax) else: - return '"%s"' % x + return '"%s"' % val else: - return x + return val + + +def quote_py_str(val): + """Version of quote_str specific for generating use in Python context (e.g., easyconfig parameters).""" + return quote_str(val, escape_newline=True, prefer_single_quotes=True) def remove_unwanted_chars(inputstring): @@ -113,10 +111,38 @@ def import_available_modules(namespace): """ modules = [] for path in sys.path: - for module in glob.glob(os.path.sep.join([path] + namespace.split('.') + ['*.py'])): + for module in sorted(glob.glob(os.path.sep.join([path] + namespace.split('.') + ['*.py']))): if not module.endswith('__init__.py'): mod_name = module.split(os.path.sep)[-1].split('.')[0] modpath = '.'.join([namespace, mod_name]) _log.debug("importing module %s" % modpath) - modules.append(__import__(modpath, globals(), locals(), [''])) + try: + mod = __import__(modpath, globals(), locals(), ['']) + except ImportError as err: + raise EasyBuildError("import_available_modules: Failed to import %s: %s", modpath, err) + modules.append(mod) return modules + + +def only_if_module_is_available(modname, pkgname=None, url=None): + """Decorator to guard functions/methods against missing required module with specified name.""" + if pkgname and url is None: + url = 'https://pypi.python.org/pypi/%s' % pkgname + + def wrap(orig): + """Decorated function, raises ImportError if specified module is not available.""" + try: + __import__(modname) + return orig + + except ImportError as err: + def error(*args): + msg = "%s; required module '%s' is not available" % (err, modname) + if pkgname: + msg += " (provided by Python package %s, available from %s)" % (pkgname, url) + elif url: + msg += " (available from %s)" % url + raise ImportError(msg) + return error + + return wrap diff --git a/easybuild/tools/variables.py b/easybuild/tools/variables.py index bee298f6ce..a6ec60c62f 100644 --- a/easybuild/tools/variables.py +++ b/easybuild/tools/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,9 +29,12 @@ @author: Stijn De Weirdt (Ghent University) @author: Kenneth Hoste (Ghent University) """ -from vsc.utils import fancylogger import copy import os +from vsc.utils import fancylogger + +from easybuild.tools.build_log import EasyBuildError + _log = fancylogger.getLogger('variables', fname=False) @@ -62,20 +65,20 @@ def join_map_class(map_classes): """Join all class_maps into single class_map""" res = {} for map_class in map_classes: - for k, v in map_class.items(): - if isinstance(k, (str,)): - var_name = k - if isinstance(v, (tuple, list)): + for key, val in map_class.items(): + if isinstance(key, (str,)): + var_name = key + if isinstance(val, (tuple, list)): # second element is documentation - klass = v[0] + klass = val[0] res[var_name] = klass - elif type(k) in (type,): + elif type(key) in (type,): # k is the class, v a list of tuples (name,doc) - klass = k + klass = key default = res.setdefault(klass, []) - default.extend([tpl[0] for tpl in v]) + default.extend([tpl[0] for tpl in val]) else: - _log.raiseException("join_map_class: impossible to join key %s value %s" % (k, v)) + raise EasyBuildError("join_map_class: impossible to join key %s value %s", key, val) return res @@ -304,7 +307,7 @@ def nextend(self, value=None, **kwargs): res = [] if value is None: # TODO ? append_empty ? - self.log.raiseException("extend_el with None value unimplemented") + raise EasyBuildError("extend_el with None value unimplemented") else: for el in value: if not self._str_ok(el): @@ -478,13 +481,17 @@ def join(self, name, *others): else it is nappend-ed """ self.log.debug("join name %s others %s" % (name, others)) + + # make sure name is defined, even if 'others' list is empty + self.setdefault(name) + for other in others: if other in self: self.log.debug("join other %s in self: other %s" % (other, self.get(other).__repr__())) for el in self.get(other): self.nappend(name, el) else: - self.log.raiseException("join: name %s; other %s not found in self." % (name, other)) + raise EasyBuildError("join: name %s; other %s not found in self.", name, other) def append(self, name, value): """Append value to element name (alias for nappend)""" diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 8528cee3ad..6b90246ba7 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,8 +37,14 @@ # note: release candidates should be versioned as a pre-release, e.g. "1.1rc1" # 1.1-rc1 would indicate a post-release, i.e., and update of 1.1, so beware! -VERSION = LooseVersion("1.15.0dev") -UNKNOWN = "UNKNOWN" +# +# important note: dev versions should follow the 'X.Y.Z.dev0' format +# see https://www.python.org/dev/peps/pep-0440/#developmental-releases +# recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like +# UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' +# This causes problems further up the dependency chain... +VERSION = LooseVersion('2.4.0.dev0') +UNKNOWN = 'UNKNOWN' def get_git_revision(): """ diff --git a/eb b/eb index 4bb70df087..51eeb7152a 100755 --- a/eb +++ b/eb @@ -39,6 +39,28 @@ # @author: Pieter De Baets (Ghent University) # @author: Jens Timmerman (Ghent University) +# Python 2.6 or more recent 2.x required +REQ_MAJ_PYVER=2 +REQ_MIN_PYVER=6 +REQ_PYVER=${REQ_MAJ_PYVER}.${REQ_MIN_PYVER} + +# make sure Python version being used is compatible +pyver=`python -V 2>&1 | cut -f2 -d' '` +pyver_maj=`echo $pyver | cut -f1 -d'.'` +pyver_min=`echo $pyver | cut -f2 -d'.'` + +if [ $pyver_maj -ne $REQ_MAJ_PYVER ] +then + echo "ERROR: EasyBuild is currently only compatible with Python v${REQ_MAJ_PYVER}.x, found v${pyver}" 1>&2 + exit 1 +fi +if [ $pyver_min -lt $REQ_MIN_PYVER ] +then + echo "ERROR: EasyBuild requires Python v${REQ_PYVER} or a more recent v${REQ_MAJ_PYVER}.x, found v${pyver}." 1>&2 + exit 2 +fi + + main_script_base_path="easybuild/main.py" python_search_path_cmd="python -c \"import sys; print ' '.join(sys.path)\"" diff --git a/setup.cfg b/setup.cfg index d1b2877e6d..baf390bd17 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,2 @@ [bdist_rpm] -requires = environment-modules, bash, python >= 2.4, python < 3 +requires = environment-modules, bash, python >= 2.6, python < 3 diff --git a/setup.py b/setup.py index 5a707b3d82..13800b9f51 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ -# # -# Copyright 2012-2013 Ghent University +## +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -21,8 +21,7 @@ # # You should have received a copy of the GNU General Public License # along with EasyBuild. If not, see . -# # - +## """ This script can be used to install easybuild-framework, e.g. using: easy_install --user . @@ -31,7 +30,7 @@ @author: Kenneth Hoste (Ghent University) """ - +import glob import os from distutils import log @@ -39,6 +38,7 @@ API_VERSION = str(VERSION).split('.')[0] + # Utility function to read README file def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() @@ -55,6 +55,7 @@ def read(fname): log.info("Installing version %s (API version %s)" % (VERSION, API_VERSION)) + def find_rel_test(): """Return list of files recursively from basedir (aka find -type f)""" basedir = os.path.join(os.path.dirname(__file__), "test", "framework") @@ -71,38 +72,29 @@ def find_rel_test(): easybuild_packages = [ "easybuild", "easybuild.framework", "easybuild.framework.easyconfig", "easybuild.framework.easyconfig.format", "easybuild.toolchains", "easybuild.toolchains.compiler", "easybuild.toolchains.mpi", - "easybuild.toolchains.fft", "easybuild.toolchains.linalg", "easybuild.tools", - "easybuild.tools.toolchain", "easybuild.tools.module_naming_scheme", "easybuild.tools.repository", - "test.framework", "test", - "vsc", "vsc.utils", + "easybuild.toolchains.fft", "easybuild.toolchains.linalg", "easybuild.tools", "easybuild.tools.deprecated", + "easybuild.tools.job", "easybuild.tools.toolchain", "easybuild.tools.module_naming_scheme", + "easybuild.tools.package", "easybuild.tools.package.package_naming_scheme", + "easybuild.tools.repository", "test.framework", "test", ] setup( - name = "easybuild-framework", - version = str(VERSION), - author = "EasyBuild community", - author_email = "easybuild@lists.ugent.be", - description = """EasyBuild is a software installation framework in Python that allows you to \ -install software in a structured and robust way. -This package contains the EasyBuild framework, which supports the creation of custom easyblocks that \ + name="easybuild-framework", + version=str(VERSION), + author="EasyBuild community", + author_email="easybuild@lists.ugent.be", + description="""The EasyBuild framework supports the creation of custom easyblocks that \ implement support for installing particular (groups of) software packages.""", - license = "GPLv2", - keywords = "software build building installation installing compilation HPC scientific", - url = "http://hpcugent.github.com/easybuild", - packages = easybuild_packages, - package_dir = {'test.framework': "test/framework"}, - package_data = {"test.framework": find_rel_test()}, - scripts = ["eb", "optcomplete.bash", "minimal_bash_completion.bash"], - data_files = [ - ('easybuild', ["easybuild/easybuild_config.py"]), - ], - long_description = """This package contains the EasyBuild -framework, which supports the creation of custom easyblocks that -implement support for installing particular (groups of) software -packages. - -""" + read("README.rst"), - classifiers = [ + license="GPLv2", + keywords="software build building installation installing compilation HPC scientific", + url="http://hpcugent.github.com/easybuild", + packages=easybuild_packages, + package_dir={'test.framework': "test/framework"}, + package_data={"test.framework": find_rel_test()}, + scripts=["eb", "optcomplete.bash", "minimal_bash_completion.bash"], + data_files=[('easybuild/scripts', glob.glob('easybuild/scripts/*'))], + long_description=read('README.rst'), + classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Console", "Intended Audience :: System Administrators", @@ -111,8 +103,9 @@ def find_rel_test(): "Programming Language :: Python :: 2.4", "Topic :: Software Development :: Build Tools", ], - platforms = "Linux", - provides = ["eb"] + easybuild_packages, - test_suite = "test.framework.suite", - zip_safe = False, + platforms="Linux", + provides=["eb"] + easybuild_packages, + test_suite="test.framework.suite", + zip_safe=False, + install_requires=["vsc-base >= 2.2.4"], ) diff --git a/test/__init__.py b/test/__init__.py index 9459a6d90a..8df00ef461 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/__init__.py b/test/framework/__init__.py index f7e235f8d2..d6e0a952fc 100644 --- a/test/framework/__init__.py +++ b/test/framework/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/asyncprocess.py b/test/framework/asyncprocess.py index c70b4b1dd4..d8637fecf5 100644 --- a/test/framework/asyncprocess.py +++ b/test/framework/asyncprocess.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/build_log.py b/test/framework/build_log.py new file mode 100644 index 0000000000..814308ba87 --- /dev/null +++ b/test/framework/build_log.py @@ -0,0 +1,152 @@ +# # +# Copyright 2015-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Unit tests for EasyBuild log infrastructure + +@author: Kenneth Hoste (Ghent University) +""" +import os +import re +import tempfile +from test.framework.utilities import EnhancedTestCase, init_config +from unittest import TestLoader +from unittest import main as unittestmain +from vsc.utils.fancylogger import getLogger, getRootLoggerName, logToFile, setLogFormat + +from easybuild.tools.build_log import LOGGING_FORMAT, EasyBuildError +from easybuild.tools.filetools import read_file, write_file + + +def raise_easybuilderror(msg, *args, **kwargs): + """Utility function: just raise a EasyBuildError.""" + raise EasyBuildError(msg, *args, **kwargs) + + +class BuildLogTest(EnhancedTestCase): + """Tests for EasyBuild log infrastructure.""" + + def tearDown(self): + """Cleanup after test.""" + # restore default logging format + setLogFormat(LOGGING_FORMAT) + + def test_easybuilderror(self): + """Tests for EasyBuildError.""" + fd, tmplog = tempfile.mkstemp() + os.close(fd) + + # set log format, for each regex searching + setLogFormat("%(name)s :: %(message)s") + + # if no logger is available, and no logger is specified, use default 'root' fancylogger + logToFile(tmplog, enable=True) + self.assertErrorRegex(EasyBuildError, 'BOOM', raise_easybuilderror, 'BOOM') + logToFile(tmplog, enable=False) + + log_re = re.compile("^%s :: BOOM \(at .*:[0-9]+ in [a-z_]+\)$" % getRootLoggerName(), re.M) + logtxt = open(tmplog, 'r').read() + self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt)) + + # test formatting of message + self.assertErrorRegex(EasyBuildError, 'BOOMBAF', raise_easybuilderror, 'BOOM%s', 'BAF') + + os.remove(tmplog) + + def test_easybuildlog(self): + """Tests for EasyBuildLog.""" + fd, tmplog = tempfile.mkstemp() + os.close(fd) + + # set log format, for each regex searching + setLogFormat("%(name)s [%(levelname)s] :: %(message)s") + + # test basic log methods + logToFile(tmplog, enable=True) + log = getLogger('test_easybuildlog') + log.setLevelName('DEBUG') + log.debug("123 debug") + log.info("foobar info") + log.warn("justawarning") + log.raiseError = False + log.error("kaput") + log.raiseError = True + try: + log.exception("oops") + except EasyBuildError: + pass + logToFile(tmplog, enable=False) + logtxt = read_file(tmplog) + + root = getRootLoggerName() + + expected_logtxt = '\n'.join([ + r"%s.test_easybuildlog \[DEBUG\] :: 123 debug" % root, + r"%s.test_easybuildlog \[INFO\] :: foobar info" % root, + r"%s.test_easybuildlog \[WARNING\] :: justawarning" % root, + r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput" % root, + r"%s.test_easybuildlog \[ERROR\] :: .*EasyBuild encountered an exception \(at .* in .*\): oops" % root, + '', + ]) + logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M) + self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt)) + + # wipe log so we can reuse it + write_file(tmplog, '') + + # test formatting log messages by providing extra arguments + logToFile(tmplog, enable=True) + log.warn("%s", "bleh"), + log.info("%s+%s = %d", '4', '2', 42) + args = ['this', 'is', 'just', 'a', 'test'] + log.debug("%s %s %s %s %s", *args) + log.raiseError = False + log.error("foo %s baz", 'baz') + log.raiseError = True + logToFile(tmplog, enable=False) + logtxt = read_file(tmplog) + expected_logtxt = '\n'.join([ + r"%s.test_easybuildlog \[WARNING\] :: bleh" % root, + r"%s.test_easybuildlog \[INFO\] :: 4\+2 = 42" % root, + r"%s.test_easybuildlog \[DEBUG\] :: this is just a test" % root, + r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): foo baz baz" % root, + '', + ]) + logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M) + self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt)) + + # test deprecated behaviour: raise EasyBuildError on log.error and log.exception + os.environ['EASYBUILD_DEPRECATED'] = '2.1' + init_config() + + log.warning("No raise for warnings") + self.assertErrorRegex(EasyBuildError, 'EasyBuild crashed with an error', log.error, 'foo') + self.assertErrorRegex(EasyBuildError, 'EasyBuild encountered an exception', log.exception, 'bar') + +def suite(): + """ returns all the testcases in this module """ + return TestLoader().loadTestsFromTestCase(BuildLogTest) + +if __name__ == '__main__': + unittestmain() diff --git a/test/framework/config.py b/test/framework/config.py index 2ce0946dbd..ecbb28fe2c 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,9 +28,9 @@ @author: Kenneth Hoste (Ghent University) @author: Stijn De Weirdt (Ghent University) """ -import copy import os import shutil +import sys import tempfile from test.framework.utilities import EnhancedTestCase, init_config from unittest import TestLoader @@ -38,14 +38,33 @@ from vsc.utils.fancylogger import setLogLevelDebug, logToScreen import easybuild.tools.options as eboptions -from easybuild.tools.config import build_path, source_paths, install_path, get_repository, get_repositorypath -from easybuild.tools.config import log_file_format, set_tmpdir, BuildOptions, ConfigurationVariables -from easybuild.tools.config import get_build_log_path, DEFAULT_PATH_SUBDIRS, init_build_options, build_option +from easybuild.tools import run +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option, build_path, source_paths, install_path, get_repositorypath +from easybuild.tools.config import set_tmpdir, BuildOptions, ConfigurationVariables +from easybuild.tools.config import get_build_log_path, DEFAULT_PATH_SUBDIRS, init_build_options from easybuild.tools.environment import modify_env -from easybuild.tools.filetools import write_file -from easybuild.tools.repository.filerepo import FileRepository -from easybuild.tools.repository.repository import init_repository - +from easybuild.tools.filetools import mkdir, write_file +from easybuild.tools.options import CONFIG_ENV_VAR_PREFIX + +EXTERNAL_MODULES_METADATA = """[cray-netcdf/4.3.2] +name = netCDF,netCDF-Fortran +version = 4.3.2,4.3.2 +prefix = NETCDF_DIR + +[cray-hdf5/1.8.13] +name = HDF5 +version = 1.8.13 +prefix = HDF5_DIR + +[foo] +name = Foo +prefix = /foo + +[bar/1.2.3] +name = bar +version = 1.2.3 +""" class EasyBuildConfigTest(EnhancedTestCase): """Test cases for EasyBuild configuration.""" @@ -54,14 +73,14 @@ class EasyBuildConfigTest(EnhancedTestCase): def setUp(self): """Prepare for running a config test.""" + reload(eboptions) super(EasyBuildConfigTest, self).setUp() self.tmpdir = tempfile.mkdtemp() def purge_environment(self): """Remove any leftover easybuild variables""" - for path in ['buildpath', 'installpath', 'sourcepath']: - var = 'EASYBUILD_%s' % path.upper() - if var in os.environ: + for var in os.environ.keys(): + if var.startswith('EASYBUILD_'): del os.environ[var] def tearDown(self): @@ -105,221 +124,8 @@ def test_default_config(self): self.assertEqual(config_options['repositorypath'], [os.path.join(eb_homedir, 'ebfiles_repo')]) self.assertEqual(config_options['logfile_format'][0], 'easybuild') self.assertEqual(config_options['logfile_format'][1], "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log") - self.assertEqual(config_options['tmp_logdir'], tempfile.gettempdir()) - - def test_legacy_env_vars(self): - """Test legacy environment variables.""" - self.purge_environment() - - # build path - test_buildpath = os.path.join(self.tmpdir, 'build', 'path') - os.environ['EASYBUILDBUILDPATH'] = test_buildpath - self.configure(args=[]) - self.assertEqual(build_path(), test_buildpath) - del os.environ['EASYBUILDBUILDPATH'] - - # source path(s) - test_sourcepaths = [ - os.path.join(self.tmpdir, 'source', 'path'), - ':'.join([ - os.path.join(self.tmpdir, 'source', 'path1'), - os.path.join(self.tmpdir, 'source', 'path2'), - ]), - ':'.join([ - os.path.join(self.tmpdir, 'source', 'path1'), - os.path.join(self.tmpdir, 'source', 'path2'), - os.path.join(self.tmpdir, 'source', 'path3'), - ]), - ] - for test_sourcepath in test_sourcepaths: - init_config() - os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath - self.configure(args=[]) - self.assertEqual(build_path(), os.path.join(os.path.expanduser('~'), '.local', 'easybuild', - DEFAULT_PATH_SUBDIRS['buildpath'])) - self.assertEqual(source_paths(), test_sourcepath.split(':')) - del os.environ['EASYBUILDSOURCEPATH'] - - test_sourcepath = os.path.join(self.tmpdir, 'source', 'path') - - # install path - init_config() - test_installpath = os.path.join(self.tmpdir, 'install', 'path') - os.environ['EASYBUILDINSTALLPATH'] = test_installpath - self.configure(args=[]) - self.assertEqual(source_paths()[0], os.path.join(os.path.expanduser('~'), '.local', 'easybuild', - DEFAULT_PATH_SUBDIRS['sourcepath'])) - self.assertEqual(install_path(), os.path.join(test_installpath, DEFAULT_PATH_SUBDIRS['subdir_software'])) - self.assertEqual(install_path(typ='mod'), os.path.join(test_installpath, - DEFAULT_PATH_SUBDIRS['subdir_modules'])) - del os.environ['EASYBUILDINSTALLPATH'] - - # prefix: should change build/install/source/repo paths - init_config() - test_prefixpath = os.path.join(self.tmpdir, 'prefix', 'path') - os.environ['EASYBUILDPREFIX'] = test_prefixpath - self.configure(args=[]) - self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath'])) - self.assertEqual(source_paths()[0], os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath'])) - self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software'])) - self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath, - DEFAULT_PATH_SUBDIRS['subdir_modules'])) - repo = init_repository(get_repository(), get_repositorypath()) - self.assertTrue(isinstance(repo, FileRepository)) - self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath'])) - - # build/source/install path overrides prefix - init_config() - os.environ['EASYBUILDBUILDPATH'] = test_buildpath - self.configure(args=[]) - self.assertEqual(build_path(), test_buildpath) - self.assertEqual(source_paths()[0], os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath'])) - self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software'])) - self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath, - DEFAULT_PATH_SUBDIRS['subdir_modules'])) - repo = init_repository(get_repository(), get_repositorypath()) - self.assertTrue(isinstance(repo, FileRepository)) - self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath'])) - del os.environ['EASYBUILDBUILDPATH'] - - init_config() - os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath - self.configure(args=[]) - self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath'])) - self.assertEqual(source_paths()[0], test_sourcepath) - self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software'])) - self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath, - DEFAULT_PATH_SUBDIRS['subdir_modules'])) - repo = init_repository(get_repository(), get_repositorypath()) - self.assertTrue(isinstance(repo, FileRepository)) - self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath'])) - del os.environ['EASYBUILDSOURCEPATH'] - - init_config() - os.environ['EASYBUILDINSTALLPATH'] = test_installpath - self.configure(args=[]) - self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath'])) - self.assertEqual(source_paths()[0], os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath'])) - self.assertEqual(install_path(), os.path.join(test_installpath, DEFAULT_PATH_SUBDIRS['subdir_software'])) - self.assertEqual(install_path(typ='mod'), os.path.join(test_installpath, - DEFAULT_PATH_SUBDIRS['subdir_modules'])) - repo = init_repository(get_repository(), get_repositorypath()) - self.assertTrue(isinstance(repo, FileRepository)) - self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath'])) - - def test_legacy_config_file(self): - """Test finding/using legacy configuration files.""" - self.purge_environment() - - cfg_fn = self.configure(args=[]) - self.assertTrue(cfg_fn.endswith('easybuild/easybuild_config.py')) - - configtxt = """ -build_path = '%(buildpath)s' -source_path = '%(sourcepath)s' -install_path = '%(installpath)s' -repository_path = '%(repopath)s' -repository = FileRepository(repository_path) -log_format = ('%(logdir)s', '%(logtmpl)s') -log_dir = '%(tmplogdir)s' -software_install_suffix = '%(softsuffix)s' -modules_install_suffix = '%(modsuffix)s' -""" - - buildpath = os.path.join(self.tmpdir, 'my', 'test', 'build', 'path') - sourcepath = os.path.join(self.tmpdir, 'my', 'test', 'source', 'path') - installpath = os.path.join(self.tmpdir, 'my', 'test', 'install', 'path') - repopath = os.path.join(self.tmpdir, 'my', 'test', 'repo', 'path') - logdir = 'somedir' - logtmpl = 'test-eb-%(name)s%(version)s_date-%(date)s__time-%(time)s.log' - tmplogdir = os.path.join(self.tmpdir, 'my', 'test', 'tmplogdir') - softsuffix = 'myfavoritesoftware' - modsuffix = 'modulesgohere' - - configdict = { - 'buildpath': buildpath, - 'sourcepath': sourcepath, - 'installpath': installpath, - 'repopath': repopath, - 'logdir': logdir, - 'logtmpl': logtmpl, - 'tmplogdir': tmplogdir, - 'softsuffix': softsuffix, - 'modsuffix': modsuffix - } - - # create user config file on default location - myconfigfile = os.path.join(self.tmpdir, '.easybuild', 'config.py') - if not os.path.exists(os.path.dirname(myconfigfile)): - os.makedirs(os.path.dirname(myconfigfile)) - write_file(myconfigfile, configtxt % configdict) - - # redefine home so we can test user config file on default location - home = os.environ.get('HOME', None) - os.environ['HOME'] = self.tmpdir - init_config() - cfg_fn = self.configure(args=[]) - if home is not None: - os.environ['HOME'] = home - - # check finding and use of config file - self.assertEqual(cfg_fn, myconfigfile) - self.assertEqual(build_path(), buildpath) - self.assertEqual(source_paths()[0], sourcepath) - self.assertEqual(install_path(), os.path.join(installpath, softsuffix)) - self.assertEqual(install_path(typ='mod'), os.path.join(installpath, modsuffix)) - repo = init_repository(get_repository(), get_repositorypath()) - self.assertTrue(isinstance(repo, FileRepository)) - self.assertEqual(repo.repo, repopath) - self.assertEqual(log_file_format(return_directory=True), logdir) - self.assertEqual(log_file_format(), logtmpl) - self.assertEqual(get_build_log_path(), tmplogdir) - - # redefine config file entries for proper testing below - buildpath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'build', 'path') - sourcepath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'source', 'path') - installpath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'install', 'path') - repopath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'repo', 'path') - logdir = 'somedir_custom' - logtmpl = 'test-custom-eb-%(name)_%(date)s%(time)s__%(version)s.log' - tmplogdir = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'tmplogdir') - softsuffix = 'myfavoritesoftware_custom' - modsuffix = 'modulesgohere_custom' - - configdict = { - 'buildpath': buildpath, - 'sourcepath': sourcepath, - 'installpath': installpath, - 'repopath': repopath, - 'logdir': logdir, - 'logtmpl': logtmpl, - 'tmplogdir': tmplogdir, - 'softsuffix': softsuffix, - 'modsuffix': modsuffix } - - # create custom config file, and point to it - mycustomconfigfile = os.path.join(self.tmpdir, 'mycustomconfig.py') - if not os.path.exists(os.path.dirname(mycustomconfigfile)): - os.makedirs(os.path.dirname(mycustomconfigfile)) - write_file(mycustomconfigfile, configtxt % configdict) - os.environ['EASYBUILDCONFIG'] = mycustomconfigfile - - # reconfigure - init_config() - cfg_fn = self.configure(args=[]) - - # verify configuration - self.assertEqual(cfg_fn, mycustomconfigfile) - self.assertEqual(build_path(), buildpath) - self.assertEqual(source_paths()[0], sourcepath) - self.assertEqual(install_path(), os.path.join(installpath, softsuffix)) - self.assertEqual(install_path(typ='mod'), os.path.join(installpath, modsuffix)) - repo = init_repository(get_repository(), get_repositorypath()) - self.assertTrue(isinstance(repo, FileRepository)) - self.assertEqual(repo.repo, repopath) - self.assertEqual(log_file_format(return_directory=True), logdir) - self.assertEqual(log_file_format(), logtmpl) - self.assertEqual(get_build_log_path(), tmplogdir) + self.assertEqual(config_options['tmpdir'], None) + self.assertEqual(config_options['tmp_logdir'], None) def test_generaloption_config(self): """Test new-style configuration (based on generaloption).""" @@ -333,6 +139,8 @@ def test_generaloption_config(self): options = init_config(args=[]) self.assertEqual(build_path(), buildpath_env_var) self.assertEqual(install_path(), os.path.join(prefix, 'software')) + self.assertEqual(get_repositorypath(), [os.path.join(prefix, 'ebfiles_repo')]) + del os.environ['EASYBUILD_PREFIX'] del os.environ['EASYBUILD_BUILDPATH'] @@ -345,53 +153,110 @@ def test_generaloption_config(self): write_file(config_file, '') args = [ - '--config', config_file, # force empty oldstyle config file + '--configfiles', config_file, # force empty config file '--prefix', prefix, '--installpath', install, '--repositorypath', repopath, + '--subdir-software', 'APPS', ] options = init_config(args=args) self.assertEqual(build_path(), os.path.join(prefix, 'build')) - self.assertEqual(install_path(), os.path.join(install, 'software')) + self.assertEqual(install_path(), os.path.join(install, 'APPS')) self.assertEqual(install_path(typ='mod'), os.path.join(install, 'modules')) self.assertEqual(options.installpath, install) - self.assertEqual(options.config, config_file) + self.assertTrue(config_file in options.configfiles) # check mixed command line/env var configuration prefix = os.path.join(self.tmpdir, 'test3') install = os.path.join(self.tmpdir, 'test4', 'install') subdir_software = 'eb-soft' args = [ - '--config', config_file, # force empty oldstyle config file + '--configfiles', config_file, # force empty config file '--installpath', install, ] os.environ['EASYBUILD_PREFIX'] = prefix os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = subdir_software + installpath_modules = tempfile.mkdtemp(prefix='installpath-modules') + os.environ['EASYBUILD_INSTALLPATH_MODULES'] = installpath_modules options = init_config(args=args) self.assertEqual(build_path(), os.path.join(prefix, 'build')) self.assertEqual(install_path(), os.path.join(install, subdir_software)) + self.assertEqual(install_path('mod'), installpath_modules) + + # subdir options *must* be relative (to --installpath) + installpath_software = tempfile.mkdtemp(prefix='installpath-software') + os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = installpath_software + error_regex = r"Found problems validating the options.*'subdir_software' must specify a \*relative\* path" + self.assertErrorRegex(EasyBuildError, error_regex, init_config) del os.environ['EASYBUILD_PREFIX'] del os.environ['EASYBUILD_SUBDIR_SOFTWARE'] + def test_error_env_var_typo(self): + """Test error reporting on use of known $EASYBUILD-prefixed env vars.""" + # all is well + init_config() + + os.environ['EASYBUILD_FOO'] = 'foo' + os.environ['EASYBUILD_THERESNOSUCHCONFIGURATIONOPTION'] = 'whatever' + + error = r"Found 2 environment variable\(s\) that are prefixed with %s " % CONFIG_ENV_VAR_PREFIX + error += "but do not match valid option\(s\): " + error += ','.join(['EASYBUILD_FOO', 'EASYBUILD_THERESNOSUCHCONFIGURATIONOPTION']) + self.assertErrorRegex(EasyBuildError, error, init_config) + + del os.environ['EASYBUILD_THERESNOSUCHCONFIGURATIONOPTION'] + del os.environ['EASYBUILD_FOO'] + + def test_install_path(self): + """Test install_path function.""" + # defaults + self.assertEqual(install_path(), os.path.join(self.test_installpath, 'software')) + self.assertEqual(install_path('software'), os.path.join(self.test_installpath, 'software')) + self.assertEqual(install_path(typ='mod'), os.path.join(self.test_installpath, 'modules')) + self.assertEqual(install_path('modules'), os.path.join(self.test_installpath, 'modules')) + + self.assertErrorRegex(EasyBuildError, "Unknown type specified", install_path, typ='foo') + + args = [ + '--subdir-software', 'SOFT', + '--installpath', '/foo', + ] + os.environ['EASYBUILD_SUBDIR_MODULES'] = 'MOD' + init_config(args=args) + self.assertEqual(install_path(), os.path.join('/foo', 'SOFT')) + self.assertEqual(install_path(typ='mod'), os.path.join('/foo', 'MOD')) + del os.environ['EASYBUILD_SUBDIR_MODULES'] + + args = [ + '--installpath', '/prefix', + '--installpath-modules', '/foo', + ] + os.environ['EASYBUILD_INSTALLPATH_SOFTWARE'] = '/bar/baz' + init_config(args=args) + self.assertEqual(install_path(), os.path.join('/bar', 'baz')) + self.assertEqual(install_path(typ='mod'), '/foo') + + del os.environ['EASYBUILD_INSTALLPATH_SOFTWARE'] + init_config(args=args) + self.assertEqual(install_path(), os.path.join('/prefix', 'software')) + self.assertEqual(install_path(typ='mod'), '/foo') + def test_generaloption_config_file(self): """Test use of new-style configuration file.""" self.purge_environment() - oldstyle_config_file = os.path.join(self.tmpdir, 'nooldconfig.py') config_file = os.path.join(self.tmpdir, 'testconfig.cfg') testpath1 = os.path.join(self.tmpdir, 'test1') testpath2 = os.path.join(self.tmpdir, 'testtwo') - write_file(oldstyle_config_file, '') - # test with config file passed via command line cfgtxt = '\n'.join([ '[config]', @@ -399,21 +264,40 @@ def test_generaloption_config_file(self): ]) write_file(config_file, cfgtxt) + installpath_software = tempfile.mkdtemp(prefix='installpath-software') args = [ '--configfiles', config_file, '--debug', '--buildpath', testpath1, + '--installpath-software', installpath_software, ] options = init_config(args=args) self.assertEqual(build_path(), testpath1) # via command line self.assertEqual(source_paths(), [os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources')]) # default - self.assertEqual(install_path(), os.path.join(testpath2, 'software')) # via config file + self.assertEqual(install_path(), installpath_software) # via cmdline arg + self.assertEqual(install_path('mod'), os.path.join(testpath2, 'modules')) # via config file + + # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory + # to check whether easyconfigs install path is auto-included in robot path + tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') + mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) + + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + shutil.copytree(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) + + orig_sys_path = sys.path[:] + sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs # test with config file passed via environment variable + installpath_modules = tempfile.mkdtemp(prefix='installpath-modules') cfgtxt = '\n'.join([ '[config]', 'buildpath = %s' % testpath1, + 'sourcepath = %(DEFAULT_REPOSITORYPATH)s', + 'repositorypath = %(DEFAULT_REPOSITORYPATH)s,somesubdir', + 'robot-paths=/tmp/foo:%(sourcepath)s:%(DEFAULT_ROBOT_PATHS)s', + 'installpath-modules=%s' % installpath_modules, ]) write_file(config_file, cfgtxt) @@ -424,9 +308,18 @@ def test_generaloption_config_file(self): ] options = init_config(args=args) - self.assertEqual(install_path(), os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'software')) # default + topdir = os.path.join(os.getenv('HOME'), '.local', 'easybuild') + self.assertEqual(install_path(), os.path.join(topdir, 'software')) # default + self.assertEqual(install_path('mod'), installpath_modules), # via config file self.assertEqual(source_paths(), [testpath2]) # via command line self.assertEqual(build_path(), testpath1) # via config file + self.assertEqual(get_repositorypath(), [os.path.join(topdir, 'ebfiles_repo'), 'somesubdir']) # via config file + robot_paths = [ + '/tmp/foo', + os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'ebfiles_repo'), + os.path.join(tmpdir, 'easybuild', 'easyconfigs'), + ] + self.assertEqual(options.robot_paths[:3], robot_paths) testpath3 = os.path.join(self.tmpdir, 'testTHREE') os.environ['EASYBUILD_SOURCEPATH'] = testpath2 @@ -438,9 +331,11 @@ def test_generaloption_config_file(self): self.assertEqual(source_paths(), [testpath2]) # via environment variable $EASYBUILD_SOURCEPATHS self.assertEqual(install_path(), os.path.join(testpath3, 'software')) # via command line + self.assertEqual(install_path('mod'), installpath_modules), # via config file self.assertEqual(build_path(), testpath1) # via config file del os.environ['EASYBUILD_CONFIGFILES'] + sys.path[:] = orig_sys_path def test_set_tmpdir(self): """Test set_tmpdir config function.""" @@ -454,13 +349,16 @@ def test_set_tmpdir(self): mytmpdir = set_tmpdir(tmpdir=tmpdir) for var in ['TMPDIR', 'TEMP', 'TMP']: - self.assertTrue(os.environ[var].startswith(os.path.join(parent, 'easybuild-'))) + self.assertTrue(os.environ[var].startswith(os.path.join(parent, 'eb-'))) self.assertEqual(os.environ[var], mytmpdir) - self.assertTrue(tempfile.gettempdir().startswith(os.path.join(parent, 'easybuild-'))) + self.assertTrue(tempfile.gettempdir().startswith(os.path.join(parent, 'eb-'))) tempfile_tmpdir = tempfile.mkdtemp() - self.assertTrue(tempfile_tmpdir.startswith(os.path.join(parent, 'easybuild-'))) + self.assertTrue(tempfile_tmpdir.startswith(os.path.join(parent, 'eb-'))) fd, tempfile_tmpfile = tempfile.mkstemp() - self.assertTrue(tempfile_tmpfile.startswith(os.path.join(parent, 'easybuild-'))) + self.assertTrue(tempfile_tmpfile.startswith(os.path.join(parent, 'eb-'))) + + # tmp_logdir follows tmpdir + self.assertEqual(get_build_log_path(), mytmpdir) # cleanup os.close(fd) @@ -528,6 +426,224 @@ def test_build_options(self): bo2 = BuildOptions() self.assertTrue(bo is bo2) + def test_XDG_CONFIG_env_vars(self): + """Test effect of XDG_CONFIG* environment variables on default configuration.""" + self.purge_environment() + + xdg_config_home = os.environ.get('XDG_CONFIG_HOME') + xdg_config_dirs = os.environ.get('XDG_CONFIG_DIRS') + + cfg_template = '\n'.join([ + '[config]', + 'prefix=%s', + ]) + + homedir = os.path.join(self.test_prefix, 'homedir', '.config') + mkdir(os.path.join(homedir, 'easybuild'), parents=True) + write_file(os.path.join(homedir, 'easybuild', 'config.cfg'), cfg_template % '/home') + + dir1 = os.path.join(self.test_prefix, 'dir1') + mkdir(os.path.join(dir1, 'easybuild.d'), parents=True) + write_file(os.path.join(dir1, 'easybuild.d', 'foo.cfg'), cfg_template % '/foo') + write_file(os.path.join(dir1, 'easybuild.d', 'bar.cfg'), cfg_template % '/bar') + + dir2 = os.path.join(self.test_prefix, 'dir2') # empty on purpose + mkdir(os.path.join(dir2, 'easybuild.d'), parents=True) + + dir3 = os.path.join(self.test_prefix, 'dir3') + mkdir(os.path.join(dir3, 'easybuild.d'), parents=True) + write_file(os.path.join(dir3, 'easybuild.d', 'foobarbaz.cfg'), cfg_template % '/foobarbaz') + + # only $XDG_CONFIG_HOME set + os.environ['XDG_CONFIG_HOME'] = homedir + cfg_files = [os.path.join(homedir, 'easybuild', 'config.cfg')] + reload(eboptions) + eb_go = eboptions.parse_options(args=[]) + self.assertEqual(eb_go.options.configfiles, cfg_files) + self.assertEqual(eb_go.options.prefix, '/home') + + # $XDG_CONFIG_HOME set, one directory listed in $XDG_CONFIG_DIRS + os.environ['XDG_CONFIG_DIRS'] = dir1 + cfg_files = [ + os.path.join(dir1, 'easybuild.d', 'bar.cfg'), + os.path.join(dir1, 'easybuild.d', 'foo.cfg'), + os.path.join(homedir, 'easybuild', 'config.cfg'), # $XDG_CONFIG_HOME goes last + ] + reload(eboptions) + eb_go = eboptions.parse_options(args=[]) + self.assertEqual(eb_go.options.configfiles, cfg_files) + self.assertEqual(eb_go.options.prefix, '/home') # last cfgfile wins + + # $XDG_CONFIG_HOME not set, multiple directories listed in $XDG_CONFIG_DIRS + del os.environ['XDG_CONFIG_HOME'] # unset, so should become default + os.environ['XDG_CONFIG_DIRS'] = os.pathsep.join([dir1, dir2, dir3]) + cfg_files = [ + os.path.join(dir1, 'easybuild.d', 'bar.cfg'), + os.path.join(dir1, 'easybuild.d', 'foo.cfg'), + os.path.join(dir3, 'easybuild.d', 'foobarbaz.cfg'), + ] + reload(eboptions) + eb_go = eboptions.parse_options(args=[]) + # note: there may be a config file in $HOME too, so don't use a strict comparison + self.assertEqual(cfg_files, eb_go.options.configfiles[:3]) + + # $XDG_CONFIG_HOME set to non-existing directory, multiple directories listed in $XDG_CONFIG_DIRS + os.environ['XDG_CONFIG_HOME'] = os.path.join(self.test_prefix, 'nosuchdir') + cfg_files = [ + os.path.join(dir1, 'easybuild.d', 'bar.cfg'), + os.path.join(dir1, 'easybuild.d', 'foo.cfg'), + os.path.join(dir3, 'easybuild.d', 'foobarbaz.cfg'), + ] + reload(eboptions) + eb_go = eboptions.parse_options(args=[]) + self.assertEqual(eb_go.options.configfiles, cfg_files) + self.assertEqual(eb_go.options.prefix, '/foobarbaz') # last cfgfile wins + + # restore $XDG_CONFIG env vars to original state + if xdg_config_home is None: + del os.environ['XDG_CONFIG_HOME'] + else: + os.environ['XDG_CONFIG_HOME'] = xdg_config_home + + if xdg_config_dirs is None: + del os.environ['XDG_CONFIG_DIRS'] + else: + os.environ['XDG_CONFIG_DIRS'] = xdg_config_dirs + reload(eboptions) + + def test_flex_robot_paths(self): + """Test prepend/appending to default robot search path via --robot-paths.""" + # unset $EASYBUILD_ROBOT_PATHS that was defined in setUp + del os.environ['EASYBUILD_ROBOT_PATHS'] + + # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory + # to check whether easyconfigs install path is auto-included in robot path + tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') + mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) + test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + tmp_ecs_dir = os.path.join(tmpdir, 'easybuild', 'easyconfigs') + shutil.copytree(test_ecs_path, tmp_ecs_dir) + + # prepend path to test easyconfigs into Python search path, so it gets picked up as --robot-paths default + orig_sys_path = sys.path[:] + sys.path = [tmpdir] + [p for p in sys.path if not os.path.exists(os.path.join(p, 'easybuild', 'easyconfigs'))] + + # default: only pick up installed easyconfigs via sys.path + eb_go = eboptions.parse_options(args=[]) + self.assertEqual(eb_go.options.robot_paths, [tmp_ecs_dir]) + + # prepend to default robot path + eb_go = eboptions.parse_options(args=['--robot-paths=/foo:']) + self.assertEqual(eb_go.options.robot_paths, ['/foo', tmp_ecs_dir]) + eb_go = eboptions.parse_options(args=['--robot-paths=/foo:/bar/baz/:']) + self.assertEqual(eb_go.options.robot_paths, ['/foo', '/bar/baz/', tmp_ecs_dir]) + + # append to default robot path + eb_go = eboptions.parse_options(args=['--robot-paths=:/bar/baz']) + self.assertEqual(eb_go.options.robot_paths, [tmp_ecs_dir, '/bar/baz']) + # append to default robot path + eb_go = eboptions.parse_options(args=['--robot-paths=:/bar/baz:/foo']) + self.assertEqual(eb_go.options.robot_paths, [tmp_ecs_dir, '/bar/baz', '/foo']) + + # prepend and append to default robot path + eb_go = eboptions.parse_options(args=['--robot-paths=/foo/bar::/baz']) + self.assertEqual(eb_go.options.robot_paths, ['/foo/bar', tmp_ecs_dir, '/baz']) + eb_go = eboptions.parse_options(args=['--robot-paths=/foo/bar::/baz:/trala']) + self.assertEqual(eb_go.options.robot_paths, ['/foo/bar', tmp_ecs_dir, '/baz', '/trala']) + eb_go = eboptions.parse_options(args=['--robot-paths=/foo/bar:/trala::/baz']) + self.assertEqual(eb_go.options.robot_paths, ['/foo/bar', '/trala', tmp_ecs_dir, '/baz']) + + # also via $EASYBUILD_ROBOT_PATHS + os.environ['EASYBUILD_ROBOT_PATHS'] = '/foo::/bar/baz' + eb_go = eboptions.parse_options(args=[]) + self.assertEqual(eb_go.options.robot_paths, ['/foo', tmp_ecs_dir, '/bar/baz']) + + # --robot-paths overrides $EASYBUILD_ROBOT_PATHS + os.environ['EASYBUILD_ROBOT_PATHS'] = '/foobar::/barbar/baz/baz' + eb_go = eboptions.parse_options(args=['--robot-paths=/one::/last']) + self.assertEqual(eb_go.options.robot_paths, ['/one', tmp_ecs_dir, '/last']) + + del os.environ['EASYBUILD_ROBOT_PATHS'] + + # also works with a cfgfile in the mix + config_file = os.path.join(self.tmpdir, 'testconfig.cfg') + cfgtxt = '\n'.join([ + '[config]', + 'robot-paths=/cfgfirst::/cfglast', + ]) + write_file(config_file, cfgtxt) + eb_go = eboptions.parse_options(args=['--configfiles=%s' % config_file]) + self.assertEqual(eb_go.options.robot_paths, ['/cfgfirst', tmp_ecs_dir, '/cfglast']) + + # cfgfile entry is lost when env var and/or cmdline options are used + os.environ['EASYBUILD_ROBOT_PATHS'] = '/envfirst::/envend' + eb_go = eboptions.parse_options(args=['--configfiles=%s' % config_file]) + self.assertEqual(eb_go.options.robot_paths, ['/envfirst', tmp_ecs_dir, '/envend']) + + del os.environ['EASYBUILD_ROBOT_PATHS'] + eb_go = eboptions.parse_options(args=['--robot-paths=/veryfirst:', '--configfiles=%s' % config_file]) + self.assertEqual(eb_go.options.robot_paths, ['/veryfirst', tmp_ecs_dir]) + + os.environ['EASYBUILD_ROBOT_PATHS'] = ':/envend' + eb_go = eboptions.parse_options(args=['--robot-paths=/veryfirst:', '--configfiles=%s' % config_file]) + self.assertEqual(eb_go.options.robot_paths, ['/veryfirst', tmp_ecs_dir]) + + del os.environ['EASYBUILD_ROBOT_PATHS'] + + # override default robot path + eb_go = eboptions.parse_options(args=['--robot-paths=/foo:/bar/baz']) + self.assertEqual(eb_go.options.robot_paths, ['/foo', '/bar/baz']) + + # paths specified via --robot still get preference + eb_go = eboptions.parse_options(args=['--robot-paths=/foo/bar::/baz', '--robot=/first']) + self.assertEqual(eb_go.options.robot_paths, ['/first', '/foo/bar', tmp_ecs_dir, '/baz']) + + sys.path[:] = orig_sys_path + + def test_external_modules_metadata(self): + """Test --external-modules-metadata.""" + # empty list by default + cfg = init_config() + self.assertEqual(cfg.external_modules_metadata, []) + + testcfgtxt = EXTERNAL_MODULES_METADATA + testcfg = os.path.join(self.test_prefix, 'test_external_modules_metadata.cfg') + write_file(testcfg, testcfgtxt) + + cfg = init_config(args=['--external-modules-metadata=%s' % testcfg]) + + netcdf = { + 'name': ['netCDF', 'netCDF-Fortran'], + 'version': ['4.3.2', '4.3.2'], + 'prefix': 'NETCDF_DIR', + } + self.assertEqual(cfg.external_modules_metadata['cray-netcdf/4.3.2'], netcdf) + hdf5 = { + 'name': ['HDF5'], + 'version': ['1.8.13'], + 'prefix': 'HDF5_DIR', + } + self.assertEqual(cfg.external_modules_metadata['cray-hdf5/1.8.13'], hdf5) + + # impartial metadata is fine + self.assertEqual(cfg.external_modules_metadata['foo'], {'name': ['Foo'], 'prefix': '/foo'}) + self.assertEqual(cfg.external_modules_metadata['bar/1.2.3'], {'name': ['bar'], 'version': ['1.2.3']}) + + # if both names and versions are specified, lists must have same lengths + write_file(testcfg, '\n'.join(['[foo/1.2.3]', 'name = foo,bar', 'version = 1.2.3'])) + args = ['--external-modules-metadata=%s' % testcfg] + err_msg = "Different length for lists of names/versions in metadata for external module" + self.assertErrorRegex(EasyBuildError, err_msg, init_config, args=args) + + def test_strict(self): + """Test use of --strict.""" + # check default + self.assertEqual(build_option('strict'), run.WARN) + + for strict_str, strict_val in [('error', run.ERROR), ('ignore', run.IGNORE), ('warn', run.WARN)]: + options = init_config(args=['--strict=%s' % strict_str]) + init_config(build_options={'strict': options.strict}) + self.assertEqual(build_option('strict'), strict_val) def suite(): diff --git a/test/framework/docs.py b/test/framework/docs.py new file mode 100644 index 0000000000..e68c5de5a9 --- /dev/null +++ b/test/framework/docs.py @@ -0,0 +1,107 @@ +# # +# Copyright 2012-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Unit tests for docs.py. +""" +import inspect +import os +import re +import sys +from unittest import TestLoader, main + +from easybuild.framework.easyconfig.licenses import license_documentation +from easybuild.tools.docs import gen_easyblocks_overview_rst +from easybuild.tools.utilities import import_available_modules +from test.framework.utilities import EnhancedTestCase, init_config + +class DocsTest(EnhancedTestCase): + + def test_gen_easyblocks(self): + """ Test gen_easyblocks_overview_rst function """ + module = 'easybuild.easyblocks.generic' + modules = import_available_modules(module) + common_params = { + 'ConfigureMake' : ['configopts', 'buildopts', 'installopts'], + } + doc_functions = ['build_step', 'configure_step', 'test_step'] + + eb_overview = gen_easyblocks_overview_rst(module, 'easyconfigs', common_params, doc_functions) + ebdoc = '\n'.join(eb_overview) + + # extensive check for ConfigureMake easyblock + check_configuremake = '\n'.join([ + ".. _ConfigureMake:", + '', + "``ConfigureMake``", + "=================", + '', + "(derives from EasyBlock)", + '', + "Dummy support for building and installing applications with configure/make/make install.", + '', + "Commonly used easyconfig parameters with ``ConfigureMake`` easyblock", + "--------------------------------------------------------------------", + "==================== ================================================================", + "easyconfig parameter description ", + "==================== ================================================================", + "configopts Extra options passed to configure (default already has --prefix)", + "buildopts Extra options passed to make step (default already has -j X) ", + "installopts Extra options for installation ", + "==================== ================================================================", + ]) + + self.assertTrue(check_configuremake in ebdoc) + names = [] + + for mod in modules: + for name, obj in inspect.getmembers(mod, inspect.isclass): + eb_class = getattr(mod, name) + # skip imported classes that are not easyblocks + if eb_class.__module__.startswith(module): + self.assertTrue(name in ebdoc) + names.append(name) + + toc = [":ref:`" + n + "`" for n in sorted(names)] + pattern = " - ".join(toc) + + regex = re.compile(pattern) + self.assertTrue(re.search(regex, ebdoc), "Pattern %s found in %s" % (regex.pattern, ebdoc)) + + def test_license_docs(self): + """Test license_documentation function.""" + lic_docs = license_documentation() + gplv3 = "GPLv3: The GNU General Public License" + self.assertTrue(gplv3 in lic_docs, "%s found in: %s" % (gplv3, lic_docs)) + + +def suite(): + """ returns all test cases in this module """ + return TestLoader().loadTestsFromTestCase(DocsTest) + +if __name__ == '__main__': + # also check the setUp for debug + # logToScreen(enable=True) + # setLogLevelDebug() + main() diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 2e528af76e..c8f19c3193 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,7 +28,6 @@ @author: Jens Timmerman (Ghent University) @author: Kenneth Hoste (Ghent University) """ -import copy import os import re import shutil @@ -44,7 +43,9 @@ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools import config from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import mkdir, write_file +from easybuild.tools.config import get_module_syntax +from easybuild.tools.filetools import mkdir, read_file, write_file +from easybuild.tools.modules import modules_tool class EasyBlockTest(EnhancedTestCase): @@ -61,7 +62,6 @@ def setUp(self): fd, self.eb_file = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(fd) - self.orig_tmp_logdir = os.environ.get('EASYBUILD_TMP_LOGDIR', None) self.test_tmp_logdir = tempfile.mkdtemp() os.environ['EASYBUILD_TMP_LOGDIR'] = self.test_tmp_logdir @@ -77,23 +77,20 @@ def test_easyblock(self): def check_extra_options_format(extra_options): """Make sure extra_options value is of correct format.""" - # EasyBuild v1.x - self.assertTrue(isinstance(extra_options, list)) - for extra_option in extra_options: - self.assertTrue(isinstance(extra_option, tuple)) - self.assertEqual(len(extra_option), 2) - self.assertTrue(isinstance(extra_option[0], basestring)) - self.assertTrue(isinstance(extra_option[1], list)) - self.assertEqual(len(extra_option[1]), 3) - # EasyBuild v2.0 (breaks backward compatibility compared to v1.x) - #self.assertTrue(isinstance(extra_options, dict)) - #for key in extra_options: - # self.assertTrue(isinstance(extra_options[key], list)) - # self.assertTrue(len(extra_options[key]), 3) + # EasyBuild v2.0: dict with keys and values + # (breaks backward compatibility compared to v1.x) + self.assertTrue(isinstance(extra_options, dict)) # conversion to a dict works + extra_options.items() + extra_options.keys() + extra_options.values() + for key in extra_options.keys(): + self.assertTrue(isinstance(extra_options[key], list)) + self.assertTrue(len(extra_options[key]), 3) name = "pi" version = "3.14" - self.contents = '\n'.join([ + self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "%s"' % name, 'version = "%s"' % version, 'homepage = "http://example.com"', @@ -113,12 +110,18 @@ def check_extra_options_format(extra_options): sys.stdout.close() sys.stdout = stdoutorig + # check whether 'This is easyblock' log message is there + tup = ('EasyBlock', 'easybuild.framework.easyblock', '.*easybuild/framework/easyblock.pyc*') + eb_log_msg_re = re.compile(r"INFO This is easyblock %s from module %s (%s)" % tup, re.M) + logtxt = read_file(eb.logfile) + self.assertTrue(eb_log_msg_re.search(logtxt), "Pattern '%s' found in: %s" % (eb_log_msg_re.pattern, logtxt)) + # test extensioneasyblock, as extension exeb1 = ExtensionEasyBlock(eb, {'name': 'foo', 'version': '0.0'}) self.assertEqual(exeb1.cfg['name'], 'foo') extra_options = exeb1.extra_options() check_extra_options_format(extra_options) - self.assertTrue('options' in [key for (key, _) in extra_options]) + self.assertTrue('options' in extra_options) # test extensioneasyblock, as easyblock exeb2 = ExtensionEasyBlock(ec) @@ -126,18 +129,18 @@ def check_extra_options_format(extra_options): self.assertEqual(exeb2.cfg['version'], '3.14') extra_options = exeb2.extra_options() check_extra_options_format(extra_options) - self.assertTrue('options' in [key for (key, _) in extra_options]) + self.assertTrue('options' in extra_options) class TestExtension(ExtensionEasyBlock): @staticmethod def extra_options(): - return ExtensionEasyBlock.extra_options([('extra_param', [None, "help", CUSTOM])]) + return ExtensionEasyBlock.extra_options({'extra_param': [None, "help", CUSTOM]}) texeb = TestExtension(eb, {'name': 'bar'}) self.assertEqual(texeb.cfg['name'], 'bar') extra_options = texeb.extra_options() check_extra_options_format(extra_options) - self.assertTrue('options' in [key for (key, _) in extra_options]) - self.assertEqual([val for (key, val) in extra_options if key == 'extra_param'][0], [None, "help", CUSTOM]) + self.assertTrue('options' in extra_options) + self.assertEqual(extra_options['extra_param'], [None, "help", CUSTOM]) # cleanup eb.close_log() @@ -146,6 +149,7 @@ def extra_options(): def test_fake_module_load(self): """Testcase for fake module load""" self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -165,6 +169,7 @@ def test_fake_module_load(self): def test_make_module_req(self): """Testcase for make_module_req""" self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -187,11 +192,20 @@ def test_make_module_req(self): guess = eb.make_module_req() - self.assertTrue(re.search("^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M)) - self.assertTrue(re.search("^prepend-path\s+CLASSPATH\s+\$root/foo.jar$", guess, re.M)) - self.assertTrue(re.search("^prepend-path\s+MANPATH\s+\$root/share/man$", guess, re.M)) - self.assertTrue(re.search("^prepend-path\s+PATH\s+\$root/bin$", guess, re.M)) - self.assertFalse(re.search("^prepend-path\s+CPATH\s+.*$", guess, re.M)) + if get_module_syntax() == 'Tcl': + self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M)) + self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/foo.jar$", guess, re.M)) + self.assertTrue(re.search(r"^prepend-path\s+MANPATH\s+\$root/share/man$", guess, re.M)) + self.assertTrue(re.search(r"^prepend-path\s+PATH\s+\$root/bin$", guess, re.M)) + self.assertFalse(re.search(r"^prepend-path\s+CPATH\s+.*$", guess, re.M)) + elif get_module_syntax() == 'Lua': + self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "bla.jar"\)\)$', guess, re.M)) + self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "foo.jar"\)\)$', guess, re.M)) + self.assertTrue(re.search(r'^prepend_path\("MANPATH", pathJoin\(root, "share/man"\)\)$', guess, re.M)) + self.assertTrue(re.search(r'^prepend_path\("PATH", pathJoin\(root, "bin"\)\)$', guess, re.M)) + self.assertFalse(re.search(r'^prepend_path\("CPATH", .*\)$', guess, re.M)) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) # cleanup eb.close_log() @@ -200,6 +214,7 @@ def test_make_module_req(self): def test_extensions_step(self): """Test the extensions_step""" self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -212,16 +227,16 @@ def test_extensions_step(self): # test for proper error message without the exts_defaultclass set eb = EasyBlock(EasyConfig(self.eb_file)) eb.installdir = config.install_path() - self.assertRaises(EasyBuildError, eb.extensions_step) - self.assertErrorRegex(EasyBuildError, "No default extension class set", eb.extensions_step) + self.assertRaises(EasyBuildError, eb.extensions_step, fetch=True) + self.assertErrorRegex(EasyBuildError, "No default extension class set", eb.extensions_step, fetch=True) # test if everything works fine if set - self.contents += "\nexts_defaultclass = ['easybuild.framework.extension', 'Extension']" + self.contents += "\nexts_defaultclass = 'DummyExtension'" self.writeEC() eb = EasyBlock(EasyConfig(self.eb_file)) eb.builddir = config.build_path() eb.installdir = config.install_path() - eb.extensions_step() + eb.extensions_step(fetch=True) # test for proper error message when skip is set, but no exts_filter is set self.assertRaises(EasyBuildError, eb.skip_extensions) @@ -234,23 +249,23 @@ def test_extensions_step(self): def test_skip_extensions_step(self): """Test the skip_extensions_step""" self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name": "dummy", "version": "dummy"}', 'exts_list = ["ext1", "ext2"]', - 'exts_filter = ("if [ %(name)s == \'ext2\' ]; then exit 0; else exit 1; fi", "")', - 'exts_defaultclass = ["easybuild.framework.extension", "Extension"]', + 'exts_filter = ("if [ %(ext_name)s == \'ext2\' ]; then exit 0; else exit 1; fi", "")', + 'exts_defaultclass = "DummyExtension"', ]) # check if skip skips correct extensions self.writeEC() eb = EasyBlock(EasyConfig(self.eb_file)) - #self.assertTrue('ext1' in eb.exts.keys() and 'ext2' in eb.exts.keys()) eb.builddir = config.build_path() eb.installdir = config.install_path() eb.skip = True - eb.extensions_step() + eb.extensions_step(fetch=True) # 'ext1' should be in eb.exts self.assertTrue('ext1' in [y for x in eb.exts for y in x.values()]) # 'ext2' should not @@ -264,45 +279,99 @@ def test_make_module_step(self): """Test the make_module_step""" name = "pi" version = "3.14" + deps = [('GCC', '4.6.4')] + hiddendeps = [('toy', '0.0-deps')] + alldeps = deps + hiddendeps # hidden deps must be included in list of deps modextravars = {'PI': '3.1415', 'FOO': 'bar'} modextrapaths = {'PATH': 'pibin', 'CPATH': 'pi/include'} self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "%s"' % name, 'version = "%s"' % version, 'homepage = "http://example.com"', 'description = "test easyconfig"', "toolchain = {'name': 'dummy', 'version': 'dummy'}", - "dependencies = [('foo', '1.2.3')]", - "builddependencies = [('bar', '9.8.7')]", + "dependencies = %s" % str(alldeps), + "hiddendependencies = %s" % str(hiddendeps), + "builddependencies = [('OpenMPI', '1.6.4-GCC-4.6.4')]", "modextravars = %s" % str(modextravars), "modextrapaths = %s" % str(modextrapaths), ]) + test_dir = os.path.dirname(os.path.abspath(__file__)) + os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules') + # test if module is generated correctly self.writeEC() - eb = EasyBlock(EasyConfig(self.eb_file)) + ec = EasyConfig(self.eb_file) + eb = EasyBlock(ec) eb.installdir = os.path.join(config.install_path(), 'pi', '3.14') + eb.check_readiness_step() modpath = os.path.join(eb.make_module_step(), name, version) + if get_module_syntax() == 'Lua': + modpath += '.lua' self.assertTrue(os.path.exists(modpath), "%s exists" % modpath) # verify contents of module - f = open(modpath, 'r') - txt = f.read() - f.close() - self.assertTrue(re.search("^#%Module", txt.split('\n')[0])) - self.assertTrue(re.search("^conflict\s+%s$" % name, txt, re.M)) - self.assertTrue(re.search("^set\s+root\s+%s$" % eb.installdir, txt, re.M)) - self.assertTrue(re.search('^setenv\s+EBROOT%s\s+".root"\s*$' % name.upper(), txt, re.M)) - self.assertTrue(re.search('^setenv\s+EBVERSION%s\s+"%s"$' % (name.upper(), version), txt, re.M)) + txt = read_file(modpath) + if get_module_syntax() == 'Tcl': + self.assertTrue(re.search(r"^#%Module", txt.split('\n')[0])) + self.assertTrue(re.search(r"^conflict\s+%s$" % name, txt, re.M)) + + self.assertTrue(re.search(r"^set\s+root\s+%s$" % eb.installdir, txt, re.M)) + ebroot_regex = re.compile(r'^setenv\s+EBROOT%s\s+"\$root"\s*$' % name.upper(), re.M) + self.assertTrue(ebroot_regex.search(txt), "%s in %s" % (ebroot_regex.pattern, txt)) + self.assertTrue(re.search(r'^setenv\s+EBVERSION%s\s+"%s"$' % (name.upper(), version), txt, re.M)) + + elif get_module_syntax() == 'Lua': + ebroot_regex = re.compile(r'^setenv\("EBROOT%s", root\)$' % name.upper(), re.M) + self.assertTrue(ebroot_regex.search(txt), "%s in %s" % (ebroot_regex.pattern, txt)) + self.assertTrue(re.search(r'^setenv\("EBVERSION%s", "%s"\)$' % (name.upper(), version), txt, re.M)) + + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + for (key, val) in modextravars.items(): - self.assertTrue(re.search('^setenv\s+%s\s+"%s"$' % (key, val), txt, re.M)) + if get_module_syntax() == 'Tcl': + regex = re.compile(r'^setenv\s+%s\s+"%s"$' % (key, val), re.M) + elif get_module_syntax() == 'Lua': + regex = re.compile(r'^setenv\("%s", "%s"\)$' % (key, val), re.M) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) + for (key, val) in modextrapaths.items(): - self.assertTrue(re.search('^prepend-path\s+%s\s+\$root/%s$' % (key, val), txt, re.M)) + if get_module_syntax() == 'Tcl': + regex = re.compile(r'^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M) + elif get_module_syntax() == 'Lua': + regex = re.compile(r'^prepend_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) + + for (name, ver) in deps: + if get_module_syntax() == 'Tcl': + regex = re.compile(r'^\s*module load %s\s*$' % os.path.join(name, ver), re.M) + elif get_module_syntax() == 'Lua': + regex = re.compile(r'^\s*load\("%s"\)$' % os.path.join(name, ver), re.M) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) + + for (name, ver) in hiddendeps: + if get_module_syntax() == 'Tcl': + regex = re.compile(r'^\s*module load %s/.%s\s*$' % (name, ver), re.M) + elif get_module_syntax() == 'Lua': + regex = re.compile(r'^\s*load\("%s/.%s"\)$' % (name, ver), re.M) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) def test_gen_dirs(self): """Test methods that generate/set build/install directory names.""" self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', "name = 'pi'", "version = '3.14'", "homepage = 'http://example.com'", @@ -352,7 +421,7 @@ def test_get_easyblock_instance(self): testdir = os.path.abspath(os.path.dirname(__file__)) import easybuild eb_blocks_path = os.path.join(testdir, 'sandbox') - if not eb_blocks_path in sys.path: + if eb_blocks_path not in sys.path: sys.path.append(eb_blocks_path) easybuild = reload(easybuild) @@ -364,6 +433,51 @@ def test_get_easyblock_instance(self): eb = get_easyblock_instance(ec) self.assertTrue(isinstance(eb, EB_toy)) + # check whether 'This is easyblock' log message is there + tup = ('EB_toy', 'easybuild.easyblocks.toy', '.*test/framework/sandbox/easybuild/easyblocks/t/toy.pyc*') + eb_log_msg_re = re.compile(r"INFO This is easyblock %s from module %s (%s)" % tup, re.M) + logtxt = read_file(eb.logfile) + self.assertTrue(eb_log_msg_re.search(logtxt), "Pattern '%s' found in: %s" % (eb_log_msg_re.pattern, logtxt)) + + def test_fetch_patches(self): + """Test fetch_patches method.""" + # adjust PYTHONPATH such that test easyblocks are found + testdir = os.path.abspath(os.path.dirname(__file__)) + ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'toy-0.0.eb'))[0] + eb = get_easyblock_instance(ec) + + eb.fetch_patches() + self.assertEqual(len(eb.patches), 2) + self.assertEqual(eb.patches[0]['name'], 'toy-0.0_typo.patch') + self.assertFalse('level' in eb.patches[0]) + + # reset + eb.patches = [] + + patches = [ + ('toy-0.0_typo.patch', 0), # should also be level 0 (not None or something else) + ('toy-0.0_typo.patch', 4), # should be level 4 + ('toy-0.0_typo.patch', 'foobar'), # sourcepath should be set to 'foobar' + ('toy-0.0.tar.gz', 'some/path'), # copy mode (not a .patch file) + ] + # check if patch levels are parsed correctly + eb.fetch_patches(patch_specs=patches) + + self.assertEqual(len(eb.patches), 4) + self.assertEqual(eb.patches[0]['name'], 'toy-0.0_typo.patch') + self.assertEqual(eb.patches[0]['level'], 0) + self.assertEqual(eb.patches[1]['name'], 'toy-0.0_typo.patch') + self.assertEqual(eb.patches[1]['level'], 4) + self.assertEqual(eb.patches[2]['name'], 'toy-0.0_typo.patch') + self.assertEqual(eb.patches[2]['sourcepath'], 'foobar') + self.assertEqual(eb.patches[3]['name'], 'toy-0.0.tar.gz'), + self.assertEqual(eb.patches[3]['copy'], 'some/path') + + patches = [ + ('toy-0.0_level4.patch', False), # should throw an error, only int's an strings allowed here + ] + self.assertRaises(EasyBuildError, eb.fetch_patches, patch_specs=patches) + def test_obtain_file(self): """Test obtain_file method.""" toy_tarball = 'toy-0.0.tar.gz' @@ -381,7 +495,7 @@ def test_obtain_file(self): # 'downloading' a file to (first) sourcepath works init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, testdir)]) shutil.copy2(toy_tarball_path, tmpdir_subdir) - res = eb.obtain_file(toy_tarball, urls=[os.path.join('file://', tmpdir_subdir)]) + res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir]) self.assertEqual(res, os.path.join(tmpdir, 't', 'toy', toy_tarball)) # finding a file in sourcepath works @@ -390,33 +504,36 @@ def test_obtain_file(self): self.assertEqual(res, toy_tarball_path) # sourcepath has preference over downloading - res = eb.obtain_file(toy_tarball, urls=[os.path.join('file://', tmpdir_subdir)]) + res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir]) self.assertEqual(res, toy_tarball_path) # obtain_file yields error for non-existing files fn = 'thisisclearlyanonexistingfile' - try: - eb.obtain_file(fn, urls=[os.path.join('file://', tmpdir_subdir)]) - except EasyBuildError, err: - fail_regex = re.compile("Couldn't find file %s anywhere, and downloading it didn't work either" % fn) - self.assertTrue(fail_regex.search(str(err))) + error_regex = "Couldn't find file %s anywhere, and downloading it didn't work either" % fn + self.assertErrorRegex(EasyBuildError, error_regex, eb.obtain_file, fn, urls=['file://%s' % tmpdir_subdir]) # file specifications via URL also work, are downloaded to (first) sourcepath init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, sandbox_sources)]) file_url = "http://hpcugent.github.io/easybuild/index.html" fn = os.path.basename(file_url) + res = None try: res = eb.obtain_file(file_url) + except EasyBuildError, err: + # if this fails, it should be because there's no online access + download_fail_regex = re.compile('socket error') + self.assertTrue(download_fail_regex.search(str(err))) + + # result may be None during offline testing + if res is not None: loc = os.path.join(tmpdir, 't', 'toy', fn) self.assertEqual(res, loc) self.assertTrue(os.path.exists(loc), "%s file is found at %s" % (fn, loc)) txt = open(loc, 'r').read() eb_regex = re.compile("EasyBuild: building software with ease") self.assertTrue(eb_regex.search(txt)) - except EasyBuildError, err: - # if this fails, it should be because there's no online access - download_fail_regex = re.compile('socket error') - self.assertTrue(download_fail_regex.search(str(err))) + else: + print "ignoring failure to download %s in test_obtain_file, testing offline?" % file_url shutil.rmtree(tmpdir) @@ -448,13 +565,177 @@ def test_check_readiness(self): shutil.rmtree(tmpdir) - def tearDown(self): - """ make sure to remove the temporary file """ - super(EasyBlockTest, self).tearDown() + def test_exclude_path_to_top_of_module_tree(self): + """ + Make sure that modules under the HierarchicalMNS are correct, + w.r.t. not including any load statements for modules that build up the path to the top of the module tree. + """ + self.orig_module_naming_scheme = config.get_module_naming_scheme() + test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + all_stops = [x[0] for x in EasyBlock.get_steps()] + build_options = { + 'check_osdeps': False, + 'robot_path': [test_ecs_path], + 'valid_stops': all_stops, + 'validate': False, + } + os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS' + init_config(build_options=build_options) + self.setup_hierarchical_modules() + + modfile_prefix = os.path.join(self.test_installpath, 'modules', 'all') + mkdir(os.path.join(modfile_prefix, 'Compiler', 'GCC', '4.8.3'), parents=True) + mkdir(os.path.join(modfile_prefix, 'MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049'), parents=True) + + impi_modfile_path = os.path.join('Compiler', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049') + imkl_modfile_path = os.path.join('MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049', 'imkl', '11.1.2.144') + if get_module_syntax() == 'Lua': + impi_modfile_path += '.lua' + imkl_modfile_path += '.lua' + + # example: for imkl on top of iimpi toolchain with HierarchicalMNS, no module load statements should be included + # not for the toolchain or any of the toolchain components, + # since both icc/ifort and impi form the path to the top of the module tree + tests = [ + ('impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb', impi_modfile_path, ['icc', 'ifort', 'iccifort']), + ('imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb', imkl_modfile_path, ['icc', 'ifort', 'impi', 'iccifort', 'iimpi']), + ] + for ec_file, modfile_path, excluded_deps in tests: + ec = EasyConfig(os.path.join(test_ecs_path, ec_file)) + eb = EasyBlock(ec) + eb.toolchain.prepare() + modpath = eb.make_module_step() + modfile_path = os.path.join(modpath, modfile_path) + modtxt = read_file(modfile_path) + + for imkl_dep in excluded_deps: + tup = (imkl_dep, modfile_path, modtxt) + failmsg = "No 'module load' statement found for '%s' not found in module %s: %s" % tup + self.assertFalse(re.search("module load %s" % imkl_dep, modtxt), failmsg) + + os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme + init_config(build_options=build_options) + + def test_patch_step(self): + """Test patch step.""" + test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') + ec = process_easyconfig(os.path.join(test_easyconfigs, 'toy-0.0.eb'))[0] + orig_sources = ec['ec']['sources'][:] + + toy_patches = [ + 'toy-0.0_typo.patch', # test for applying patch + ('toy-extra.txt', 'toy-0.0'), # test for patch-by-copy + ] + self.assertEqual(ec['ec']['patches'], toy_patches) + + # test applying patches without sources + ec['ec']['sources'] = [] + eb = EasyBlock(ec['ec']) + eb.fetch_step() + eb.extract_step() + self.assertErrorRegex(EasyBuildError, '.*', eb.patch_step) - os.remove(self.eb_file) - if self.orig_tmp_logdir is not None: - os.environ['EASYBUILD_TMP_LOGDIR'] = self.orig_tmp_logdir + # test actual patching of unpacked sources + ec['ec']['sources'] = orig_sources + eb = EasyBlock(ec['ec']) + eb.fetch_step() + eb.extract_step() + eb.patch_step() + # verify that patches were applied + toydir = os.path.join(eb.builddir, 'toy-0.0') + self.assertEqual(sorted(os.listdir(toydir)), ['toy-extra.txt', 'toy.source', 'toy.source.orig']) + self.assertTrue("and very proud of it" in read_file(os.path.join(toydir, 'toy.source'))) + self.assertEqual(read_file(os.path.join(toydir, 'toy-extra.txt')), 'moar!\n') + + def test_extensions_sanity_check(self): + """Test sanity check aspect of extensions.""" + test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') + toy_ec = EasyConfig(os.path.join(test_ecs_dir, 'toy-0.0-gompi-1.3.12-test.eb')) + + # purposely put sanity check command in place that breaks the build, + # to check whether sanity check is only run once; + # sanity check commands are checked after checking sanity check paths, so this should work + toy_ec.update('sanity_check_commands', [("%(installdir)s/bin/toy && rm %(installdir)s/bin/toy", '')]) + + # this import only works here, since EB_toy is a test easyblock + from easybuild.easyblocks.toy import EB_toy + eb = EB_toy(toy_ec) + eb.silent = True + eb.run_all_steps(True) + + def test_parallel(self): + """Test defining of parallellism.""" + toy_ec = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'toy-0.0.eb') + toytxt = read_file(toy_ec) + + handle, toy_ec1 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') + os.close(handle) + write_file(toy_ec1, toytxt + "\nparallel = 123") + + handle, toy_ec2 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') + os.close(handle) + write_file(toy_ec2, toytxt + "\nparallel = 123\nmaxparallel = 67") + + # default: parallellism is derived from # available cores + ulimit + test_eb = EasyBlock(EasyConfig(toy_ec)) + test_eb.check_readiness_step() + self.assertTrue(isinstance(test_eb.cfg['parallel'], int) and test_eb.cfg['parallel'] > 0) + + # only 'parallel' easyconfig parameter specified (no 'parallel' build option) + test_eb = EasyBlock(EasyConfig(toy_ec1)) + test_eb.check_readiness_step() + self.assertEqual(test_eb.cfg['parallel'], 123) + + # both 'parallel' and 'maxparallel' easyconfig parameters specified (no 'parallel' build option) + test_eb = EasyBlock(EasyConfig(toy_ec2)) + test_eb.check_readiness_step() + self.assertEqual(test_eb.cfg['parallel'], 67) + + # only 'parallel' build option specified + init_config(build_options={'parallel': '97', 'validate': False}) + test_eb = EasyBlock(EasyConfig(toy_ec)) + test_eb.check_readiness_step() + self.assertEqual(test_eb.cfg['parallel'], 97) + + # both 'parallel' build option and easyconfig parameter specified (no 'maxparallel') + test_eb = EasyBlock(EasyConfig(toy_ec1)) + test_eb.check_readiness_step() + self.assertEqual(test_eb.cfg['parallel'], 97) + + # both 'parallel' and 'maxparallel' easyconfig parameters specified + 'parallel' build option + test_eb = EasyBlock(EasyConfig(toy_ec2)) + test_eb.check_readiness_step() + self.assertEqual(test_eb.cfg['parallel'], 67) + + def test_guess_start_dir(self): + """Test guessing the start dir.""" + test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') + ec = process_easyconfig(os.path.join(test_easyconfigs, 'toy-0.0.eb'))[0] + + def check_start_dir(expected_start_dir): + """Check start dir.""" + eb = EasyBlock(ec['ec']) + eb.silent = True + eb.cfg['stop'] = 'patch' + eb.run_all_steps(False) + eb.guess_start_dir() + abs_expected_start_dir = os.path.join(eb.builddir, expected_start_dir) + self.assertTrue(os.path.samefile(eb.cfg['start_dir'], abs_expected_start_dir)) + self.assertTrue(os.path.samefile(os.getcwd(), abs_expected_start_dir)) + + # default (no start_dir specified): use unpacked dir as start dir + self.assertEqual(ec['ec']['start_dir'], None) + check_start_dir('toy-0.0') + + # using start_dir equal to the one we're in is OK + ec['ec']['start_dir'] = '%(name)s-%(version)s' + self.assertEqual(ec['ec']['start_dir'], 'toy-0.0') + check_start_dir('toy-0.0') + + # clean error when specified start dir does not exist + ec['ec']['start_dir'] = 'thisstartdirisnotthere' + err_pattern = "Specified start dir .*/toy-0.0/thisstartdirisnotthere does not exist" + self.assertErrorRegex(EasyBuildError, err_pattern, check_start_dir, 'whatever') def suite(): diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index ec7c411f94..225f848ba4 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,11 +29,12 @@ @author: Kenneth Hoste (Ghent University) @author: Stijn De Weirdt (Ghent University) """ - +import copy import os import re import shutil import tempfile +from distutils.version import LooseVersion from test.framework.utilities import EnhancedTestCase, init_config from unittest import TestLoader, main from vsc.utils.fancylogger import setLogLevelDebug, logToScreen @@ -41,20 +42,37 @@ import easybuild.tools.build_log import easybuild.framework.easyconfig as easyconfig from easybuild.framework.easyblock import EasyBlock -from easybuild.framework.easyconfig.easyconfig import EasyConfig -from easybuild.framework.easyconfig.easyconfig import create_paths, det_installversion -from easybuild.framework.easyconfig.easyconfig import fetch_parameter_from_easyconfig_file, get_easyblock_class +from easybuild.framework.easyconfig.constants import EXTERNAL_MODULE_MARKER +from easybuild.framework.easyconfig.easyconfig import ActiveMNS, EasyConfig +from easybuild.framework.easyconfig.easyconfig import create_paths +from easybuild.framework.easyconfig.easyconfig import get_easyblock_class +from easybuild.framework.easyconfig.licenses import License, LicenseGPLv3 +from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig +from easybuild.framework.easyconfig.templates import to_template_str +from easybuild.framework.easyconfig.tools import dep_graph, find_related_easyconfigs, parse_easyconfigs from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak_one from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import module_classes +from easybuild.tools.configobj import ConfigObj from easybuild.tools.filetools import read_file, write_file from easybuild.tools.module_naming_scheme.toolchain import det_toolchain_compilers, det_toolchain_mpi from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version +from easybuild.tools.robot import resolve_dependencies from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.utilities import quote_str from test.framework.utilities import find_full_path +EXPECTED_DOTTXT_TOY_DEPS = """digraph graphname { +"GCC/4.7.2 (EXT)"; +toy; +ictce; +toy -> ictce; +toy -> "GCC/4.7.2 (EXT)"; +} +""" + + class EasyConfigTest(EnhancedTestCase): """ easyconfig tests """ contents = None @@ -69,8 +87,6 @@ def setUp(self): if os.path.exists(self.eb_file): os.remove(self.eb_file) - self.orig_current_version = easybuild.tools.build_log.CURRENT_VERSION - def prep(self): """Prepare for test.""" # (re)cleanup last test file @@ -83,7 +99,6 @@ def prep(self): def tearDown(self): """ make sure to remove the temporary file """ - easybuild.tools.build_log.CURRENT_VERSION = self.orig_current_version super(EasyConfigTest, self).tearDown() if os.path.exists(self.eb_file): os.remove(self.eb_file) @@ -96,13 +111,14 @@ def test_empty(self): self.assertErrorRegex(EasyBuildError, "expected a valid path", EasyConfig, "") def test_mandatory(self): - """ make sure all checking of mandatory variables works """ + """ make sure all checking of mandatory parameters works """ self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', ]) self.prep() - self.assertErrorRegex(EasyBuildError, "mandatory variables? .* not provided", EasyConfig, self.eb_file) + self.assertErrorRegex(EasyBuildError, "mandatory parameters not provided", EasyConfig, self.eb_file) self.contents += '\n' + '\n'.join([ 'homepage = "http://example.com"', @@ -120,8 +136,9 @@ def test_mandatory(self): self.assertEqual(eb['description'], "test easyconfig") def test_validation(self): - """ test other validations beside mandatory variables """ + """ test other validations beside mandatory parameters """ self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -151,15 +168,16 @@ def test_validation(self): self.prep() self.assertErrorRegex(EasyBuildError, "SyntaxError", EasyConfig, self.eb_file) - def test_shared_lib_ext(self): + def test_shlib_ext(self): """ inside easyconfigs shared_lib_ext should be set """ self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name":"dummy", "version": "dummy"}', - 'sanity_check_paths = { "files": ["lib/lib.%s" % shared_lib_ext] }', + 'sanity_check_paths = { "files": ["lib/lib.%s" % SHLIB_EXT] }', ]) self.prep() eb = EasyConfig(self.eb_file) @@ -168,6 +186,7 @@ def test_shared_lib_ext(self): def test_dependency(self): """ test all possible ways of specifying dependencies """ self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -210,10 +229,14 @@ def test_dependency(self): self.assertErrorRegex(EasyBuildError, "Dependency foo of unsupported type", eb._parse_dependency, "foo") self.assertErrorRegex(EasyBuildError, "without name", eb._parse_dependency, ()) self.assertErrorRegex(EasyBuildError, "without version", eb._parse_dependency, {'name': 'test'}) + err_msg = "Incorrect external dependency specification" + self.assertErrorRegex(EasyBuildError, err_msg, eb._parse_dependency, (EXTERNAL_MODULE_MARKER,)) + self.assertErrorRegex(EasyBuildError, err_msg, eb._parse_dependency, ('foo', '1.2.3', EXTERNAL_MODULE_MARKER)) def test_extra_options(self): """ extra_options should allow other variables to be stored """ self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -224,7 +247,7 @@ def test_extra_options(self): ]) self.prep() eb = EasyConfig(self.eb_file) - self.assertRaises(KeyError, lambda: eb['custom_key']) + self.assertErrorRegex(EasyBuildError, "unknown easyconfig parameter", lambda: eb['custom_key']) extra_vars = {'custom_key': ['default', "This is a default key", easyconfig.CUSTOM]} @@ -247,15 +270,10 @@ def test_extra_options(self): # test if extra toolchain options are being passed self.assertEqual(eb.toolchain.options['static'], True) - # test legacy behavior of passing a list of tuples rather than a dict - eb = EasyConfig(self.eb_file, extra_options=extra_vars.items()) - self.assertEqual(eb['custom_key'], 'test') - + # test extra mandatory parameters extra_vars.update({'mandatory_key': ['default', 'another mandatory key', easyconfig.MANDATORY]}) - - # test extra mandatory vars - self.assertErrorRegex(EasyBuildError, r"mandatory variables? \S* not provided", - EasyConfig, self.eb_file, extra_vars) + self.assertErrorRegex(EasyBuildError, r"mandatory parameters not provided", + EasyConfig, self.eb_file, extra_options=extra_vars) self.contents += '\nmandatory_key = "value"' self.prep() @@ -269,6 +287,7 @@ def test_exts_list(self): os.environ['EASYBUILD_SOURCEPATH'] = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') init_config() self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -284,8 +303,8 @@ def test_exts_list(self): ' "source_urls": [("http://example.com", "suffix")],' ' "patches": ["toy-0.0.eb"],', # dummy patch to avoid downloading fail ' "checksums": [', - ' "504c7036558938f997c1c269a01d7458",', # checksum for source (gzip-1.4.eb) - ' "ddd5161154f5db67701525123129ff09",', # checksum for patch (toy-0.0.eb) + ' "a5464d79c2c8d4935e383ebd070b305e",', # MD5 checksum for source (gzip-1.4.eb) + ' "fad34da3432ee2fd4d6554b86c8df4bf",', # MD5 checksum for patch (toy-0.0.eb) ' ],', ' }),', ']', @@ -298,6 +317,7 @@ def test_exts_list(self): def test_suggestions(self): """ If a typo is present, suggestions should be provided (if possible) """ self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -321,11 +341,12 @@ def test_tweaking(self): os.close(fd) patches = ["t1.patch", ("t2.patch", 1), ("t3.patch", "test"), ("t4.h", "include")] self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'homepage = "http://www.example.com"', 'description = "dummy description"', 'version = "3.14"', - 'toolchain = {"name":"GCC", "version": "4.6.3"}', + 'toolchain = {"name": "GCC", "version": "4.6.3"}', 'patches = %s', ]) % str(patches) self.prep() @@ -333,8 +354,8 @@ def test_tweaking(self): ver = "1.2.3" verpref = "myprefix" versuff = "mysuffix" - tcname = "mytc" - tcver = "4.1.2" + tcname = "gompi" + tcver = "1.4.10" new_patches = ['t5.patch', 't6.patch'] homepage = "http://www.justatest.com" @@ -366,7 +387,6 @@ def test_tweaking(self): 'toolchain_name': tcname, 'patches': new_patches[:1], 'homepage': homepage, - 'foo': "bar" } tweak_one(self.eb_file, tweaked_fn, tweaks) @@ -416,24 +436,6 @@ def test_installversion(self): installver = det_full_ec_version(cfg) self.assertEqual(installver, correct_installver) - def test_legacy_installversion(self): - """Test generation of install version (legacy).""" - - ver = "3.14" - verpref = "myprefix|" - versuff = "|mysuffix" - tcname = "GCC" - tcver = "4.6.3" - dummy = "dummy" - - correct_installver = "%s%s-%s-%s%s" % (verpref, ver, tcname, tcver, versuff) - installver = det_installversion(ver, tcname, tcver, verpref, versuff) - self.assertEqual(installver, correct_installver) - - correct_installver = "%s%s%s" % (verpref, ver, versuff) - installver = det_installversion(ver, dummy, tcver, verpref, versuff) - self.assertEqual(installver, correct_installver) - def test_obtain_easyconfig(self): """test obtaining an easyconfig file given certain specifications""" @@ -447,41 +449,51 @@ def test_obtain_easyconfig(self): "pi-3.15-GCC-4.3.2.eb", "pi-3.15-GCC-4.4.5.eb", "foo-1.2.3-GCC-4.3.2.eb"] - eb_files = [(fns[0], "\n".join(['name = "pi"', - 'version = "3.12"', - 'homepage = "http://example.com"', - 'description = "test easyconfig"', - 'toolchain = {"name": "dummy", "version": "dummy"}', - 'patches = %s' % patches - ])), - (fns[1], "\n".join(['name = "pi"', - 'version = "3.13"', - 'homepage = "http://example.com"', - 'description = "test easyconfig"', - 'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver), - 'patches = %s' % patches - ])), - (fns[2], "\n".join(['name = "pi"', - 'version = "3.15"', - 'homepage = "http://example.com"', - 'description = "test easyconfig"', - 'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver), - 'patches = %s' % patches - ])), - (fns[3], "\n".join(['name = "pi"', - 'version = "3.15"', - 'homepage = "http://example.com"', - 'description = "test easyconfig"', - 'toolchain = {"name": "%s", "version": "4.5.1"}' % tcname, - 'patches = %s' % patches - ])), - (fns[4], "\n".join(['name = "foo"', - 'version = "1.2.3"', - 'homepage = "http://example.com"', - 'description = "test easyconfig"', - 'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver), - 'foo_extra1 = "bar"', - ])) + eb_files = [(fns[0], "\n".join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.12"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = {"name": "dummy", "version": "dummy"}', + 'patches = %s' % patches + ])), + (fns[1], "\n".join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.13"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver), + 'patches = %s' % patches + ])), + (fns[2], "\n".join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.15"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver), + 'patches = %s' % patches + ])), + (fns[3], "\n".join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.15"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = {"name": "%s", "version": "4.5.1"}' % tcname, + 'patches = %s' % patches + ])), + (fns[4], "\n".join([ + 'easyblock = "ConfigureMake"', + 'name = "foo"', + 'version = "1.2.3"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver), + 'foo_extra1 = "bar"', + ])) ] @@ -496,7 +508,10 @@ def test_obtain_easyconfig(self): self.assertErrorRegex(EasyBuildError, error_regexp, obtain_ec_for, specs, [self.ec_dir], None) # should find matching easyconfig file - specs = {'name': 'foo', 'version': '1.2.3'} + specs = { + 'name': 'foo', + 'version': '1.2.3' + } res = obtain_ec_for(specs, [self.ec_dir], None) self.assertEqual(res[0], False) self.assertEqual(res[1], os.path.join(self.ec_dir, fns[-1])) @@ -519,7 +534,7 @@ def test_obtain_easyconfig(self): 'toolchain_name': tcname, 'toolchain_version': tcver, 'version': ver, - 'foo': 'bar123' + 'start_dir': 'bar123' }) res = obtain_ec_for(specs, [self.ec_dir], None) self.assertEqual(res[1], "%s-%s-%s-%s%s.eb" % (name, ver, tcname, tcver, suff)) @@ -530,11 +545,16 @@ def test_obtain_easyconfig(self): self.assertEqual(ec['version'], specs['version']) self.assertEqual(ec['versionsuffix'], specs['versionsuffix']) self.assertEqual(ec['toolchain'], {'name': tcname, 'version': tcver}) - # can't check for key 'foo', because EasyConfig ignores parameter names it doesn't know about - txt = read_file(res[1]) - self.assertTrue(re.search('foo = "%s"' % specs['foo'], txt)) + self.assertEqual(ec['start_dir'], specs['start_dir']) os.remove(res[1]) + specs.update({ + 'foo': 'bar123' + }) + self.assertErrorRegex(EasyBuildError, "Unkown easyconfig parameter: foo", + obtain_ec_for, specs, [self.ec_dir], None) + del specs['foo'] + # should pick correct version, i.e. not newer than what's specified, if a choice needs to be made ver = '3.14' specs.update({'version': ver}) @@ -543,7 +563,7 @@ def test_obtain_easyconfig(self): ec = EasyConfig(res[1]) self.assertEqual(ec['version'], specs['version']) txt = read_file(res[1]) - self.assertTrue(re.search("version = [\"']%s[\"'] .*was: [\"']3.13[\"']" % ver, txt)) + self.assertTrue(re.search("^version = [\"']%s[\"']$" % ver, txt, re.M)) os.remove(res[1]) # should pick correct toolchain version as well, i.e. now newer than what's specified, if a choice needs to be made @@ -557,16 +577,16 @@ def test_obtain_easyconfig(self): self.assertEqual(ec['version'], specs['version']) self.assertEqual(ec['toolchain']['version'], specs['toolchain_version']) txt = read_file(res[1]) - pattern = "toolchain = .*version.*[\"']%s[\"'].*was: .*version.*[\"']%s[\"']" % (specs['toolchain_version'], tcver) - self.assertTrue(re.search(pattern, txt)) + pattern = "^toolchain = .*version.*[\"']%s[\"'].*}$" % specs['toolchain_version'] + self.assertTrue(re.search(pattern, txt, re.M)) os.remove(res[1]) - # should be able to prepend to list of patches and handle list of dependencies new_patches = ['two.patch', 'three.patch'] specs.update({ 'patches': new_patches[:], 'dependencies': [('foo', '1.2.3'), ('bar', '666', '-bleh', ('gompi', '1.4.10'))], + 'hiddendependencies': [('test', '3.2.1')], }) parsed_deps = [ { @@ -577,6 +597,9 @@ def test_obtain_easyconfig(self): 'dummy': False, 'short_mod_name': 'foo/1.2.3-GCC-4.4.5', 'full_mod_name': 'foo/1.2.3-GCC-4.4.5', + 'hidden': False, + 'external_module': False, + 'external_module_metadata': {}, }, { 'name': 'bar', @@ -586,13 +609,48 @@ def test_obtain_easyconfig(self): 'dummy': False, 'short_mod_name': 'bar/666-gompi-1.4.10-bleh', 'full_mod_name': 'bar/666-gompi-1.4.10-bleh', + 'hidden': False, + 'external_module': False, + 'external_module_metadata': {}, + }, + { + 'name': 'test', + 'version': '3.2.1', + 'versionsuffix': '', + 'toolchain': ec['toolchain'], + 'dummy': False, + 'short_mod_name': 'test/.3.2.1-GCC-4.4.5', + 'full_mod_name': 'test/.3.2.1-GCC-4.4.5', + 'hidden': True, + 'external_module': False, + 'external_module_metadata': {}, }, ] + + # hidden dependencies must be included in list of dependencies + res = obtain_ec_for(specs, [self.ec_dir], None) + self.assertEqual(res[0], True) + error_pattern = "Hidden dependencies with visible module names .* not in list of dependencies: .*" + self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, res[1]) + + specs['dependencies'].append(('test', '3.2.1')) + res = obtain_ec_for(specs, [self.ec_dir], None) self.assertEqual(res[0], True) ec = EasyConfig(res[1]) self.assertEqual(ec['patches'], specs['patches']) - self.assertEqual(ec['dependencies'], parsed_deps) + self.assertEqual(ec.dependencies(), parsed_deps) + + # hidden dependencies are filtered from list of dependencies + self.assertFalse('test/3.2.1-GCC-4.4.5' in [d['full_mod_name'] for d in ec['dependencies']]) + self.assertTrue('test/.3.2.1-GCC-4.4.5' in [d['full_mod_name'] for d in ec['hiddendependencies']]) + os.remove(res[1]) + + # hidden dependencies are also filtered from list of dependencies when validation is skipped + res = obtain_ec_for(specs, [self.ec_dir], None) + ec = EasyConfig(res[1], validate=False) + self.assertFalse('test/3.2.1-GCC-4.4.5' in [d['full_mod_name'] for d in ec['dependencies']]) + self.assertTrue('test/.3.2.1-GCC-4.4.5' in [d['full_mod_name'] for d in ec['hiddendependencies']]) os.remove(res[1]) # verify append functionality for lists @@ -658,6 +716,7 @@ def test_templating(self): } # don't use any escaping insanity here, since it is templated itself self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "%(name)s"', 'version = "%(version)s"', 'homepage = "http://example.com/%%(nameletter)s/%%(nameletterlower)s"', @@ -710,6 +769,7 @@ def test_constant_doc(self): def test_build_options(self): """Test configure/build/install options, both strings and lists.""" orig_contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -779,6 +839,7 @@ def test_build_options(self): def test_buildininstalldir(self): """Test specifying build in install dir.""" self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -812,24 +873,26 @@ def test_format_equivalence_basic(self): ('gzip-1.4.eb', 'gzip.eb', {'version': '1.4'}), ('gzip-1.4.eb', 'gzip.eb', {'version': '1.4', 'toolchain': {'name': 'dummy', 'version': 'dummy'}}), ('gzip-1.4-GCC-4.6.3.eb', 'gzip.eb', {'version': '1.4', 'toolchain': {'name': 'GCC', 'version': '4.6.3'}}), - ('gzip-1.5-goolf-1.4.10.eb', 'gzip.eb', {'version': '1.5', 'toolchain': {'name': 'goolf', 'version': '1.4.10'}}), - ('gzip-1.5-ictce-4.1.13.eb', 'gzip.eb', {'version': '1.5', 'toolchain': {'name': 'ictce', 'version': '4.1.13'}}), + ('gzip-1.5-goolf-1.4.10.eb', 'gzip.eb', + {'version': '1.5', 'toolchain': {'name': 'goolf', 'version': '1.4.10'}}), + ('gzip-1.5-ictce-4.1.13.eb', 'gzip.eb', + {'version': '1.5', 'toolchain': {'name': 'ictce', 'version': '4.1.13'}}), ]: ec1 = EasyConfig(os.path.join(easyconfigs_path, 'v1.0', eb_file1), validate=False) ec2 = EasyConfig(os.path.join(easyconfigs_path, 'v2.0', eb_file2), validate=False, build_specs=specs) ec2_dict = ec2.asdict() - # reset mandatory attributes from format2 that are not in format 1 - for attr in ['docurls', 'software_license', 'software_license_urls']: + # reset mandatory attributes from format2 that are not defined in format 1 easyconfigs + for attr in ['docurls', 'software_license_urls']: ec2_dict[attr] = None - self.assertEqual(ec1.asdict(), ec2_dict) + self.assertEqual(ec1.asdict(), ec2_dict, "Parsed %s is equivalent with %s" % (eb_file1, eb_file2)) # restore easybuild.tools.build_log.EXPERIMENTAL = orig_experimental - def test_fetch_parameter_from_easyconfig_file(self): - """Test fetch_easyblock_from_easyconfig_file function.""" + def test_fetch_parameters_from_easyconfig(self): + """Test fetch_parameters_from_easyconfig function.""" test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') toy_ec_file = os.path.join(test_ecs_dir, 'toy-0.0.eb') @@ -837,12 +900,11 @@ def test_fetch_parameter_from_easyconfig_file(self): (toy_ec_file, 'toy', None), (os.path.join(test_ecs_dir, 'goolf-1.4.10.eb'), 'goolf', 'Toolchain'), ]: - name = fetch_parameter_from_easyconfig_file(ec_file, 'name') + name, easyblock = fetch_parameters_from_easyconfig(read_file(ec_file), ['name', 'easyblock']) self.assertEqual(name, correct_name) - easyblock = fetch_parameter_from_easyconfig_file(ec_file, 'easyblock') self.assertEqual(easyblock, correct_easyblock) - self.assertEqual(fetch_parameter_from_easyconfig_file(toy_ec_file, 'description'), "Toy C program.") + self.assertEqual(fetch_parameters_from_easyconfig(read_file(toy_ec_file), ['description'])[0], "Toy C program.") def test_get_easyblock_class(self): """Test get_easyblock_class function.""" @@ -857,8 +919,10 @@ def test_get_easyblock_class(self): ]: self.assertEqual(get_easyblock_class(easyblock), easyblock_class) - self.assertEqual(get_easyblock_class(None, name='gzip'), ConfigureMake) + self.assertEqual(get_easyblock_class(None, name='gzip', default_fallback=False), None) self.assertEqual(get_easyblock_class(None, name='toy'), EB_toy) + self.assertErrorRegex(EasyBuildError, "Failed to import EB_TOY", get_easyblock_class, None, name='TOY') + self.assertEqual(get_easyblock_class(None, name='TOY', error_on_failed_import=False), None) def test_easyconfig_paths(self): """Test create_paths function.""" @@ -871,27 +935,6 @@ def test_easyconfig_paths(self): ] self.assertEqual(cand_paths, expected_paths) - def test_deprecated_options(self): - """Test whether deprecated options are handled correctly.""" - deprecated_options = [ - ('makeopts', 'buildopts', 'CC=foo'), - ('premakeopts', 'prebuildopts', ['PATH=%(builddir)s/foo:$PATH', 'PATH=%(builddir)s/bar:$PATH']), - ] - clean_contents = [ - 'name = "pi"', - 'version = "3.14"', - 'homepage = "http://example.com"', - 'description = "test easyconfig"', - 'toolchain = {"name": "dummy", "version": "dummy"}', - 'buildininstalldir = True', - ] - # alternative option is ready to use - for depr_opt, new_opt, val in deprecated_options: - self.contents = '\n'.join(clean_contents + ['%s = %s' % (depr_opt, quote_str(val))]) - self.prep() - ec = EasyConfig(self.eb_file) - self.assertEqual(ec[depr_opt], ec[new_opt]) - def test_toolchain_inspection(self): """Test whether available toolchain inspection functionality is working.""" build_options = { @@ -939,6 +982,564 @@ def test_filter_deps(self): opts = init_config(args=['--filter-deps=zlib,ncurses']) self.assertEqual(opts.filter_deps, ['zlib', 'ncurses']) + def test_replaced_easyconfig_parameters(self): + """Test handling of replaced easyconfig parameters.""" + test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') + ec = EasyConfig(os.path.join(test_ecs_dir, 'toy-0.0.eb')) + replaced_parameters = { + 'license': ('license_file', '2.0'), + 'makeopts': ('buildopts', '2.0'), + 'premakeopts': ('prebuildopts', '2.0'), + } + for key, (newkey, ver) in replaced_parameters.items(): + error_regex = "NO LONGER SUPPORTED since v%s.*'%s' is replaced by '%s'" % (ver, key, newkey) + self.assertErrorRegex(EasyBuildError, error_regex, ec.get, key) + self.assertErrorRegex(EasyBuildError, error_regex, lambda k: ec[k], key) + def foo(key): + ec[key] = 'foo' + self.assertErrorRegex(EasyBuildError, error_regex, foo, key) + + def test_deprecated_easyconfig_parameters(self): + """Test handling of replaced easyconfig parameters.""" + os.environ.pop('EASYBUILD_DEPRECATED') + easybuild.tools.build_log.CURRENT_VERSION = self.orig_current_version + init_config() + + test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') + ec = EasyConfig(os.path.join(test_ecs_dir, 'toy-0.0.eb')) + + orig_deprecated_parameters = copy.deepcopy(easyconfig.parser.DEPRECATED_PARAMETERS) + easyconfig.parser.DEPRECATED_PARAMETERS.update({ + 'foobar': ('barfoo', '0.0'), # deprecated since forever + 'foobarbarfoo': ('barfoofoobar', '1000000000'), # won't be actually deprecated for a while + }) + + # copy classes before reloading, so we can restore them (other isinstance checks fail) + orig_EasyConfig = copy.deepcopy(easyconfig.easyconfig.EasyConfig) + orig_ActiveMNS = copy.deepcopy(easyconfig.easyconfig.ActiveMNS) + reload(easyconfig.parser) + + for key, (newkey, depr_ver) in easyconfig.parser.DEPRECATED_PARAMETERS.items(): + if LooseVersion(depr_ver) <= easybuild.tools.build_log.CURRENT_VERSION: + # deprecation error + error_regex = "DEPRECATED.*since v%s.*'%s' is deprecated.*use '%s' instead" % (depr_ver, key, newkey) + self.assertErrorRegex(EasyBuildError, error_regex, ec.get, key) + self.assertErrorRegex(EasyBuildError, error_regex, lambda k: ec[k], key) + def foo(key): + ec[key] = 'foo' + self.assertErrorRegex(EasyBuildError, error_regex, foo, key) + else: + # only deprecation warning, but key is replaced when getting/setting + ec[key] = 'test123' + self.assertEqual(ec[newkey], 'test123') + self.assertEqual(ec[key], 'test123') + ec[newkey] = '123test' + self.assertEqual(ec[newkey], '123test') + self.assertEqual(ec[key], '123test') + + easyconfig.parser.DEPRECATED_PARAMETERS = orig_deprecated_parameters + reload(easyconfig.parser) + easyconfig.easyconfig.EasyConfig = orig_EasyConfig + easyconfig.easyconfig.ActiveMNS = orig_ActiveMNS + + def test_unknown_easyconfig_parameter(self): + """Check behaviour when unknown easyconfig parameters are used.""" + self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.14"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = {"name": "dummy", "version": "dummy"}', + ]) + self.prep() + ec = EasyConfig(self.eb_file) + self.assertFalse('therenosucheasyconfigparameterlikethis' in ec) + error_regex = "unknown easyconfig parameter" + self.assertErrorRegex(EasyBuildError, error_regex, lambda k: ec[k], 'therenosucheasyconfigparameterlikethis') + def set_ec_key(key): + """Dummy function to set easyconfig parameter in 'ec' EasyConfig instance""" + ec[key] = 'foobar' + self.assertErrorRegex(EasyBuildError, error_regex, set_ec_key, 'therenosucheasyconfigparameterlikethis') + + def test_external_dependencies(self): + """Test specifying external (build) dependencies.""" + ectxt = read_file(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0-deps.eb')) + toy_ec = os.path.join(self.test_prefix, 'toy-0.0-external-deps.eb') + + # just specify some of the test modules we ship, doesn't matter where they come from + ectxt += "\ndependencies += [('foobar/1.2.3', EXTERNAL_MODULE)]" + ectxt += "\nbuilddependencies = [('somebuilddep/0.1', EXTERNAL_MODULE)]" + write_file(toy_ec, ectxt) + + ec = EasyConfig(toy_ec) + + builddeps = ec.builddependencies() + self.assertEqual(len(builddeps), 1) + self.assertEqual(builddeps[0]['short_mod_name'], 'somebuilddep/0.1') + self.assertEqual(builddeps[0]['full_mod_name'], 'somebuilddep/0.1') + self.assertEqual(builddeps[0]['external_module'], True) + + deps = ec.dependencies() + self.assertEqual(len(deps), 4) + correct_deps = ['ictce/4.1.13', 'GCC/4.7.2', 'foobar/1.2.3', 'somebuilddep/0.1'] + self.assertEqual([d['short_mod_name'] for d in deps], correct_deps) + self.assertEqual([d['full_mod_name'] for d in deps], correct_deps) + self.assertEqual([d['external_module'] for d in deps], [False, True, True, True]) + + metadata = os.path.join(self.test_prefix, 'external_modules_metadata.cfg') + metadatatxt = '\n'.join(['[foobar/1.2.3]', 'name = foo,bar', 'version = 1.2.3,3.2.1', 'prefix = /foo/bar']) + write_file(metadata, metadatatxt) + cfg = init_config(args=['--external-modules-metadata=%s' % metadata]) + build_options = { + 'external_modules_metadata': cfg.external_modules_metadata, + 'valid_module_classes': module_classes(), + } + init_config(build_options=build_options) + ec = EasyConfig(toy_ec) + self.assertEqual(ec.dependencies()[2]['short_mod_name'], 'foobar/1.2.3') + self.assertEqual(ec.dependencies()[2]['external_module'], True) + metadata = { + 'name': ['foo', 'bar'], + 'version': ['1.2.3', '3.2.1'], + 'prefix': '/foo/bar', + } + self.assertEqual(ec.dependencies()[2]['external_module_metadata'], metadata) + + def test_update(self): + """Test use of update() method for EasyConfig instances.""" + toy_ebfile = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'toy-0.0.eb') + ec = EasyConfig(toy_ebfile) + + # for string values: append + ec.update('unpack_options', '--strip-components=1') + self.assertEqual(ec['unpack_options'].strip(), '--strip-components=1') + + ec.update('description', "- just a test") + self.assertEqual(ec['description'].strip(), "Toy C program. - just a test") + + # spaces in between multiple updates for stirng values + ec.update('configopts', 'CC="$CC"') + ec.update('configopts', 'CXX="$CXX"') + self.assertTrue(ec['configopts'].strip().endswith('CC="$CC" CXX="$CXX"')) + + # for list values: extend + ec.update('patches', ['foo.patch', 'bar.patch']) + self.assertEqual(ec['patches'], ['toy-0.0_typo.patch', ('toy-extra.txt', 'toy-0.0'), 'foo.patch', 'bar.patch']) + + def test_hide_hidden_deps(self): + """Test use of --hide-deps on hiddendependencies.""" + test_dir = os.path.dirname(os.path.abspath(__file__)) + ec_file = os.path.join(test_dir, 'easyconfigs', 'gzip-1.4-GCC-4.6.3.eb') + ec = EasyConfig(ec_file) + self.assertEqual(ec['hiddendependencies'][0]['full_mod_name'], 'toy/.0.0-deps') + self.assertEqual(ec['dependencies'], []) + + build_options = { + 'hide_deps': ['toy'], + 'valid_module_classes': module_classes(), + } + init_config(build_options=build_options) + ec = EasyConfig(ec_file) + self.assertEqual(ec['hiddendependencies'][0]['full_mod_name'], 'toy/.0.0-deps') + self.assertEqual(ec['dependencies'], []) + + def test_quote_str(self): + """Test quote_str function.""" + teststrings = { + 'foo' : '"foo"', + 'foo\'bar' : '"foo\'bar"', + 'foo\'bar"baz' : '"""foo\'bar"baz"""', + "foo'bar\"baz" : '"""foo\'bar"baz"""', + "foo\nbar" : '"foo\nbar"', + 'foo bar' : '"foo bar"' + } + + for t in teststrings: + self.assertEqual(quote_str(t), teststrings[t]) + + # test escape_newline + self.assertEqual(quote_str("foo\nbar", escape_newline=False), '"foo\nbar"') + self.assertEqual(quote_str("foo\nbar", escape_newline=True), '"""foo\nbar"""') + + # test prefer_single_quotes + self.assertEqual(quote_str("foo", prefer_single_quotes=True), "'foo'") + self.assertEqual(quote_str('foo bar', prefer_single_quotes=True), '"foo bar"') + self.assertEqual(quote_str("foo'bar", prefer_single_quotes=True), '"foo\'bar"') + + # non-string values + n = 42 + self.assertEqual(quote_str(n), 42) + self.assertEqual(quote_str(["foo", "bar"]), ["foo", "bar"]) + self.assertEqual(quote_str(('foo', 'bar')), ('foo', 'bar')) + + def test_dump(self): + """Test EasyConfig's dump() method.""" + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + ecfiles = [ + 'toy-0.0.eb', + 'goolf-1.4.10.eb', + 'ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb', + 'gzip-1.4-GCC-4.6.3.eb', + ] + for ecfile in ecfiles: + test_ec = os.path.join(self.test_prefix, 'test.eb') + + ec = EasyConfig(os.path.join(test_ecs_dir, ecfile)) + ec.dump(test_ec) + ectxt = read_file(test_ec) + + patterns = [r"^name = ['\"]", r"^version = ['0-9\.]", r'^description = ["\']'] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(ectxt), "Pattern '%s' found in: %s" % (regex.pattern, ectxt)) + + # parse result again + dumped_ec = EasyConfig(test_ec) + + # check that selected parameters still have the same value + params = [ + 'name', + 'toolchain', + 'dependencies', # checking this is important w.r.t. filtered hidden dependencies being restored in dump + ] + for param in params: + self.assertEqual(ec[param], dumped_ec[param]) + + def test_dump_autopep8(self): + """Test dump() with autopep8 usage enabled (only if autopep8 is available).""" + try: + import autopep8 + os.environ['EASYBUILD_DUMP_AUTOPEP8'] = '1' + init_config() + self.test_dump() + del os.environ['EASYBUILD_DUMP_AUTOPEP8'] + except ImportError: + print "Skipping test_dump_autopep8, since autopep8 is not available" + + def test_dump_extra(self): + """Test EasyConfig's dump() method for files containing extra values""" + rawtxt = '\n'.join([ + "easyblock = 'EB_foo'", + '', + "name = 'foo'", + "version = '0.0.1'", + "versionsuffix = '_bar'", + '', + "homepage = 'http://foo.com/'", + 'description = "foo description"', + '', + "toolchain = {'version': 'dummy', 'name': 'dummy'}", + '', + "dependencies = [", + " ('GCC', '4.6.4', '-test'),", + " ('MPICH', '1.8', '', ('GCC', '4.6.4')),", + " ('bar', '1.0'),", + " ('foobar/1.2.3', EXTERNAL_MODULE),", + "]", + '', + "foo_extra1 = 'foobar'", + ]) + + handle, testec = tempfile.mkstemp(prefix=self.test_prefix, suffix='.eb') + os.close(handle) + + ec = EasyConfig(None, rawtxt=rawtxt) + ec.dump(testec) + ectxt = read_file(testec) + self.assertEqual(rawtxt, ectxt) + + dumped_ec = EasyConfig(testec) + + def test_dump_template(self): + """ Test EasyConfig's dump() method for files containing templates""" + rawtxt = '\n'.join([ + "easyblock = 'EB_foo'", + '', + "name = 'Foo'", + "version = '0.0.1'", + "versionsuffix = '-test'", + '', + "homepage = 'http://foo.com/'", + 'description = "foo description"', + '', + "toolchain = {", + " 'version': 'dummy',", + " 'name': 'dummy',", + '}', + '', + "sources = [", + " 'foo-0.0.1.tar.gz',", + ']', + '', + "dependencies = [", + " ('bar', '1.2.3', '-test'),", + ']', + '', + "preconfigopts = '--opt1=%s' % name", + "configopts = '--opt2=0.0.1'", + '', + "sanity_check_paths = {", + " 'files': ['files/foo/foobar', 'files/x-test'],", + " 'dirs':[],", + '}', + '', + "foo_extra1 = 'foobar'" + ]) + + handle, testec = tempfile.mkstemp(prefix=self.test_prefix, suffix='.eb') + os.close(handle) + + ec = EasyConfig(None, rawtxt=rawtxt) + ec.dump(testec) + ectxt = read_file(testec) + + self.assertTrue(ec.enable_templating) # templating should still be enabled after calling dump() + + patterns = [ + r"easyblock = 'EB_foo'", + r"name = 'Foo'", + r"version = '0.0.1'", + r"versionsuffix = '-test'", + r"homepage = 'http://foo.com/'", + r'description = "foo description"', # no templating for description + r"sources = \[SOURCELOWER_TAR_GZ\]", + r"dependencies = \[\n \('bar', '1.2.3', '%\(versionsuffix\)s'\),\n\]", + r"preconfigopts = '--opt1=%\(name\)s'", + r"configopts = '--opt2=%\(version\)s'", + r"sanity_check_paths = {\n 'files': \['files/%\(namelower\)s/foobar', 'files/x-test'\]", + ] + + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(ectxt), "Pattern '%s' found in: %s" % (regex.pattern, ectxt)) + + # reparsing the dumped easyconfig file should work + ecbis = EasyConfig(testec) + + def test_dump_comments(self): + """ Test dump() method for files containing comments """ + rawtxt = '\n'.join([ + "# #", + "# some header comment", + "# #", + "easyblock = 'EB_foo'", + '', + "name = 'Foo' # name comment", + "version = '0.0.1'", + "versionsuffix = '-test'", + '', + "# comment on the homepage", + "homepage = 'http://foo.com/'", + 'description = "foo description with a # in it" # test', + '', + "# toolchain comment", + '', + "toolchain = {", + " 'version': 'dummy',", + " 'name': 'dummy'", + '}', + '', + "sanity_check_paths = {", + " 'files': ['files/foobar'], # comment on files", + " 'dirs':[]", + '}', + '', + "foo_extra1 = 'foobar'", + "# trailing comment", + ]) + + handle, testec = tempfile.mkstemp(prefix=self.test_prefix, suffix='.eb') + os.close(handle) + + ec = EasyConfig(None, rawtxt=rawtxt) + ec.dump(testec) + ectxt = read_file(testec) + + patterns = [ + r"# #\n# some header comment\n# #", + r"name = 'Foo' # name comment", + r"# comment on the homepage\nhomepage = 'http://foo.com/'", + r'description = "foo description with a # in it" # test', + r"# toolchain comment\ntoolchain = {", + r" 'files': \['files/foobar'\], # comment on files", + r" 'dirs': \[\],", + ] + + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(ectxt), "Pattern '%s' found in: %s" % (regex.pattern, ectxt)) + + self.assertTrue(ectxt.endswith("# trailing comment")) + + # reparsing the dumped easyconfig file should work + ecbis = EasyConfig(testec) + + def test_to_template_str(self): + """ Test for to_template_str method """ + + # reverse dict of known template constants; template values (which are keys here) must be 'string-in-string + templ_const = { + "template":'TEMPLATE_VALUE', + "%(name)s-%(version)s": 'NAME_VERSION', + } + + templ_val = { + 'foo':'name', + '0.0.1':'version', + '-test':'special_char', + } + + self.assertEqual(to_template_str("template", templ_const, templ_val), 'TEMPLATE_VALUE') + self.assertEqual(to_template_str("foo/bar/0.0.1/", templ_const, templ_val), "%(name)s/bar/%(version)s/") + self.assertEqual(to_template_str("foo-0.0.1", templ_const, templ_val), 'NAME_VERSION') + templ_list = to_template_str("['-test', 'dontreplacenamehere']", templ_const, templ_val) + self.assertEqual(templ_list, "['%(special_char)s', 'dontreplacenamehere']") + templ_dict = to_template_str("{'a': 'foo', 'b': 'notemplate'}", templ_const, templ_val) + self.assertEqual(templ_dict, "{'a': '%(name)s', 'b': 'notemplate'}") + self.assertEqual(to_template_str("('foo', '0.0.1')", templ_const, templ_val), "('%(name)s', '%(version)s')") + + def test_dep_graph(self): + """Test for dep_graph.""" + try: + import pygraph + + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + build_options = { + 'external_modules_metadata': ConfigObj(), + 'valid_module_classes': module_classes(), + 'robot_path': [test_easyconfigs], + 'silent': True, + } + init_config(build_options=build_options) + + ec_file = os.path.join(test_easyconfigs, 'toy-0.0-deps.eb') + ec_files = [(ec_file, False)] + ecs, _ = parse_easyconfigs(ec_files) + + dot_file = os.path.join(self.test_prefix, 'test.dot') + ordered_ecs = resolve_dependencies(ecs, retain_all_deps=True) + dep_graph(dot_file, ordered_ecs) + + # hard check for expect .dot file contents + # 3 nodes should be there: 'GCC/4.7.2 (EXT)', 'toy', and 'ictce/4.1.13' + # and 2 edges: 'toy -> ictce' and 'toy -> "GCC/4.7.2 (EXT)"' + dottxt = read_file(dot_file) + self.assertEqual(dottxt, EXPECTED_DOTTXT_TOY_DEPS) + + except ImportError: + print "Skipping test_dep_graph, since pygraph is not available" + + def test_ActiveMNS_det_full_module_name(self): + """Test det_full_module_name method of ActiveMNS.""" + build_options = { + 'valid_module_classes': module_classes(), + 'external_modules_metadata': ConfigObj(), + } + + init_config(build_options=build_options) + ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'toy-0.0-deps.eb') + ec = EasyConfig(ec_file) + + self.assertEqual(ActiveMNS().det_full_module_name(ec), 'toy/0.0-deps') + self.assertEqual(ActiveMNS().det_full_module_name(ec['dependencies'][0]), 'ictce/4.1.13') + self.assertEqual(ActiveMNS().det_full_module_name(ec['dependencies'][1]), 'GCC/4.7.2') + + ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'gzip-1.4-GCC-4.6.3.eb') + ec = EasyConfig(ec_file) + hiddendep = ec['hiddendependencies'][0] + self.assertEqual(ActiveMNS().det_full_module_name(hiddendep), 'toy/.0.0-deps') + self.assertEqual(ActiveMNS().det_full_module_name(hiddendep, force_visible=True), 'toy/0.0-deps') + + def test_find_related_easyconfigs(self): + """Test find_related_easyconfigs function.""" + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + ec_file = os.path.join(test_easyconfigs, 'GCC-4.6.3.eb') + ec = EasyConfig(ec_file) + + # exact match: GCC-4.6.3.eb + res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] + self.assertEqual(res, ['GCC-4.6.3.eb']) + + # tweak version to 4.6.1, GCC/4.6.x easyconfigs are found as closest match + ec['version'] = '4.6.1' + res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] + self.assertEqual(res, ['GCC-4.6.3.eb', 'GCC-4.6.4.eb']) + + # tweak version to 4.5.0, GCC/4.x easyconfigs are found as closest match + ec['version'] = '4.5.0' + res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] + expected = ['GCC-4.6.3.eb', 'GCC-4.6.4.eb', 'GCC-4.7.2.eb', 'GCC-4.8.2.eb', 'GCC-4.8.3.eb', 'GCC-4.9.2.eb'] + self.assertEqual(res, expected) + + ec_file = os.path.join(test_easyconfigs, 'toy-0.0-deps.eb') + ec = EasyConfig(ec_file) + + # exact match + res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] + self.assertEqual(res, ['toy-0.0-deps.eb']) + + # tweak toolchain name/version and versionsuffix => closest match with same toolchain name is found + ec['toolchain'] = {'name': 'gompi', 'version': '1.5.16'} + ec['versionsuffix'] = '-foobar' + res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] + self.assertEqual(res, ['toy-0.0-gompi-1.3.12-test.eb']) + + # restore original versionsuffix => matching versionsuffix wins over matching toolchain (name) + ec['versionsuffix'] = '-deps' + res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)] + self.assertEqual(res, ['toy-0.0-deps.eb']) + + # no matches for unknown software name + ec['name'] = 'nosuchsoftware' + self.assertEqual(find_related_easyconfigs(test_easyconfigs, ec), []) + + def test_modaltsoftname(self): + """Test specifying an alternative name for the software name, to use when determining module name.""" + ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'toy-0.0-deps.eb') + ectxt = read_file(ec_file) + modified_ec_file = os.path.join(self.test_prefix, os.path.basename(ec_file)) + write_file(modified_ec_file, ectxt + "\nmodaltsoftname = 'notreallyatoy'") + ec = EasyConfig(modified_ec_file) + self.assertEqual(ec.full_mod_name, 'notreallyatoy/0.0-deps') + self.assertEqual(ec.short_mod_name, 'notreallyatoy/0.0-deps') + self.assertEqual(ec['name'], 'toy') + + def test_software_license(self): + """Tests related to software_license easyconfig parameter.""" + # default: None + ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'toy-0.0.eb') + ec = EasyConfig(ec_file) + ec.validate_license() + self.assertEqual(ec['software_license'], None) + self.assertEqual(ec.software_license, None) + + # specified software license gets handled correctly + ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'gzip-1.4.eb') + ec = EasyConfig(ec_file) + ec.validate_license() + # constant GPLv3 is resolved as string + self.assertEqual(ec['software_license'], 'LicenseGPLv3') + # software_license is defined as License subclass + self.assertTrue(isinstance(ec.software_license, LicenseGPLv3)) + self.assertTrue(issubclass(ec.software_license.__class__, License)) + + ec['software_license'] = 'LicenseThatDoesNotExist' + err_pat = r"Invalid license LicenseThatDoesNotExist \(known licenses:" + self.assertErrorRegex(EasyBuildError, err_pat, ec.validate_license) + + def test_param_value_type_checking(self): + """Test value tupe checking of easyconfig parameters.""" + ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'gzip-1.4-broken.eb') + # name/version parameters have values of wrong type in this broken easyconfig + error_msg_pattern = "Type checking of easyconfig parameter values failed: .*'version'.*" + self.assertErrorRegex(EasyBuildError, error_msg_pattern, EasyConfig, ec_file, auto_convert_value_types=False) + + # test default behaviour: auto-converting of mismatching value types + ec = EasyConfig(ec_file) + self.assertEqual(ec['version'], '1.4') + + def suite(): """ returns all the testcases in this module """ return TestLoader().loadTestsFromTestCase(EasyConfigTest) diff --git a/test/framework/easyconfigformat.py b/test/framework/easyconfigformat.py index 393cd3c4f4..e52175461e 100644 --- a/test/framework/easyconfigformat.py +++ b/test/framework/easyconfigformat.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyconfigparser.py b/test/framework/easyconfigparser.py index fa4194b9d1..c73cb01c04 100644 --- a/test/framework/easyconfigparser.py +++ b/test/framework/easyconfigparser.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,8 +34,11 @@ import easybuild.tools.build_log from easybuild.framework.easyconfig.format.format import Dependency +from easybuild.framework.easyconfig.format.pyheaderconfigobj import build_easyconfig_constants_dict from easybuild.framework.easyconfig.format.version import EasyVersion from easybuild.framework.easyconfig.parser import EasyConfigParser +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import read_file TESTDIRBASE = os.path.join(os.path.dirname(__file__), 'easyconfigs') @@ -146,6 +149,56 @@ def test_v20_deps(self): # restore easybuild.tools.build_log.EXPERIMENTAL = orig_experimental + def test_raw(self): + """Test passing of raw contents to EasyConfigParser.""" + ec_file1 = os.path.join(TESTDIRBASE, 'v1.0', 'GCC-4.6.3.eb') + ec_txt1 = read_file(ec_file1) + ec_file2 = os.path.join(TESTDIRBASE, 'v1.0', 'gzip-1.5-goolf-1.4.10.eb') + ec_txt2 = read_file(ec_file2) + + ecparser = EasyConfigParser(ec_file1) + self.assertEqual(ecparser.rawcontent, ec_txt1) + + ecparser = EasyConfigParser(rawcontent=ec_txt2) + self.assertEqual(ecparser.rawcontent, ec_txt2) + + # rawcontent supersedes passed filepath + ecparser = EasyConfigParser(ec_file1, rawcontent=ec_txt2) + self.assertEqual(ecparser.rawcontent, ec_txt2) + ec = ecparser.get_config_dict() + self.assertEqual(ec['name'], 'gzip') + self.assertEqual(ec['toolchain']['name'], 'goolf') + + self.assertErrorRegex(EasyBuildError, "Neither filename nor rawcontent provided", EasyConfigParser) + + def test_easyconfig_constants(self): + """Test available easyconfig constants.""" + constants = build_easyconfig_constants_dict() + # make sure both keys and values are only strings + for constant_name in constants: + self.assertTrue(isinstance(constant_name, basestring), "Constant name %s is a string" % constant_name) + val = constants[constant_name] + self.assertTrue(isinstance(val, basestring), "Constant value %s is a string" % val) + + # check a couple of randomly picked constant values + self.assertEqual(constants['SOURCE_TAR_GZ'], '%(name)s-%(version)s.tar.gz') + self.assertEqual(constants['PYPI_SOURCE'], 'https://pypi.python.org/packages/source/%(nameletter)s/%(name)s') + self.assertEqual(constants['GPLv2'], 'LicenseGPLv2') + self.assertEqual(constants['EXTERNAL_MODULE'], 'EXTERNAL_MODULE') + + def test_check_value_types(self): + """Test checking of easyconfig parameter value types.""" + test_ec = os.path.join(TESTDIRBASE, 'gzip-1.4-broken.eb') + error_msg_pattern = "Type checking of easyconfig parameter values failed: .*'version'.*" + ecp = EasyConfigParser(test_ec, auto_convert_value_types=False) + self.assertErrorRegex(EasyBuildError, error_msg_pattern, ecp.get_config_dict) + + # test default behaviour: auto-converting of mismatched value types + ecp = EasyConfigParser(test_ec) + ecdict = ecp.get_config_dict() + self.assertEqual(ecdict['version'], '1.4') + + def suite(): """ returns all the testcases in this module """ return TestLoader().loadTestsFromTestCase(EasyConfigParserTest) diff --git a/test/framework/easyconfigs/CUDA-5.5.22-GCC-4.8.2.eb b/test/framework/easyconfigs/CUDA-5.5.22-GCC-4.8.2.eb new file mode 100644 index 0000000000..00bf4df2f9 --- /dev/null +++ b/test/framework/easyconfigs/CUDA-5.5.22-GCC-4.8.2.eb @@ -0,0 +1,31 @@ +## +# This file is an EasyBuild reciPY as per https://github.com/hpcugent/easybuild +# +# Copyright:: Copyright 2012-2014 Cyprus Institute / CaSToRC, Uni.Lu/LCSB, NTUA, Ghent University +# Authors:: George Tsouloupas , Fotis Georgatos , Kenneth Hoste +# License:: MIT/GPL +# $Id$ +# +# This work implements a part of the HPCBIOS project and is a component of the policy: +# http://hpcbios.readthedocs.org/en/latest/HPCBIOS_2012-99.html +## +# should be EB_CUDA, but OK for testing purposes +easyblock = 'EB_toy' + +name = 'CUDA' +version = '5.5.22' + +homepage = 'https://developer.nvidia.com/cuda-toolkit' +description = """CUDA (formerly Compute Unified Device Architecture) is a parallel + computing platform and programming model created by NVIDIA and implemented by the + graphics processing units (GPUs) that they produce. CUDA gives developers access + to the virtual instruction set and memory of the parallel computational elements in CUDA GPUs.""" + +toolchain = {'name': 'GCC', 'version': '4.8.2'} + +# eg. http://developer.download.nvidia.com/compute/cuda/5_5/rel/installers/cuda_5.5.22_linux_64.run +source_urls = ['http://developer.download.nvidia.com/compute/cuda/5_5/rel/installers/'] + +sources = ['%(namelower)s_%(version)s_linux_64.run'] + +moduleclass = 'system' diff --git a/test/framework/easyconfigs/FFTW-3.3.3-gompi-1.4.10.eb b/test/framework/easyconfigs/FFTW-3.3.3-gompi-1.4.10.eb index 06b0c2e2e1..bead8318f4 100644 --- a/test/framework/easyconfigs/FFTW-3.3.3-gompi-1.4.10.eb +++ b/test/framework/easyconfigs/FFTW-3.3.3-gompi-1.4.10.eb @@ -1,3 +1,5 @@ +easyblock = 'ConfigureMake' + name = 'FFTW' version = '3.3.3' diff --git a/test/framework/easyconfigs/GCC-4.6.3.eb b/test/framework/easyconfigs/GCC-4.6.3.eb index 3b4c4c53c9..8f9e3c6a1f 100644 --- a/test/framework/easyconfigs/GCC-4.6.3.eb +++ b/test/framework/easyconfigs/GCC-4.6.3.eb @@ -1,3 +1,6 @@ +# should be EB_GCC, but OK for testing purposes +easyblock = 'EB_toy' + name="GCC" version='4.6.3' diff --git a/test/framework/easyconfigs/GCC-4.6.4.eb b/test/framework/easyconfigs/GCC-4.6.4.eb index bf4adc61a6..baf448818b 100644 --- a/test/framework/easyconfigs/GCC-4.6.4.eb +++ b/test/framework/easyconfigs/GCC-4.6.4.eb @@ -1,3 +1,6 @@ +# should be EB_GCC, but OK for testing purposes +easyblock = 'EB_toy' + name = "GCC" version = '4.6.4' diff --git a/test/framework/easyconfigs/GCC-4.7.2.eb b/test/framework/easyconfigs/GCC-4.7.2.eb index 7b4dfcf410..d4b386baae 100644 --- a/test/framework/easyconfigs/GCC-4.7.2.eb +++ b/test/framework/easyconfigs/GCC-4.7.2.eb @@ -1,3 +1,6 @@ +# should be EB_GCC, but OK for testing purposes +easyblock = 'EB_toy' + name = "GCC" version = '4.7.2' diff --git a/test/framework/easyconfigs/GCC-4.8.2.eb b/test/framework/easyconfigs/GCC-4.8.2.eb new file mode 100644 index 0000000000..a7723b5eb9 --- /dev/null +++ b/test/framework/easyconfigs/GCC-4.8.2.eb @@ -0,0 +1,31 @@ +# should be EB_GCC, but OK for testing purposes +easyblock = 'EB_toy' + +name = "GCC" +version = '4.8.2' + +homepage = 'http://gcc.gnu.org/' +description = """The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...).""" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +source_urls = [ + 'http://ftpmirror.gnu.org/%(namelower)s/%(namelower)s-%(version)s', # GCC auto-resolving HTTP mirror + 'http://ftpmirror.gnu.org/gmp', # idem for GMP + 'http://ftpmirror.gnu.org/mpfr', # idem for MPFR + 'http://www.multiprecision.org/mpc/download', # MPC official +] +sources = [ + SOURCELOWER_TAR_GZ, + 'gmp-5.1.3.tar.bz2', + 'mpfr-3.1.2.tar.gz', + 'mpc-1.0.1.tar.gz', +] + +languages = ['c', 'c++', 'fortran', 'lto'] + +# building GCC sometimes fails if make parallelism is too high, so let's limit it +maxparallel = 4 + +moduleclass = 'compiler' diff --git a/test/framework/easyconfigs/GCC-4.8.3.eb b/test/framework/easyconfigs/GCC-4.8.3.eb new file mode 100644 index 0000000000..14e91d37d2 --- /dev/null +++ b/test/framework/easyconfigs/GCC-4.8.3.eb @@ -0,0 +1,31 @@ +# should be EB_GCC, but OK for testing purposes +easyblock = 'EB_toy' + +name = "GCC" +version = '4.8.3' + +homepage = 'http://gcc.gnu.org/' +description = """The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...).""" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +source_urls = [ + 'http://ftpmirror.gnu.org/%(namelower)s/%(namelower)s-%(version)s', # GCC auto-resolving HTTP mirror + 'http://ftpmirror.gnu.org/gmp', # idem for GMP + 'http://ftpmirror.gnu.org/mpfr', # idem for MPFR + 'http://www.multiprecision.org/mpc/download', # MPC official +] +sources = [ + SOURCELOWER_TAR_GZ, + 'gmp-5.1.3.tar.bz2', + 'mpfr-3.1.2.tar.gz', + 'mpc-1.0.1.tar.gz', +] + +languages = ['c', 'c++', 'fortran', 'lto'] + +# building GCC sometimes fails if make parallelism is too high, so let's limit it +maxparallel = 4 + +moduleclass = 'compiler' diff --git a/test/framework/easyconfigs/GCC-4.9.2.eb b/test/framework/easyconfigs/GCC-4.9.2.eb new file mode 100644 index 0000000000..ec651b931d --- /dev/null +++ b/test/framework/easyconfigs/GCC-4.9.2.eb @@ -0,0 +1,36 @@ +# should be EB_GCC, but OK for testing purposes +easyblock = 'EB_toy' + +name = "GCC" +version = '4.9.2' + +homepage = 'http://gcc.gnu.org/' +description = """The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...).""" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +source_urls = [ + 'http://ftpmirror.gnu.org/%(namelower)s/%(namelower)s-%(version)s', # GCC auto-resolving HTTP mirror + 'http://ftpmirror.gnu.org/gmp', # idem for GMP + 'http://ftpmirror.gnu.org/mpfr', # idem for MPFR + 'http://www.multiprecision.org/mpc/download', # MPC official +] + +mpfr_version = '3.1.2' + +sources = [ + SOURCELOWER_TAR_BZ2, + 'gmp-6.0.0a.tar.bz2', + 'mpfr-%s.tar.gz' % mpfr_version, + 'mpc-1.0.2.tar.gz', +] + +patches = [('mpfr-%s-allpatches-20140630.patch' % mpfr_version, '../mpfr-%s' % mpfr_version)] + +languages = ['c', 'c++', 'fortran', 'lto'] + +# building GCC sometimes fails if make parallelism is too high, so let's limit it +maxparallel = 4 + +moduleclass = 'compiler' diff --git a/test/framework/easyconfigs/OpenBLAS-0.2.6-gompi-1.4.10-LAPACK-3.4.2.eb b/test/framework/easyconfigs/OpenBLAS-0.2.6-gompi-1.4.10-LAPACK-3.4.2.eb index 8f1b31ff70..bd9785f7cc 100644 --- a/test/framework/easyconfigs/OpenBLAS-0.2.6-gompi-1.4.10-LAPACK-3.4.2.eb +++ b/test/framework/easyconfigs/OpenBLAS-0.2.6-gompi-1.4.10-LAPACK-3.4.2.eb @@ -1,3 +1,5 @@ +easyblock = 'ConfigureMake' + name = 'OpenBLAS' version = '0.2.6' @@ -44,7 +46,7 @@ installopts = threading + " PREFIX=%(installdir)s" sanity_check_paths = { 'files': ['include/cblas.h', 'include/f77blas.h', 'include/lapacke_config.h', 'include/lapacke.h', 'include/lapacke_mangling.h', 'include/lapacke_utils.h', 'include/openblas_config.h', - 'lib/libopenblas.a', 'lib/libopenblas.%s' % shared_lib_ext], + 'lib/libopenblas.a', 'lib/libopenblas.%s' % SHLIB_EXT], 'dirs': [], } diff --git a/test/framework/easyconfigs/OpenMPI-1.6.4-GCC-4.6.4.eb b/test/framework/easyconfigs/OpenMPI-1.6.4-GCC-4.6.4.eb index 053791e834..bd0832e690 100644 --- a/test/framework/easyconfigs/OpenMPI-1.6.4-GCC-4.6.4.eb +++ b/test/framework/easyconfigs/OpenMPI-1.6.4-GCC-4.6.4.eb @@ -1,3 +1,5 @@ +easyblock = 'ConfigureMake' + name = 'OpenMPI' version = '1.6.4' @@ -25,7 +27,7 @@ else: sanity_check_paths = { 'files': ["bin/%s" % binfile for binfile in ["ompi_info", "opal_wrapper", "orterun"]] + - ["lib/lib%s.%s" % (libfile, shared_lib_ext) for libfile in ["mpi_cxx", "mpi_f77", "mpi_f90", + ["lib/lib%s.%s" % (libfile, SHLIB_EXT) for libfile in ["mpi_cxx", "mpi_f77", "mpi_f90", "mpi", "ompitrace", "open-pal", "open-rte", "vt", "vt-hyb", "vt-mpi", "vt-mpi-unify"]] + diff --git a/test/framework/easyconfigs/OpenMPI-1.6.4-GCC-4.7.2.eb b/test/framework/easyconfigs/OpenMPI-1.6.4-GCC-4.7.2.eb index fa3425f1ab..1505eba3ad 100644 --- a/test/framework/easyconfigs/OpenMPI-1.6.4-GCC-4.7.2.eb +++ b/test/framework/easyconfigs/OpenMPI-1.6.4-GCC-4.7.2.eb @@ -1,3 +1,5 @@ +easyblock = 'ConfigureMake' + name = 'OpenMPI' version = '1.6.4' @@ -25,7 +27,7 @@ else: sanity_check_paths = { 'files': ["bin/%s" % binfile for binfile in ["ompi_info", "opal_wrapper", "orterun"]] + - ["lib/lib%s.%s" % (libfile, shared_lib_ext) for libfile in ["mpi_cxx", "mpi_f77", "mpi_f90", + ["lib/lib%s.%s" % (libfile, SHLIB_EXT) for libfile in ["mpi_cxx", "mpi_f77", "mpi_f90", "mpi", "ompitrace", "open-pal", "open-rte", "vt", "vt-hyb", "vt-mpi", "vt-mpi-unify"]] + diff --git a/test/framework/easyconfigs/ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb b/test/framework/easyconfigs/ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb index 8f7ad295d1..14f049732b 100644 --- a/test/framework/easyconfigs/ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb +++ b/test/framework/easyconfigs/ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb @@ -1,3 +1,6 @@ +# should be EB_ScaLAPACK, but OK for testing purposes +easyblock = 'EB_toy' + name = 'ScaLAPACK' version = '2.0.2' diff --git a/test/framework/easyconfigs/gzip-1.4-GCC-4.6.3.eb b/test/framework/easyconfigs/gzip-1.4-GCC-4.6.3.eb index 9f1c615c51..0775b46e33 100644 --- a/test/framework/easyconfigs/gzip-1.4-GCC-4.6.3.eb +++ b/test/framework/easyconfigs/gzip-1.4-GCC-4.6.3.eb @@ -9,6 +9,7 @@ # This work implements a part of the HPCBIOS project and is a component of the policy: # http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html ## +easyblock = 'ConfigureMake' name = 'gzip' version = '1.4' @@ -25,6 +26,9 @@ sources = ['%s-%s.tar.gz'%(name,version)] # download location for source files source_urls = [GNU_SOURCE] +hiddendependencies = [('toy', '0.0', '-deps', True)] +dependencies = hiddendependencies # hidden deps must be included in list of deps + # make sure the gzip and gunzip binaries are available after installation sanity_check_paths = { 'files': ["bin/gunzip", "bin/gzip"], @@ -34,3 +38,6 @@ sanity_check_paths = { # run 'gzip -h' and 'gzip --version' after installation sanity_check_commands = [True, ('gzip', '--version')] +software_license = GPLv3 + +moduleclass = 'tools' diff --git a/test/framework/easyconfigs/gzip-1.4-broken.eb b/test/framework/easyconfigs/gzip-1.4-broken.eb new file mode 100644 index 0000000000..716c1804b6 --- /dev/null +++ b/test/framework/easyconfigs/gzip-1.4-broken.eb @@ -0,0 +1,41 @@ +## +# This file is an EasyBuild reciPY as per https://github.com/hpcugent/easybuild +# +# Copyright:: Copyright (c) 2012-2013 Cyprus Institute / CaSToRC +# Authors:: Thekla Loizou +# License:: MIT/GPL +# $Id$ +# +# This work implements a part of the HPCBIOS project and is a component of the policy: +# http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html +## +easyblock = 'ConfigureMake' + +name = 'gzip' +# wrong type of value (on purpose, for testing), should be a string +version = 1.4 # '1.4' + +homepage = "http://www.gzip.org/" +description = "gzip (GNU zip) is a popular data compression program as a replacement for compress" + +# test toolchain specification +toolchain = {'name':'dummy','version':'dummy'} + +# source tarball filename +sources = [SOURCE_TAR_GZ] + +# download location for source files +source_urls = ['http://ftpmirror.gnu.org/gzip'] + +# make sure the gzip and gunzip binaries are available after installation +sanity_check_paths = { + 'files': ["bin/gunzip", "bin/gzip"], + 'dirs': [], +} + +# run 'gzip -h' and 'gzip --version' after installation +sanity_check_commands = [True, ('gzip', '--version')] + +software_license = GPLv3 + +moduleclass = 'tools' diff --git a/test/framework/easyconfigs/gzip-1.4.eb b/test/framework/easyconfigs/gzip-1.4.eb index ab769c6b69..f00f8d7197 100644 --- a/test/framework/easyconfigs/gzip-1.4.eb +++ b/test/framework/easyconfigs/gzip-1.4.eb @@ -9,6 +9,7 @@ # This work implements a part of the HPCBIOS project and is a component of the policy: # http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html ## +easyblock = 'ConfigureMake' name = 'gzip' version = '1.4' @@ -34,3 +35,6 @@ sanity_check_paths = { # run 'gzip -h' and 'gzip --version' after installation sanity_check_commands = [True, ('gzip', '--version')] +software_license = GPLv3 + +moduleclass = 'tools' diff --git a/test/framework/easyconfigs/gzip-1.5-goolf-1.4.10.eb b/test/framework/easyconfigs/gzip-1.5-goolf-1.4.10.eb index 08ce4ddc61..90465d7ce4 100644 --- a/test/framework/easyconfigs/gzip-1.5-goolf-1.4.10.eb +++ b/test/framework/easyconfigs/gzip-1.5-goolf-1.4.10.eb @@ -9,6 +9,7 @@ # This work implements a part of the HPCBIOS project and is a component of the policy: # http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html ## +easyblock = 'ConfigureMake' name = 'gzip' version = '1.5' @@ -31,3 +32,6 @@ sanity_check_paths = { # run 'gzip -h' and 'gzip --version' after installation sanity_check_commands = [True, ('gzip', '--version')] +software_license = GPLv3 + +moduleclass = 'tools' diff --git a/test/framework/easyconfigs/gzip-1.5-ictce-4.1.13.eb b/test/framework/easyconfigs/gzip-1.5-ictce-4.1.13.eb index 2552d296e9..d00ed713a4 100644 --- a/test/framework/easyconfigs/gzip-1.5-ictce-4.1.13.eb +++ b/test/framework/easyconfigs/gzip-1.5-ictce-4.1.13.eb @@ -9,6 +9,7 @@ # This work implements a part of the HPCBIOS project and is a component of the policy: # http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html ## +easyblock = 'ConfigureMake' name = 'gzip' version = '1.5' @@ -31,3 +32,6 @@ sanity_check_paths = { # run 'gzip -h' and 'gzip --version' after installation sanity_check_commands = [True, ('gzip', '--version')] +software_license = GPLv3 + +moduleclass = 'tools' diff --git a/test/framework/easyconfigs/hwloc-1.6.2-GCC-4.6.4.eb b/test/framework/easyconfigs/hwloc-1.6.2-GCC-4.6.4.eb index 116ee4cb8c..3c07bbe614 100644 --- a/test/framework/easyconfigs/hwloc-1.6.2-GCC-4.6.4.eb +++ b/test/framework/easyconfigs/hwloc-1.6.2-GCC-4.6.4.eb @@ -1,3 +1,5 @@ +easyblock = 'ConfigureMake' + name = 'hwloc' version = '1.6.2' diff --git a/test/framework/easyconfigs/hwloc-1.6.2-GCC-4.7.2.eb b/test/framework/easyconfigs/hwloc-1.6.2-GCC-4.7.2.eb index 00a3ce7444..19d34c3d1a 100644 --- a/test/framework/easyconfigs/hwloc-1.6.2-GCC-4.7.2.eb +++ b/test/framework/easyconfigs/hwloc-1.6.2-GCC-4.7.2.eb @@ -1,3 +1,5 @@ +easyblock = 'ConfigureMake' + name = 'hwloc' version = '1.6.2' diff --git a/test/framework/easyconfigs/icc-2013.5.192-GCC-4.8.3.eb b/test/framework/easyconfigs/icc-2013.5.192-GCC-4.8.3.eb new file mode 100644 index 0000000000..a084c374d0 --- /dev/null +++ b/test/framework/easyconfigs/icc-2013.5.192-GCC-4.8.3.eb @@ -0,0 +1,26 @@ +# should be EB_icc, but OK for testing purposes +easyblock = 'EB_toy' + +name = 'icc' +version = '2013.5.192' + +homepage = 'http://software.intel.com/en-us/intel-compilers/' +description = "C and C++ compiler from Intel" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +sources = ['l_ccompxe_%(version)s.tgz'] + +gcc = 'GCC' +gccver = '4.8.3' +versionsuffix = '-%s-%s' % (gcc, gccver) + +dependencies = [(gcc, gccver)] + +dontcreateinstalldir = 'True' + +# license file +import os +license_file = os.path.join(os.getenv('HOME'), "licenses", "intel", "license.lic") + +moduleclass = 'compiler' diff --git a/test/framework/easyconfigs/iccifort-2013.5.192-GCC-4.8.3.eb b/test/framework/easyconfigs/iccifort-2013.5.192-GCC-4.8.3.eb new file mode 100644 index 0000000000..9a152f81fe --- /dev/null +++ b/test/framework/easyconfigs/iccifort-2013.5.192-GCC-4.8.3.eb @@ -0,0 +1,17 @@ +easyblock = "Toolchain" + +name = 'iccifort' +version = '2013.5.192' +versionsuffix = '-GCC-4.8.3' + +homepage = 'http://software.intel.com/en-us/intel-cluster-toolkit-compiler/' +description = """Intel C, C++ and Fortran compilers""" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +dependencies = [ + ('icc', version, versionsuffix), + ('ifort', version, versionsuffix), +] + +moduleclass = 'toolchain' diff --git a/test/framework/easyconfigs/ifort-2013.3.163.eb b/test/framework/easyconfigs/ifort-2013.3.163.eb new file mode 100644 index 0000000000..09c286af0b --- /dev/null +++ b/test/framework/easyconfigs/ifort-2013.3.163.eb @@ -0,0 +1,20 @@ +# should be EB_ifort, but OK for testing purposes +easyblock = 'EB_toy' + +name = 'ifort' +version = '2013.3.163' + +homepage = 'http://software.intel.com/en-us/intel-compilers/' +description = "Fortran compiler from Intel" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +sources = ['l_fcompxe_%s.tgz' % version] + +dontcreateinstalldir = 'True' + +# license file +import os +license_file = os.path.join(os.getenv('HOME'), "licenses", "intel", "license.lic") + +moduleclass = 'compiler' diff --git a/test/framework/easyconfigs/ifort-2013.5.192-GCC-4.8.3.eb b/test/framework/easyconfigs/ifort-2013.5.192-GCC-4.8.3.eb new file mode 100644 index 0000000000..8a74093865 --- /dev/null +++ b/test/framework/easyconfigs/ifort-2013.5.192-GCC-4.8.3.eb @@ -0,0 +1,26 @@ +# should be EB_ifort, but OK for testing purposes +easyblock = 'EB_toy' + +name = 'ifort' +version = '2013.5.192' + +homepage = 'http://software.intel.com/en-us/intel-compilers/' +description = "Fortran compiler from Intel" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +sources = ['l_fcompxe_%(version)s.tgz'] + +gcc = 'GCC' +gccver = '4.8.3' +versionsuffix = '-%s-%s' % (gcc, gccver) + +dependencies = [(gcc, gccver)] + +dontcreateinstalldir = 'True' + +# license file +import os +license_file = os.path.join(os.getenv('HOME'), "licenses", "intel", "license.lic") + +moduleclass = 'compiler' diff --git a/test/framework/easyconfigs/iimpi-5.5.3-GCC-4.8.3.eb b/test/framework/easyconfigs/iimpi-5.5.3-GCC-4.8.3.eb new file mode 100644 index 0000000000..221f1fb36d --- /dev/null +++ b/test/framework/easyconfigs/iimpi-5.5.3-GCC-4.8.3.eb @@ -0,0 +1,21 @@ +easyblock = "Toolchain" + +name = 'iimpi' +version = '5.5.3' +versionsuffix = '-GCC-4.8.3' + +homepage = 'http://software.intel.com/en-us/intel-cluster-toolkit-compiler/' +description = """Intel C/C++ and Fortran compilers, alongside Intel MPI.""" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +suff = '5.192' +compver = '2013.%s' % suff + +dependencies = [ # version/released + ('icc', compver, versionsuffix), # 28 Apr 2014 + ('ifort', compver, versionsuffix), # 28 Apr 2014 + ('impi', '4.1.3.049', '', ('iccifort', '%s%s' % (compver, versionsuffix))), # 06 Mar 2014 +] + +moduleclass = 'toolchain' diff --git a/test/framework/easyconfigs/imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb b/test/framework/easyconfigs/imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb new file mode 100644 index 0000000000..a23cf2c3c9 --- /dev/null +++ b/test/framework/easyconfigs/imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb @@ -0,0 +1,25 @@ +# should be EB_imkl, but OK for testing purposes +easyblock = 'EB_toy' + +name = 'imkl' +version = '11.1.2.144' + +homepage = 'http://software.intel.com/en-us/intel-mkl/' +description = """Intel Math Kernel Library is a library of highly optimized, + extensively threaded math routines for science, engineering, and financial + applications that require maximum performance. Core math functions include + BLAS, LAPACK, ScaLAPACK, Sparse Solvers, Fast Fourier Transforms, Vector Math, and more.""" + +toolchain = {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'} + +sources = ['l_mkl_%(version)s.tgz'] + +dontcreateinstalldir = 'True' + +interfaces = True + +# license file +import os +license_file = os.path.join(os.getenv('HOME'), "licenses", "intel", "license.lic") + +moduleclass = 'numlib' diff --git a/test/framework/easyconfigs/impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb b/test/framework/easyconfigs/impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb new file mode 100644 index 0000000000..e325ca2e75 --- /dev/null +++ b/test/framework/easyconfigs/impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb @@ -0,0 +1,22 @@ +# should be EB_impi, but OK for testing purposes +easyblock = 'EB_toy' + +name = 'impi' +version = '4.1.3.049' + +homepage = 'http://software.intel.com/en-us/intel-mpi-library/' +description = """The Intel(R) MPI Library for Linux* OS is a multi-fabric message + passing library based on ANL MPICH2 and OSU MVAPICH2. The Intel MPI Library for + Linux OS implements the Message Passing Interface, version 2 (MPI-2) specification.""" + +toolchain = {'name': 'iccifort', 'version': '2013.5.192-GCC-4.8.3'} + +sources = ['l_mpi_p_%(version)s.tgz'] + +dontcreateinstalldir = 'True' + +# license file +import os +license_file = os.path.join(os.getenv('HOME'), "licenses", "intel", "license.lic") + +moduleclass = 'mpi' diff --git a/test/framework/easyconfigs/impi-4.1.3.049.eb b/test/framework/easyconfigs/impi-4.1.3.049.eb new file mode 100644 index 0000000000..4267dce6b6 --- /dev/null +++ b/test/framework/easyconfigs/impi-4.1.3.049.eb @@ -0,0 +1,22 @@ +# should be EB_impi, but OK for testing purposes +easyblock = 'EB_toy' + +name = 'impi' +version = '4.1.3.049' + +homepage = 'http://software.intel.com/en-us/intel-mpi-library/' +description = """The Intel(R) MPI Library for Linux* OS is a multi-fabric message + passing library based on ANL MPICH2 and OSU MVAPICH2. The Intel MPI Library for + Linux OS implements the Message Passing Interface, version 2 (MPI-2) specification.""" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +sources = ['l_mpi_p_%(version)s.tgz'] + +dontcreateinstalldir = 'True' + +# license file +import os +license_file = os.path.join(os.getenv('HOME'), "licenses", "intel", "license.lic") + +moduleclass = 'mpi' diff --git a/test/framework/easyconfigs/toy-0.0-deps.eb b/test/framework/easyconfigs/toy-0.0-deps.eb index 6ab5bce42f..84e6e22bc1 100644 --- a/test/framework/easyconfigs/toy-0.0-deps.eb +++ b/test/framework/easyconfigs/toy-0.0-deps.eb @@ -18,7 +18,10 @@ checksums = [[ ]] patches = ['toy-0.0_typo.patch'] -dependencies = [('gompi', '1.3.12')] +dependencies = [ + ('ictce', '4.1.13', '', True), + ('GCC/4.7.2', EXTERNAL_MODULE), +] sanity_check_paths = { 'files': [('bin/yot', 'bin/toy')], diff --git a/test/framework/easyconfigs/toy-0.0-gompi-1.3.12.eb b/test/framework/easyconfigs/toy-0.0-gompi-1.3.12-test.eb similarity index 94% rename from test/framework/easyconfigs/toy-0.0-gompi-1.3.12.eb rename to test/framework/easyconfigs/toy-0.0-gompi-1.3.12-test.eb index 632c832da3..d4de1ae5fb 100644 --- a/test/framework/easyconfigs/toy-0.0-gompi-1.3.12.eb +++ b/test/framework/easyconfigs/toy-0.0-gompi-1.3.12-test.eb @@ -1,5 +1,6 @@ name = 'toy' version = '0.0' +versionsuffix = '-test' homepage = 'http://hpcugent.github.com/easybuild' description = "Toy C program." @@ -20,6 +21,7 @@ patches = ['toy-0.0_typo.patch'] exts_list = [ ('bar', '0.0'), + ('barbar', '0.0'), ] sanity_check_paths = { diff --git a/test/framework/easyconfigs/toy-0.0.eb b/test/framework/easyconfigs/toy-0.0.eb index 08f6303826..d4d9e4e108 100644 --- a/test/framework/easyconfigs/toy-0.0.eb +++ b/test/framework/easyconfigs/toy-0.0.eb @@ -15,7 +15,10 @@ checksums = [[ ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'), ('size', 273), ]] -patches = ['toy-0.0_typo.patch'] +patches = [ + 'toy-0.0_typo.patch', + ('toy-extra.txt', 'toy-0.0'), +] sanity_check_paths = { 'files': [('bin/yot', 'bin/toy')], @@ -25,3 +28,4 @@ sanity_check_paths = { postinstallcmds = ["echo TOY > %(installdir)s/README"] moduleclass = 'tools' +# trailing comment, leave this here, it may trigger bugs with extract_comments() diff --git a/test/framework/easyconfigs/v1.0/GCC-4.6.3.eb b/test/framework/easyconfigs/v1.0/GCC-4.6.3.eb index 3b4c4c53c9..8f9e3c6a1f 100644 --- a/test/framework/easyconfigs/v1.0/GCC-4.6.3.eb +++ b/test/framework/easyconfigs/v1.0/GCC-4.6.3.eb @@ -1,3 +1,6 @@ +# should be EB_GCC, but OK for testing purposes +easyblock = 'EB_toy' + name="GCC" version='4.6.3' diff --git a/test/framework/easyconfigs/v1.0/gzip-1.4-GCC-4.6.3.eb b/test/framework/easyconfigs/v1.0/gzip-1.4-GCC-4.6.3.eb index 9f1c615c51..4480e210df 100644 --- a/test/framework/easyconfigs/v1.0/gzip-1.4-GCC-4.6.3.eb +++ b/test/framework/easyconfigs/v1.0/gzip-1.4-GCC-4.6.3.eb @@ -9,6 +9,7 @@ # This work implements a part of the HPCBIOS project and is a component of the policy: # http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html ## +easyblock = 'ConfigureMake' name = 'gzip' version = '1.4' @@ -34,3 +35,6 @@ sanity_check_paths = { # run 'gzip -h' and 'gzip --version' after installation sanity_check_commands = [True, ('gzip', '--version')] +software_license = GPLv3 + +moduleclass = 'tools' diff --git a/test/framework/easyconfigs/v1.0/gzip-1.4.eb b/test/framework/easyconfigs/v1.0/gzip-1.4.eb index ab769c6b69..f00f8d7197 100644 --- a/test/framework/easyconfigs/v1.0/gzip-1.4.eb +++ b/test/framework/easyconfigs/v1.0/gzip-1.4.eb @@ -9,6 +9,7 @@ # This work implements a part of the HPCBIOS project and is a component of the policy: # http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html ## +easyblock = 'ConfigureMake' name = 'gzip' version = '1.4' @@ -34,3 +35,6 @@ sanity_check_paths = { # run 'gzip -h' and 'gzip --version' after installation sanity_check_commands = [True, ('gzip', '--version')] +software_license = GPLv3 + +moduleclass = 'tools' diff --git a/test/framework/easyconfigs/v1.0/gzip-1.5-goolf-1.4.10.eb b/test/framework/easyconfigs/v1.0/gzip-1.5-goolf-1.4.10.eb index 08ce4ddc61..90465d7ce4 100644 --- a/test/framework/easyconfigs/v1.0/gzip-1.5-goolf-1.4.10.eb +++ b/test/framework/easyconfigs/v1.0/gzip-1.5-goolf-1.4.10.eb @@ -9,6 +9,7 @@ # This work implements a part of the HPCBIOS project and is a component of the policy: # http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html ## +easyblock = 'ConfigureMake' name = 'gzip' version = '1.5' @@ -31,3 +32,6 @@ sanity_check_paths = { # run 'gzip -h' and 'gzip --version' after installation sanity_check_commands = [True, ('gzip', '--version')] +software_license = GPLv3 + +moduleclass = 'tools' diff --git a/test/framework/easyconfigs/v1.0/gzip-1.5-ictce-4.1.13.eb b/test/framework/easyconfigs/v1.0/gzip-1.5-ictce-4.1.13.eb index 2552d296e9..d00ed713a4 100644 --- a/test/framework/easyconfigs/v1.0/gzip-1.5-ictce-4.1.13.eb +++ b/test/framework/easyconfigs/v1.0/gzip-1.5-ictce-4.1.13.eb @@ -9,6 +9,7 @@ # This work implements a part of the HPCBIOS project and is a component of the policy: # http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html ## +easyblock = 'ConfigureMake' name = 'gzip' version = '1.5' @@ -31,3 +32,6 @@ sanity_check_paths = { # run 'gzip -h' and 'gzip --version' after installation sanity_check_commands = [True, ('gzip', '--version')] +software_license = GPLv3 + +moduleclass = 'tools' diff --git a/test/framework/easyconfigs/v2.0/GCC.eb b/test/framework/easyconfigs/v2.0/GCC.eb index 6a1baf1bef..b753004a6e 100644 --- a/test/framework/easyconfigs/v2.0/GCC.eb +++ b/test/framework/easyconfigs/v2.0/GCC.eb @@ -5,6 +5,9 @@ docstring test @author: Stijn De Weirdt (UGent) @maintainer: Kenneth Hoste (UGent) """ +# should be EB_GCC, but OK for testing purposes +easyblock = 'EB_toy' + name = "GCC" homepage = 'http://gcc.gnu.org/' diff --git a/test/framework/easyconfigs/v2.0/doesnotexist.eb b/test/framework/easyconfigs/v2.0/doesnotexist.eb index 7067a83a07..7a85355ce6 100644 --- a/test/framework/easyconfigs/v2.0/doesnotexist.eb +++ b/test/framework/easyconfigs/v2.0/doesnotexist.eb @@ -5,6 +5,7 @@ docstring test @author: Stijn De Weirdt (UGent) @maintainer: Kenneth Hoste (UGent) """ +easyblock = 'ConfigureMake' name = 'doesnotexist' diff --git a/test/framework/easyconfigs/v2.0/gzip.eb b/test/framework/easyconfigs/v2.0/gzip.eb index de795268c7..ada19a7566 100644 --- a/test/framework/easyconfigs/v2.0/gzip.eb +++ b/test/framework/easyconfigs/v2.0/gzip.eb @@ -41,4 +41,5 @@ versions = 1.4, 1.5 toolchains = dummy == dummy, goolf, GCC == 4.6.3, goolf == 1.4.10, ictce == 4.1.13 [DEFAULT] -moduleclass = base +easyblock = ConfigureMake +moduleclass = tools diff --git a/test/framework/easyconfigs/v2.0/libpng.eb b/test/framework/easyconfigs/v2.0/libpng.eb index 1cc9175bca..ecc5bd59f6 100644 --- a/test/framework/easyconfigs/v2.0/libpng.eb +++ b/test/framework/easyconfigs/v2.0/libpng.eb @@ -28,6 +28,7 @@ versions = 1.5.10, 1.5.11, 1.5.13, 1.5.14, 1.6.2, 1.6.3, 1.6.6 toolchains = goolf == 1.4.10, ictce == 4.1.13, goalf == 1.1.0-no-OFED [DEFAULT] +easyblock = ConfigureMake moduleclass = lib [DEPENDENCIES] diff --git a/test/framework/easyconfigversion.py b/test/framework/easyconfigversion.py index 21411c7be2..a7331b4d45 100644 --- a/test/framework/easyconfigversion.py +++ b/test/framework/easyconfigversion.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2014 Ghent University +# Copyright 2014-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/ebconfigobj.py b/test/framework/ebconfigobj.py index 667d1ba7b0..cad6b30e73 100644 --- a/test/framework/ebconfigobj.py +++ b/test/framework/ebconfigobj.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2014 Ghent University +# Copyright 2014-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 367c91e5a0..4e81c2b96e 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,16 +28,19 @@ @author: Toon Willems (Ghent University) @author: Kenneth Hoste (Ghent University) @author: Stijn De Weirdt (Ghent University) +@author: Ward Poelmans (Ghent University) """ import os import shutil import stat import tempfile -from test.framework.utilities import EnhancedTestCase, find_full_path +import urllib2 +from test.framework.utilities import EnhancedTestCase, init_config from unittest import TestLoader, main import easybuild.tools.filetools as ft from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.multidiff import multidiff class FileToolsTest(EnhancedTestCase): @@ -51,23 +54,13 @@ class FileToolsTest(EnhancedTestCase): ('0_foo+0x0x#-$__', 'EB_0_underscore_foo_plus_0x0x_hash__minus__dollar__underscore__underscore_'), ] - def setUp(self): - """Set up testcase.""" - super(FileToolsTest, self).setUp() - self.legacySetUp() - - def legacySetUp(self): - self.log.deprecated("legacySetUp", "2.0") - cfg_path = os.path.join('easybuild', 'easybuild_config.py') - cfg_full_path = find_full_path(cfg_path) - self.assertTrue(cfg_full_path) - def test_extract_cmd(self): """Test various extract commands.""" tests = [ ('test.zip', "unzip -qq test.zip"), ('/some/path/test.tar', "tar xf /some/path/test.tar"), ('test.tar.gz', "tar xzf test.tar.gz"), + ('test.TAR.GZ', "tar xzf test.TAR.GZ"), ('test.tgz', "tar xzf test.tgz"), ('test.gtgz', "tar xzf test.gtgz"), ('test.bz2', "bunzip2 test.bz2"), @@ -76,15 +69,20 @@ def test_extract_cmd(self): ('test.tb2', "tar xjf test.tb2"), ('test.tar.bz2', "tar xjf test.tar.bz2"), ('test.gz', "gunzip -c test.gz > test"), + ('untar.gz', "gunzip -c untar.gz > untar"), ("/some/path/test.gz", "gunzip -c /some/path/test.gz > test"), ('test.xz', "unxz test.xz"), ('test.tar.xz', "unxz test.tar.xz --stdout | tar x"), ('test.txz', "unxz test.txz --stdout | tar x"), + ('test.iso', "7z x test.iso"), + ('test.tar.Z', "tar xZf test.tar.Z"), ] for (fn, expected_cmd) in tests: cmd = ft.extract_cmd(fn) self.assertEqual(expected_cmd, cmd) + self.assertEqual("unzip -qq -o test.zip", ft.extract_cmd('test.zip', True)) + def test_convert_name(self): """Test convert_name function.""" name = ft.convert_name("test+test-test") @@ -97,6 +95,18 @@ def test_cwd(self): # used to be part of test_parse_log_error self.assertEqual(os.getcwd(), ft.find_base_dir()) + def test_find_base_dir(self): + """test if we find the correct base dir""" + tmpdir = tempfile.mkdtemp() + + foodir = os.path.join(tmpdir, 'foo') + os.mkdir(foodir) + os.mkdir(os.path.join(tmpdir, '.bar')) + os.mkdir(os.path.join(tmpdir, 'easybuild')) + + os.chdir(tmpdir) + self.assertTrue(os.path.samefile(foodir, ft.find_base_dir())) + def test_encode_class_name(self): """Test encoding of class names.""" for (class_name, encoded_class_name) in self.class_names: @@ -190,9 +200,38 @@ def test_download_file(self): target_location = os.path.join(self.test_buildpath, 'some', 'subdir', fn) # provide local file path as source URL test_dir = os.path.abspath(os.path.dirname(__file__)) - source_url = os.path.join('file://', test_dir, 'sandbox', 'sources', 'toy', fn) + source_url = 'file://%s/sandbox/sources/toy/%s' % (test_dir, fn) + res = ft.download_file(fn, source_url, target_location) + self.assertEqual(res, target_location, "'download' of local file works") + + # non-existing files result in None return value + self.assertEqual(ft.download_file(fn, 'file://%s/nosuchfile' % test_dir, target_location), None) + + # install broken proxy handler for opening local files + # this should make urllib2.urlopen use this broken proxy for downloading from a file:// URL + proxy_handler = urllib2.ProxyHandler({'file': 'file://%s/nosuchfile' % test_dir}) + urllib2.install_opener(urllib2.build_opener(proxy_handler)) + + # downloading over a broken proxy results in None return value (failed download) + # this tests whether proxies are taken into account by download_file + self.assertEqual(ft.download_file(fn, source_url, target_location), None, "download over broken proxy fails") + + # restore a working file handler, and retest download of local file + urllib2.install_opener(urllib2.build_opener(urllib2.FileHandler())) res = ft.download_file(fn, source_url, target_location) - self.assertEqual(res, target_location) + self.assertEqual(res, target_location, "'download' of local file works after removing broken proxy") + + # make sure specified timeout is parsed correctly (as a float, not a string) + opts = init_config(args=['--download-timeout=5.3']) + init_config(build_options={'download_timeout': opts.download_timeout}) + target_location = os.path.join(self.test_prefix, 'jenkins_robots.txt') + url = 'https://jenkins1.ugent.be/robots.txt' + try: + urllib2.urlopen(url) + res = ft.download_file(fn, url, target_location) + self.assertEqual(res, target_location, "download with specified timeout works") + except urllib2.URLError: + print "Skipping timeout test in test_download_file (working offline)" def test_mkdir(self): """Test mkdir function.""" @@ -243,6 +282,41 @@ def check_mkdir(path, error=None, **kwargs): shutil.rmtree(tmpdir) + def test_path_matches(self): + # set up temporary directories + tmpdir = tempfile.mkdtemp() + path1 = os.path.join(tmpdir, 'path1') + ft.mkdir(path1) + path2 = os.path.join(tmpdir, 'path2') + ft.mkdir(path1) + symlink = os.path.join(tmpdir, 'symlink') + os.symlink(path1, symlink) + missing = os.path.join(tmpdir, 'missing') + + self.assertFalse(ft.path_matches(missing, [path1, path2])) + self.assertFalse(ft.path_matches(path1, [missing])) + self.assertFalse(ft.path_matches(path1, [missing, path2])) + self.assertFalse(ft.path_matches(path2, [missing, symlink])) + self.assertTrue(ft.path_matches(path1, [missing, symlink])) + + # cleanup + shutil.rmtree(tmpdir) + + def test_read_write_file(self): + """Test reading/writing files.""" + tmpdir = tempfile.mkdtemp() + + fp = os.path.join(tmpdir, 'test.txt') + txt = "test123" + ft.write_file(fp, txt) + self.assertEqual(ft.read_file(fp), txt) + + txt2 = '\n'.join(['test', '123']) + ft.write_file(fp, txt2, append=True) + self.assertEqual(ft.read_file(fp), txt+txt2) + + shutil.rmtree(tmpdir) + def test_det_patched_files(self): """Test det_patched_files function.""" pf = os.path.join(os.path.dirname(__file__), 'sandbox', 'sources', 'toy', 'toy-0.0_typo.patch') @@ -266,6 +340,119 @@ def test_guess_patch_level(self): ]: self.assertEqual(ft.guess_patch_level([patched_file], self.test_buildpath), correct_patch_level) + def test_move_logs(self): + """Test move_logs function.""" + fh, fp = tempfile.mkstemp() + os.close(fh) + ft.write_file(fp, 'foobar') + ft.write_file(fp + '.1', 'moarfoobar') + ft.move_logs(fp, os.path.join(self.test_prefix, 'foo.log')) + + self.assertEqual(ft.read_file(os.path.join(self.test_prefix, 'foo.log')), 'foobar') + self.assertEqual(ft.read_file(os.path.join(self.test_prefix, 'foo.log.1')), 'moarfoobar') + + ft.write_file(os.path.join(self.test_prefix, 'bar.log'), 'bar') + ft.write_file(os.path.join(self.test_prefix, 'bar.log_1'), 'barbar') + + fh, fp = tempfile.mkstemp() + os.close(fh) + ft.write_file(fp, 'moarbar') + ft.write_file(fp + '.1', 'evenmoarbar') + ft.move_logs(fp, os.path.join(self.test_prefix, 'bar.log')) + + logs = ['bar.log', 'bar.log.1', 'bar.log_0', 'bar.log_1', + os.path.basename(self.logfile), + 'foo.log', 'foo.log.1'] + self.assertEqual(sorted([f for f in os.listdir(self.test_prefix) if 'log' in f]), logs) + self.assertEqual(ft.read_file(os.path.join(self.test_prefix, 'bar.log_0')), 'bar') + self.assertEqual(ft.read_file(os.path.join(self.test_prefix, 'bar.log_1')), 'barbar') + self.assertEqual(ft.read_file(os.path.join(self.test_prefix, 'bar.log')), 'moarbar') + self.assertEqual(ft.read_file(os.path.join(self.test_prefix, 'bar.log.1')), 'evenmoarbar') + + def test_multidiff(self): + """Test multidiff function.""" + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + other_toy_ecs = [ + os.path.join(test_easyconfigs, 'toy-0.0-deps.eb'), + os.path.join(test_easyconfigs, 'toy-0.0-gompi-1.3.12-test.eb'), + ] + + # default (colored) + lines = multidiff(os.path.join(test_easyconfigs, 'toy-0.0.eb'), other_toy_ecs).split('\n') + expected = "Comparing \x1b[0;35mtoy-0.0.eb\x1b[0m with toy-0.0-deps.eb, toy-0.0-gompi-1.3.12-test.eb" + + red = "\x1b[0;41m" + green = "\x1b[0;42m" + endcol = "\x1b[0m" + + self.assertEqual(lines[0], expected) + self.assertEqual(lines[1], "=====") + + # different versionsuffix + self.assertEqual(lines[2], "3 %s- versionsuffix = '-test'%s (1/2) toy-0.0-gompi-1.3.12-test.eb" % (red, endcol)) + self.assertEqual(lines[3], "3 %s- versionsuffix = '-deps'%s (1/2) toy-0.0-deps.eb" % (red, endcol)) + + # different toolchain in toy-0.0-gompi-1.3.12-test: '+' line (removed chars in toolchain name/version, in red) + expected = "7 %(endcol)s-%(endcol)s toolchain = {" + expected += "'name': '%(endcol)s%(red)sgo%(endcol)sm\x1b[0m%(red)spi%(endcol)s', " + expected = expected % {'endcol': endcol, 'green': green, 'red': red} + self.assertTrue(lines[7].startswith(expected)) + # different toolchain in toy-0.0-gompi-1.3.12-test: '+' line (added chars in toolchain name/version, in green) + expected = "7 %(endcol)s+%(endcol)s toolchain = {" + expected += "'name': '%(endcol)s%(green)sdu%(endcol)sm\x1b[0m%(green)smy%(endcol)s', " + expected = expected % {'endcol': endcol, 'green': green, 'red': red} + self.assertTrue(lines[8].startswith(expected)) + + # no postinstallcmds in toy-0.0-deps.eb + expected = "28 %s+ postinstallcmds = " % green + self.assertTrue(any([line.startswith(expected) for line in lines])) + self.assertTrue("29 %s+%s (1/2) toy-0.0-deps.eb" % (green, endcol) in lines) + self.assertEqual(lines[-1], "=====") + + lines = multidiff(os.path.join(test_easyconfigs, 'toy-0.0.eb'), other_toy_ecs, colored=False).split('\n') + self.assertEqual(lines[0], "Comparing toy-0.0.eb with toy-0.0-deps.eb, toy-0.0-gompi-1.3.12-test.eb") + self.assertEqual(lines[1], "=====") + + # different versionsuffix + self.assertEqual(lines[2], "3 - versionsuffix = '-test' (1/2) toy-0.0-gompi-1.3.12-test.eb") + self.assertEqual(lines[3], "3 - versionsuffix = '-deps' (1/2) toy-0.0-deps.eb") + + # different toolchain in toy-0.0-gompi-1.3.12-test: '+' line with squigly line underneath to mark removed chars + expected = "7 - toolchain = {'name': 'gompi', 'version': '1.3.12'} (1/2) toy" + self.assertTrue(lines[7].startswith(expected)) + expected = " ? ^^ ^^ ^^^^^^" + self.assertEqual(lines[8], expected) + # different toolchain in toy-0.0-gompi-1.3.12-test: '-' line with squigly line underneath to mark added chars + expected = "7 + toolchain = {'name': 'dummy', 'version': 'dummy'} (1/2) toy" + self.assertTrue(lines[9].startswith(expected)) + expected = " ? ^^ ^^ ^^^^^" + self.assertEqual(lines[10], expected) + + # no postinstallcmds in toy-0.0-deps.eb + expected = "28 + postinstallcmds = " + self.assertTrue(any([line.startswith(expected) for line in lines])) + self.assertTrue("29 + (1/2) toy-0.0-deps.eb" in lines) + + self.assertEqual(lines[-1], "=====") + + def test_weld_paths(self): + """Test weld_paths.""" + # works like os.path.join is there's no overlap + self.assertEqual(ft.weld_paths('/foo/bar', 'foobar/baz'), '/foo/bar/foobar/baz/') + self.assertEqual(ft.weld_paths('foo', 'bar/'), 'foo/bar/') + self.assertEqual(ft.weld_paths('foo/', '/bar'), '/bar/') + self.assertEqual(ft.weld_paths('/foo/', '/bar'), '/bar/') + + # overlap is taken into account + self.assertEqual(ft.weld_paths('foo/bar', 'bar/baz'), 'foo/bar/baz/') + self.assertEqual(ft.weld_paths('foo/bar/baz', 'bar/baz'), 'foo/bar/baz/') + self.assertEqual(ft.weld_paths('foo/bar', 'foo/bar/baz'), 'foo/bar/baz/') + self.assertEqual(ft.weld_paths('foo/bar', 'foo/bar'), 'foo/bar/') + self.assertEqual(ft.weld_paths('/foo/bar', 'foo/bar'), '/foo/bar/') + self.assertEqual(ft.weld_paths('/foo/bar', '/foo/bar'), '/foo/bar/') + self.assertEqual(ft.weld_paths('/foo', '/foo/bar/baz'), '/foo/bar/baz/') + + def suite(): """ returns all the testcases in this module """ return TestLoader().loadTestsFromTestCase(FileToolsTest) diff --git a/test/framework/format_convert.py b/test/framework/format_convert.py index 4cb5e70f3b..a9ae19f51e 100644 --- a/test/framework/format_convert.py +++ b/test/framework/format_convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2014 Ghent University +# Copyright 2014-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -61,6 +61,12 @@ def test_listofstrings(self): res = ListOfStrings(txt.replace(ListOfStrings.SEPARATOR_LIST, ListOfStrings.SEPARATOR_LIST + ' ')) self.assertEqual(res, dest) + # empty string yields a list with an empty string + self.assertEqual(ListOfStrings(''), ['']) + + # empty entries are retained + self.assertEqual(ListOfStrings('a,,b'), ['a', '', 'b']) + def test_dictofstrings(self): """Test dict of strings""" # test default separators diff --git a/test/framework/general.py b/test/framework/general.py new file mode 100644 index 0000000000..2f8cd84053 --- /dev/null +++ b/test/framework/general.py @@ -0,0 +1,103 @@ +## +# Copyright 2015-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Unit tests for general aspects of the EasyBuild framework + +@author: Kenneth hoste (Ghent University) +""" +import os +import re +from test.framework.utilities import EnhancedTestCase +from unittest import TestLoader, main + +import vsc + +import easybuild.framework +from easybuild.tools.filetools import read_file +from easybuild.tools.utilities import only_if_module_is_available + + +class GeneralTest(EnhancedTestCase): + """Test for general aspects of EasyBuild framework.""" + + def test_vsc_location(self): + """Make sure location of imported vsc module is not the framework itself.""" + # cfr. https://github.com/hpcugent/easybuild-framework/pull/1160 + # easybuild.framework.__file__ provides location to /easybuild/framework/__init__.py + framework_loc = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(easybuild.framework.__file__)))) + # vsc.__file__ provides location to /vsc/__init__.py + vsc_loc = os.path.dirname(os.path.dirname(os.path.abspath(vsc.__file__))) + # make sure vsc is being imported from outside of framework + msg = "vsc-base is not provided by EasyBuild framework itself, found location: %s" % vsc_loc + self.assertFalse(os.path.samefile(framework_loc, vsc_loc), msg) + + def test_error_reporting(self): + """Make sure error reporting is done correctly (no more log.error, log.exception).""" + # easybuild.framework.__file__ provides location to /easybuild/framework/__init__.py + easybuild_loc = os.path.dirname(os.path.dirname(os.path.abspath(easybuild.framework.__file__))) + + log_method_regexes = [ + re.compile("log\.error\("), + re.compile("log\.exception\("), + re.compile("log\.raiseException\("), + ] + + for dirpath, _, filenames in os.walk(easybuild_loc): + for filename in [f for f in filenames if f.endswith('.py')]: + path = os.path.join(dirpath, filename) + txt = read_file(path) + for regex in log_method_regexes: + self.assertFalse(regex.search(txt), "No match for '%s' in %s" % (regex.pattern, path)) + + def test_only_if_module_is_available(self): + """Test only_if_module_is_available decorator.""" + @only_if_module_is_available('easybuild') + def foo(): + pass + + foo() + + @only_if_module_is_available('nosuchmoduleoutthere', pkgname='nosuchpkg') + def bar(): + pass + + err_pat = "required module 'nosuchmoduleoutthere' is not available.*package nosuchpkg.*pypi/nosuchpkg" + self.assertErrorRegex(ImportError, err_pat, bar) + + class Foo(): + @only_if_module_is_available('thisdoesnotexist', url='http://example.com') + def foobar(self): + pass + + err_pat = r"required module 'thisdoesnotexist' is not available \(available from http://example.com\)" + self.assertErrorRegex(ImportError, err_pat, Foo().foobar) + + +def suite(): + """ returns all the testcases in this module """ + return TestLoader().loadTestsFromTestCase(GeneralTest) + +if __name__ == '__main__': + main() diff --git a/test/framework/github.py b/test/framework/github.py index 109abf5901..2d6b8ca347 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,12 +29,20 @@ """ import os +import re +import shutil +import tempfile from test.framework.utilities import EnhancedTestCase from unittest import TestLoader, main +from urllib2 import URLError -from easybuild.tools.github import Githubfs, fetch_github_token +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import read_file, write_file +import easybuild.tools.github as gh +# test account, for which a token is available +GITHUB_TEST_ACCOUNT = 'easybuild_test' # the user who's repo to test GITHUB_USER = "hpcugent" # the repo of this user to use in this test @@ -50,48 +58,113 @@ class GithubTest(EnhancedTestCase): def setUp(self): """setup""" super(GithubTest, self).setUp() - github_user = 'easybuild_test' - github_token = fetch_github_token(github_user) - if github_token is None: + self.github_token = gh.fetch_github_token(GITHUB_TEST_ACCOUNT) + if self.github_token is None: self.ghfs = None else: - self.ghfs = Githubfs(GITHUB_USER, GITHUB_REPO, GITHUB_BRANCH, github_user, None, github_token) + self.ghfs = gh.Githubfs(GITHUB_USER, GITHUB_REPO, GITHUB_BRANCH, GITHUB_TEST_ACCOUNT, None, self.github_token) def test_walk(self): """test the gitubfs walk function""" - # TODO: this will not work when rate limited, so we should have a test account token here - if self.ghfs is not None: - try: - expected = [(None, ['a_directory', 'second_dir'], ['README.md']), - ('a_directory', ['a_subdirectory'], ['a_file.txt']), ('a_directory/a_subdirectory', [], - ['a_file.txt']), ('second_dir', [], ['a_file.txt'])] - self.assertEquals([x for x in self.ghfs.walk(None)], expected) - except IOError: - pass - else: + if self.github_token is None: print "Skipping test_walk, no GitHub token available?" + return + + try: + expected = [(None, ['a_directory', 'second_dir'], ['README.md']), + ('a_directory', ['a_subdirectory'], ['a_file.txt']), ('a_directory/a_subdirectory', [], + ['a_file.txt']), ('second_dir', [], ['a_file.txt'])] + self.assertEquals([x for x in self.ghfs.walk(None)], expected) + except IOError: + pass def test_read_api(self): """Test the githubfs read function""" - if self.ghfs is not None: - try: - self.assertEquals(self.ghfs.read("a_directory/a_file.txt").strip(), "this is a line of text") - except IOError: - pass - else: + if self.github_token is None: print "Skipping test_read_api, no GitHub token available?" + return + + try: + self.assertEquals(self.ghfs.read("a_directory/a_file.txt").strip(), "this is a line of text") + except IOError: + pass def test_read(self): """Test the githubfs read function without using the api""" - if self.ghfs is not None: - try: - fp = self.ghfs.read("a_directory/a_file.txt", api=False) - self.assertEquals(open(fp, 'r').read().strip(), "this is a line of text") - os.remove(fp) - except (IOError, OSError): - pass - else: + if self.github_token is None: print "Skipping test_read, no GitHub token available?" + return + + try: + fp = self.ghfs.read("a_directory/a_file.txt", api=False) + self.assertEquals(open(fp, 'r').read().strip(), "this is a line of text") + os.remove(fp) + except (IOError, OSError): + pass + + def test_fetch_easyconfigs_from_pr(self): + """Test fetch_easyconfigs_from_pr function.""" + if self.github_token is None: + print "Skipping test_fetch_easyconfigs_from_pr, no GitHub token available?" + return + + tmpdir = tempfile.mkdtemp() + # PR for ictce/6.2.5, see https://github.com/hpcugent/easybuild-easyconfigs/pull/726/files + all_ecs = ['gzip-1.6-ictce-6.2.5.eb', 'icc-2013_sp1.2.144.eb', 'ictce-6.2.5.eb', 'ifort-2013_sp1.2.144.eb', + 'imkl-11.1.2.144.eb', 'impi-4.1.3.049.eb'] + try: + ec_files = gh.fetch_easyconfigs_from_pr(726, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT) + self.assertEqual(all_ecs, sorted([os.path.basename(f) for f in ec_files])) + self.assertEqual(all_ecs, sorted(os.listdir(tmpdir))) + + # PR for EasyBuild v1.13.0 release (250+ commits, 218 files changed) + err_msg = "PR #897 contains more than .* commits, can't obtain last commit" + self.assertErrorRegex(EasyBuildError, err_msg, gh.fetch_easyconfigs_from_pr, 897, + github_user=GITHUB_TEST_ACCOUNT) + + except URLError, err: + print "Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err + + shutil.rmtree(tmpdir) + + def test_fetch_latest_commit_sha(self): + """Test fetch_latest_commit_sha function.""" + sha = gh.fetch_latest_commit_sha('easybuild-framework', 'hpcugent') + self.assertTrue(re.match('^[0-9a-f]{40}$', sha)) + sha = gh.fetch_latest_commit_sha('easybuild-easyblocks', 'hpcugent', branch='develop') + self.assertTrue(re.match('^[0-9a-f]{40}$', sha)) + + def test_download_repo(self): + """Test download_repo function.""" + # default: download tarball for master branch of hpcugent/easybuild-easyconfigs repo + path = gh.download_repo(path=self.test_prefix) + repodir = os.path.join(self.test_prefix, 'hpcugent', 'easybuild-easyconfigs-master') + self.assertTrue(os.path.samefile(path, repodir)) + self.assertTrue(os.path.exists(repodir)) + shafile = os.path.join(repodir, 'latest-sha') + self.assertTrue(re.match('^[0-9a-f]{40}$', read_file(shafile))) + self.assertTrue(os.path.exists(os.path.join(repodir, 'easybuild', 'easyconfigs', 'f', 'foss', 'foss-2015a.eb'))) + + # existing downloaded repo is not reperformed, except if SHA is different + account, repo, branch = 'boegel', 'easybuild-easyblocks', 'develop' + repodir = os.path.join(self.test_prefix, account, '%s-%s' % (repo, branch)) + latest_sha = gh.fetch_latest_commit_sha(repo, account, branch=branch) + + # put 'latest-sha' fail in place, check whether repo was (re)downloaded (should not) + shafile = os.path.join(repodir, 'latest-sha') + write_file(shafile, latest_sha) + path = gh.download_repo(repo=repo, branch=branch, account=account, path=self.test_prefix) + self.assertTrue(os.path.samefile(path, repodir)) + self.assertEqual(os.listdir(repodir), ['latest-sha']) + + # remove 'latest-sha' file and verify that download was performed + os.remove(shafile) + path = gh.download_repo(repo=repo, branch=branch, account=account, path=self.test_prefix) + self.assertTrue(os.path.samefile(path, repodir)) + self.assertTrue('easybuild' in os.listdir(repodir)) + self.assertTrue(re.match('^[0-9a-f]{40}$', read_file(shafile))) + self.assertTrue(os.path.exists(os.path.join(repodir, 'easybuild', 'easyblocks', '__init__.py'))) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/include.py b/test/framework/include.py new file mode 100644 index 0000000000..07e7476b94 --- /dev/null +++ b/test/framework/include.py @@ -0,0 +1,214 @@ +# # +# Copyright 2013-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Unit tests for eb command line options. + +@author: Kenneth Hoste (Ghent University) +""" +import os +import sys +from test.framework.utilities import EnhancedTestCase +from unittest import TestLoader +from unittest import main as unittestmain + +from easybuild.tools.filetools import mkdir, write_file +from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains + + +def up(path, cnt): + """Return path N times up.""" + if cnt > 0: + path = up(os.path.dirname(path), cnt-1) + return path + + +class IncludeTest(EnhancedTestCase): + """Testcases for command line options.""" + + logfile = None + + def test_include_easyblocks(self): + """Test include_easyblocks().""" + test_easyblocks = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'easybuild', 'easyblocks') + + # put a couple of custom easyblocks in place, to test + myeasyblocks = os.path.join(self.test_prefix, 'myeasyblocks') + mkdir(os.path.join(myeasyblocks, 'generic'), parents=True) + + myfoo_easyblock_txt = '\n'.join([ + "from easybuild.easyblocks.generic.configuremake import ConfigureMake", + "class EB_Foo(ConfigureMake):", + " pass", + ]) + write_file(os.path.join(myeasyblocks, 'myfoo.py'), myfoo_easyblock_txt) + + mybar_easyblock_txt = '\n'.join([ + "from easybuild.framework.easyblock import EasyBlock", + "class Bar(EasyBlock):", + " pass", + ]) + write_file(os.path.join(myeasyblocks, 'generic', 'mybar.py'), mybar_easyblock_txt) + + # expand set of known easyblocks with our custom ones + glob_paths = [os.path.join(myeasyblocks, '*'), os.path.join(myeasyblocks, '*/*.py')] + included_easyblocks_path = include_easyblocks(self.test_prefix, glob_paths) + + expected_paths = ['__init__.py', 'easyblocks/__init__.py', 'easyblocks/myfoo.py', + 'easyblocks/generic/__init__.py', 'easyblocks/generic/mybar.py'] + for filepath in expected_paths: + fullpath = os.path.join(included_easyblocks_path, 'easybuild', filepath) + self.assertTrue(os.path.exists(fullpath), "%s exists" % fullpath) + + # path to included easyblocks should be prepended to Python search path + self.assertEqual(sys.path[0], included_easyblocks_path) + + # importing custom easyblocks should work + import easybuild.easyblocks.myfoo + myfoo_pyc_path = easybuild.easyblocks.myfoo.__file__ + myfoo_real_py_path = os.path.realpath(os.path.join(os.path.dirname(myfoo_pyc_path), 'myfoo.py')) + self.assertTrue(os.path.samefile(up(myfoo_real_py_path, 1), myeasyblocks)) + + import easybuild.easyblocks.generic.mybar + mybar_pyc_path = easybuild.easyblocks.generic.mybar.__file__ + mybar_real_py_path = os.path.realpath(os.path.join(os.path.dirname(mybar_pyc_path), 'mybar.py')) + self.assertTrue(os.path.samefile(up(mybar_real_py_path, 2), myeasyblocks)) + + # existing (test) easyblocks are unaffected + import easybuild.easyblocks.foofoo + foofoo_path = os.path.dirname(os.path.dirname(easybuild.easyblocks.foofoo.__file__)) + self.assertTrue(os.path.samefile(foofoo_path, test_easyblocks)) + + def test_include_easyblocks_priority(self): + """Test whether easyblocks included via include_easyblocks() get prioroity over others.""" + test_easyblocks = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'easybuild', 'easyblocks') + + # make sure that test 'foo' easyblocks is there + import easybuild.easyblocks.foo + foo_path = os.path.dirname(os.path.dirname(easybuild.easyblocks.foo.__file__)) + self.assertTrue(os.path.samefile(foo_path, test_easyblocks)) + + # inject custom 'foo' easyblocks + myeasyblocks = os.path.join(self.test_prefix, 'myeasyblocks') + mkdir(myeasyblocks) + + # 'undo' import of foo easyblock + del sys.modules['easybuild.easyblocks.foo'] + + foo_easyblock_txt = '\n'.join([ + "from easybuild.framework.easyblock import EasyBlock", + "class EB_Foo(EasyBlock):", + " pass", + ]) + write_file(os.path.join(myeasyblocks, 'foo.py'), foo_easyblock_txt) + include_easyblocks(self.test_prefix, [os.path.join(myeasyblocks, 'foo.py')]) + + foo_pyc_path = easybuild.easyblocks.foo.__file__ + foo_real_py_path = os.path.realpath(os.path.join(os.path.dirname(foo_pyc_path), 'foo.py')) + self.assertFalse(os.path.samefile(os.path.dirname(foo_pyc_path), test_easyblocks)) + self.assertTrue(os.path.samefile(foo_real_py_path, os.path.join(myeasyblocks, 'foo.py'))) + + # 'undo' import of foo easyblock + del sys.modules['easybuild.easyblocks.foo'] + + def test_include_mns(self): + """Test include_module_naming_schemes().""" + testdir = os.path.dirname(os.path.abspath(__file__)) + test_mns = os.path.join(testdir, 'sandbox', 'easybuild', 'module_naming_scheme') + + my_mns = os.path.join(self.test_prefix, 'my_mns') + mkdir(my_mns) + + my_mns_txt = '\n'.join([ + "from easybuild.tools.module_naming_scheme import ModuleNamingScheme", + "class MyMNS(ModuleNamingScheme):", + " pass", + ]) + write_file(os.path.join(my_mns, 'my_mns.py'), my_mns_txt) + + # include custom MNS + included_mns_path = include_module_naming_schemes(self.test_prefix, [os.path.join(my_mns, '*.py')]) + + expected_paths = ['__init__.py', 'tools/__init__.py', 'tools/module_naming_scheme/__init__.py', + 'tools/module_naming_scheme/my_mns.py'] + for filepath in expected_paths: + fullpath = os.path.join(included_mns_path, 'easybuild', filepath) + self.assertTrue(os.path.exists(fullpath), "%s exists" % fullpath) + + # path to included MNSs should be prepended to Python search path + self.assertEqual(sys.path[0], included_mns_path) + + # importing custom MNS should work + import easybuild.tools.module_naming_scheme.my_mns + my_mns_pyc_path = easybuild.tools.module_naming_scheme.my_mns.__file__ + my_mns_real_py_path = os.path.realpath(os.path.join(os.path.dirname(my_mns_pyc_path), 'my_mns.py')) + self.assertTrue(os.path.samefile(up(my_mns_real_py_path, 1), my_mns)) + + def test_include_toolchains(self): + """Test include_toolchains().""" + my_toolchains = os.path.join(self.test_prefix, 'my_toolchains') + mkdir(my_toolchains) + for subdir in ['compiler', 'fft', 'linalg', 'mpi']: + mkdir(os.path.join(my_toolchains, subdir)) + + my_tc_txt = '\n'.join([ + "from easybuild.toolchains.compiler.my_compiler import MyCompiler", + "class MyTc(MyCompiler):", + " pass", + ]) + write_file(os.path.join(my_toolchains, 'my_tc.py'), my_tc_txt) + + my_compiler_txt = '\n'.join([ + "from easybuild.tools.toolchain.compiler import Compiler", + "class MyCompiler(Compiler):", + " pass", + ]) + write_file(os.path.join(my_toolchains, 'compiler', 'my_compiler.py'), my_compiler_txt) + + # include custom MNS + glob_paths = [os.path.join(my_toolchains, '*.py'), os.path.join(my_toolchains, '*', '*.py')] + included_tcs_path = include_toolchains(self.test_prefix, glob_paths) + + expected_paths = ['__init__.py', 'toolchains/__init__.py', 'toolchains/compiler/__init__.py', + 'toolchains/my_tc.py', 'toolchains/compiler/my_compiler.py'] + for filepath in expected_paths: + fullpath = os.path.join(included_tcs_path, 'easybuild', filepath) + self.assertTrue(os.path.exists(fullpath), "%s exists" % fullpath) + + # path to included MNSs should be prepended to Python search path + self.assertEqual(sys.path[0], included_tcs_path) + + # importing custom MNS should work + import easybuild.toolchains.my_tc + my_tc_pyc_path = easybuild.toolchains.my_tc.__file__ + my_tc_real_py_path = os.path.realpath(os.path.join(os.path.dirname(my_tc_pyc_path), 'my_tc.py')) + self.assertTrue(os.path.samefile(up(my_tc_real_py_path, 1), my_toolchains)) + + +def suite(): + """ returns all the testcases in this module """ + return TestLoader().loadTestsFromTestCase(IncludeTest) + +if __name__ == '__main__': + unittestmain() diff --git a/test/framework/license.py b/test/framework/license.py index 55c389a3db..445a1d1c5c 100644 --- a/test/framework/license.py +++ b/test/framework/license.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,7 +30,7 @@ from test.framework.utilities import EnhancedTestCase from unittest import TestLoader, main -from easybuild.framework.easyconfig.licenses import License, VeryRestrictive, what_licenses +from easybuild.framework.easyconfig.licenses import License, LicenseVeryRestrictive, what_licenses class LicenseTest(EnhancedTestCase): @@ -39,9 +39,9 @@ class LicenseTest(EnhancedTestCase): def test_common_ones(self): """Check if a number of common licenses can be found""" lics = what_licenses() - commonlicenses = ['VeryRestrictive', 'GPLv2', 'GPLv3'] + commonlicenses = ['LicenseVeryRestrictive', 'LicenseGPLv2', 'LicenseGPLv3'] for lic in commonlicenses: - self.assertTrue(lic in lics) + self.assertTrue(lic in lics, "%s found in %s" % (lic, lics.keys())) def test_default_license(self): """Verify that the default License class is very restrictive""" @@ -51,9 +51,18 @@ def test_default_license(self): def test_veryrestrictive_license(self): """Verify that the very restrictive class is very restrictive""" - self.assertFalse(VeryRestrictive.DISTRIBUTE_SOURCE) - self.assertTrue(VeryRestrictive.GROUP_SOURCE) - self.assertTrue(VeryRestrictive.GROUP_BINARY) + self.assertFalse(LicenseVeryRestrictive.DISTRIBUTE_SOURCE) + self.assertTrue(LicenseVeryRestrictive.GROUP_SOURCE) + self.assertTrue(LicenseVeryRestrictive.GROUP_BINARY) + + def test_licenses(self): + """Test format of available licenses.""" + lics = what_licenses() + for lic in lics: + self.assertTrue(isinstance(lic, basestring)) + self.assertTrue(lic.startswith('License')) + self.assertTrue(issubclass(lics[lic], License)) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 8c5f9167c9..b4bbcdf244 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,26 +34,28 @@ import sys import tempfile from test.framework.utilities import EnhancedTestCase, init_config -from unittest import TestLoader, main +from unittest import TestLoader, TestSuite, TextTestRunner, main from vsc.utils.fancylogger import setLogLevelDebug, logToScreen from vsc.utils.missing import get_subclasses import easybuild.tools.module_generator from easybuild.framework.easyconfig.tools import process_easyconfig from easybuild.tools import config -from easybuild.tools.module_generator import ModuleGenerator +from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl from easybuild.tools.module_naming_scheme.utilities import is_valid_module_name from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig.easyconfig import EasyConfig, ActiveMNS from easybuild.tools.build_log import EasyBuildError -from test.framework.utilities import find_full_path +from test.framework.utilities import find_full_path, init_config class ModuleGeneratorTest(EnhancedTestCase): - """ testcase for ModuleGenerator """ + """Tests for module_generator module.""" + + MODULE_GENERATOR_CLASS = None def setUp(self): - """ initialize ModuleGenerator with test Application """ + """Test setup.""" super(ModuleGeneratorTest, self).setUp() # find .eb file eb_path = os.path.join(os.path.join(os.path.dirname(__file__), 'easyconfigs'), 'gzip-1.4.eb') @@ -62,83 +64,161 @@ def setUp(self): ec = EasyConfig(eb_full_path) self.eb = EasyBlock(ec) - self.modgen = ModuleGenerator(self.eb) + self.modgen = self.MODULE_GENERATOR_CLASS(self.eb) self.modgen.app.installdir = tempfile.mkdtemp(prefix='easybuild-modgen-test-') self.orig_module_naming_scheme = config.get_module_naming_scheme() - def tearDown(self): - """cleanup""" - super(ModuleGeneratorTest, self).tearDown() - os.remove(self.eb.logfile) - shutil.rmtree(self.modgen.app.installdir) - def test_descr(self): """Test generation of module description (which includes '#%Module' header).""" + gzip_txt = "gzip (GNU zip) is a popular data compression program as a replacement for compress " gzip_txt += "- Homepage: http://www.gzip.org/" - expected = '\n'.join([ - "#%Module", - "", - "proc ModulesHelp { } {", - " puts stderr { %s" % gzip_txt, - " }", - "}", - "", - "module-whatis {Description: %s}" % gzip_txt, - "", - "set root %s" % self.modgen.app.installdir, - "", - "conflict gzip", - "", - ]) + + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + expected = '\n'.join([ + "#%Module", + "proc ModulesHelp { } {", + " puts stderr { %s" % gzip_txt, + " }", + "}", + '', + "module-whatis {Description: %s}" % gzip_txt, + '', + "set root %s" % self.modgen.app.installdir, + '', + "conflict gzip", + '', + ]) + + else: + expected = '\n'.join([ + 'help([[%s]])' % gzip_txt, + "whatis([[Name: gzip]])" , + "whatis([[Version: 1.4]])" , + "whatis([[Description: %s]])" % gzip_txt, + "whatis([[Homepage: http://www.gzip.org/]])", + '', + 'local root = "%s"' % self.modgen.app.installdir, + '', + 'conflict("gzip")', + '', + ]) desc = self.modgen.get_description() self.assertEqual(desc, expected) def test_load(self): """Test load part in generated module file.""" - expected = [ - "", - "if { ![is-loaded mod_name] } {", - " module load mod_name", - "}", - "", - ] - self.assertEqual('\n'.join(expected), self.modgen.load_module("mod_name")) - # with recursive unloading: no if is-loaded guard - expected = [ - "", - "module load mod_name", - "", - ] - self.assertEqual('\n'.join(expected), self.modgen.load_module("mod_name", recursive_unload=True)) + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + # default: guarded module load (which implies no recursive unloading) + expected = [ + '', + "if { ![ is-loaded mod_name ] } {", + " module load mod_name", + "}", + '', + ] + self.assertEqual('\n'.join(expected), self.modgen.load_module("mod_name")) + + # with recursive unloading: no if is-loaded guard + expected = [ + '', + "module load mod_name", + '', + ] + self.assertEqual('\n'.join(expected), self.modgen.load_module("mod_name", recursive_unload=True)) + + init_config(build_options={'recursive_mod_unload': True}) + self.assertEqual('\n'.join(expected), self.modgen.load_module("mod_name")) + else: + # default: guarded module load (which implies no recursive unloading) + expected = '\n'.join([ + '', + 'if not isloaded("mod_name") then', + ' load("mod_name")', + 'end', + '', + ]) + self.assertEqual(expected,self.modgen.load_module("mod_name")) + + # with recursive unloading: no if isloaded guard + expected = '\n'.join([ + '', + 'load("mod_name")', + '', + ]) + self.assertEqual(expected, self.modgen.load_module("mod_name", recursive_unload=True)) + + init_config(build_options={'recursive_mod_unload': True}) + self.assertEqual(expected,self.modgen.load_module("mod_name")) def test_unload(self): """Test unload part in generated module file.""" - expected = '\n'.join([ - "", - "if { [is-loaded mod_name] } {", - " module unload mod_name", - "}", - "", - ]) - self.assertEqual(expected, self.modgen.unload_module("mod_name")) + + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + expected = '\n'.join([ + '', + "if { [ is-loaded mod_name ] } {", + " module unload mod_name", + "}", + '', + ]) + self.assertEqual(expected, self.modgen.unload_module("mod_name")) + else: + expected = '\n'.join([ + '', + 'if isloaded("mod_name") then', + ' unload("mod_name")', + "end", + '', + ]) + self.assertEqual(expected, self.modgen.unload_module("mod_name")) def test_prepend_paths(self): """Test generating prepend-paths statements.""" # test prepend_paths - expected = ''.join([ - "prepend-path\tkey\t\t$root/path1\n", - "prepend-path\tkey\t\t$root/path2\n", - ]) - self.assertEqual(expected, self.modgen.prepend_paths("key", ["path1", "path2"])) - - expected = "prepend-path\tbar\t\t$root/foo\n" - self.assertEqual(expected, self.modgen.prepend_paths("bar", "foo")) - self.assertEqual("prepend-path\tkey\t\t/abs/path\n", self.modgen.prepend_paths("key", ["/abs/path"], allow_abs=True)) + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + expected = ''.join([ + "prepend-path\tkey\t\t$root/path1\n", + "prepend-path\tkey\t\t$root/path2\n", + "prepend-path\tkey\t\t$root\n", + ]) + paths = ['path1', 'path2', ''] + self.assertEqual(expected, self.modgen.prepend_paths("key", paths)) + # 2nd call should still give same result, no side-effects like manipulating passed list 'paths'! + self.assertEqual(expected, self.modgen.prepend_paths("key", paths)) + + expected = "prepend-path\tbar\t\t$root/foo\n" + self.assertEqual(expected, self.modgen.prepend_paths("bar", "foo")) + + res = self.modgen.prepend_paths("key", ["/abs/path"], allow_abs=True) + self.assertEqual("prepend-path\tkey\t\t/abs/path\n", res) + + res = self.modgen.prepend_paths('key', ['1234@example.com'], expand_relpaths=False) + self.assertEqual("prepend-path\tkey\t\t1234@example.com\n", res) + + else: + expected = ''.join([ + 'prepend_path("key", pathJoin(root, "path1"))\n', + 'prepend_path("key", pathJoin(root, "path2"))\n', + 'prepend_path("key", root)\n', + ]) + paths = ['path1', 'path2', ''] + self.assertEqual(expected, self.modgen.prepend_paths("key", paths)) + # 2nd call should still give same result, no side-effects like manipulating passed list 'paths'! + self.assertEqual(expected, self.modgen.prepend_paths("key", paths)) + + expected = 'prepend_path("bar", pathJoin(root, "foo"))\n' + self.assertEqual(expected, self.modgen.prepend_paths("bar", "foo")) + + expected = 'prepend_path("key", "/abs/path")\n' + self.assertEqual(expected, self.modgen.prepend_paths("key", ["/abs/path"], allow_abs=True)) + + res = self.modgen.prepend_paths('key', ['1234@example.com'], expand_relpaths=False) + self.assertEqual('prepend_path("key", "1234@example.com")\n', res) self.assertErrorRegex(EasyBuildError, "Absolute path %s/foo passed to prepend_paths " \ "which only expects relative paths." % self.modgen.app.installdir, @@ -146,37 +226,56 @@ def test_prepend_paths(self): def test_use(self): """Test generating module use statements.""" - expected = '\n'.join([ - "module use /some/path", - "module use /foo/bar/baz", - ]) - self.assertEqual(self.modgen.use(["/some/path", "/foo/bar/baz"]), expected) + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + expected = ''.join([ + "module use /some/path\n", + "module use /foo/bar/baz\n", + ]) + self.assertEqual(self.modgen.use(["/some/path", "/foo/bar/baz"]), expected) + else: + expected = ''.join([ + 'prepend_path("MODULEPATH", "/some/path")\n', + 'prepend_path("MODULEPATH", "/foo/bar/baz")\n', + ]) + self.assertEqual(self.modgen.use(["/some/path", "/foo/bar/baz"]), expected) + def test_env(self): """Test setting of environment variables.""" # test set_environment - self.assertEqual('setenv\tkey\t\t"value"\n', self.modgen.set_environment("key", "value")) - self.assertEqual("setenv\tkey\t\t'va\"lue'\n", self.modgen.set_environment("key", 'va"lue')) - self.assertEqual('setenv\tkey\t\t"va\'lue"\n', self.modgen.set_environment("key", "va'lue")) - self.assertEqual('setenv\tkey\t\t"""va"l\'ue"""\n', self.modgen.set_environment("key", """va"l'ue""")) - + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + self.assertEqual('setenv\tkey\t\t"value"\n', self.modgen.set_environment("key", "value")) + self.assertEqual("setenv\tkey\t\t'va\"lue'\n", self.modgen.set_environment("key", 'va"lue')) + self.assertEqual('setenv\tkey\t\t"va\'lue"\n', self.modgen.set_environment("key", "va'lue")) + self.assertEqual('setenv\tkey\t\t"""va"l\'ue"""\n', self.modgen.set_environment("key", """va"l'ue""")) + else: + self.assertEqual('setenv("key", "value")\n', self.modgen.set_environment("key", "value")) + def test_alias(self): """Test setting of alias in modulefiles.""" - # test set_alias - self.assertEqual('set-alias\tkey\t\t"value"\n', self.modgen.set_alias("key", "value")) - self.assertEqual("set-alias\tkey\t\t'va\"lue'\n", self.modgen.set_alias("key", 'va"lue')) - self.assertEqual('set-alias\tkey\t\t"va\'lue"\n', self.modgen.set_alias("key", "va'lue")) - self.assertEqual('set-alias\tkey\t\t"""va"l\'ue"""\n', self.modgen.set_alias("key", """va"l'ue""")) + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + # test set_alias + self.assertEqual('set-alias\tkey\t\t"value"\n', self.modgen.set_alias("key", "value")) + self.assertEqual("set-alias\tkey\t\t'va\"lue'\n", self.modgen.set_alias("key", 'va"lue')) + self.assertEqual('set-alias\tkey\t\t"va\'lue"\n', self.modgen.set_alias("key", "va'lue")) + self.assertEqual('set-alias\tkey\t\t"""va"l\'ue"""\n', self.modgen.set_alias("key", """va"l'ue""")) + else: + self.assertEqual('setalias("key", "value")\n', self.modgen.set_alias("key", "value")) def test_load_msg(self): """Test including a load message in the module file.""" - tcl_load_msg = '\nif [ module-info mode load ] {\n puts stderr "test"\n}\n' - self.assertEqual(tcl_load_msg, self.modgen.msg_on_load('test')) - - def test_tcl_footer(self): - """Test including a Tcl footer.""" - tcltxt = 'puts stderr "foo"' - self.assertEqual(tcltxt, self.modgen.add_tcl_footer(tcltxt)) + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + tcl_load_msg = '\n'.join([ + '', + "if { [ module-info mode load ] } {", + " puts stderr \"test \\$test \\$test", + "test \\$foo \\$bar\"", + "}", + '', + ]) + self.assertEqual(tcl_load_msg, self.modgen.msg_on_load('test $test \\$test\ntest $foo \\$bar')) + else: + pass def test_module_naming_scheme(self): """Test using default module naming scheme.""" @@ -189,6 +288,7 @@ def test_module_naming_scheme(self): build_options = { 'check_osdeps': False, + 'external_modules_metadata': {}, 'robot_path': [ecs_dir], 'valid_stops': all_stops, 'validate': False, @@ -198,7 +298,7 @@ def test_module_naming_scheme(self): def test_mns(): """Test default module naming scheme.""" # test default naming scheme - for ec_file in ec_files: + for ec_file in [f for f in ec_files if not 'broken' in os.path.basename(f)]: ec_path = os.path.abspath(ec_file) ecs = process_easyconfig(ec_path, validate=False) # derive module name directly from easyconfig file name @@ -251,7 +351,7 @@ def test_mns(): init_config(build_options=build_options) err_pattern = 'nosucheasyconfigparameteravailable' - self.assertErrorRegex(KeyError, err_pattern, EasyConfig, os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb')) + self.assertErrorRegex(EasyBuildError, err_pattern, EasyConfig, os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb')) # test simple custom module naming scheme os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingScheme' @@ -276,10 +376,10 @@ def test_mns(): # note: these checksums will change if another easyconfig parameter is added ec2mod_map = { 'GCC-4.6.3.eb': 'GCC/9e9ab5a1e978f0843b5aedb63ac4f14c51efb859', - 'gzip-1.4.eb': 'gzip/8805ec3152d2a4a08b6c06d740c23abe1a4d059f', - 'gzip-1.4-GCC-4.6.3.eb': 'gzip/863557cc81811f8c3f4426a4b45aa269fa54130b', - 'gzip-1.5-goolf-1.4.10.eb': 'gzip/b63c2b8cc518905473ccda023100b2d3cff52d55', - 'gzip-1.5-ictce-4.1.13.eb': 'gzip/3d49f0e112708a95f79ed38b91b506366c0299ab', + 'gzip-1.4.eb': 'gzip/53d5c13e85cb6945bd43a58d1c8d4a4c02f3462d', + 'gzip-1.4-GCC-4.6.3.eb': 'gzip/585eba598f33c64ef01c6fa47af0fc37f3751311', + 'gzip-1.5-goolf-1.4.10.eb': 'gzip/fceb41e04c26b540b7276c4246d1ecdd1e8251c9', + 'gzip-1.5-ictce-4.1.13.eb': 'gzip/ae16b3a0a330d4323987b360c0d024f244ac4498', 'toy-0.0.eb': 'toy/44a206d9e8c14130cc9f79e061468303c6e91b53', 'toy-0.0-multiple.eb': 'toy/44a206d9e8c14130cc9f79e061468303c6e91b53', } @@ -287,30 +387,35 @@ def test_mns(): # test determining module name for dependencies (i.e. non-parsed easyconfigs) # using a module naming scheme that requires all easyconfig parameters + ec2mod_map['gzip-1.5-goolf-1.4.10.eb'] = 'gzip/.fceb41e04c26b540b7276c4246d1ecdd1e8251c9' for dep_ec, dep_spec in [ ('GCC-4.6.3.eb', { 'name': 'GCC', 'version': '4.6.3', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, + 'hidden': False, }), ('gzip-1.5-goolf-1.4.10.eb', { 'name': 'gzip', 'version': '1.5', 'versionsuffix': '', 'toolchain': {'name': 'goolf', 'version': '1.4.10'}, + 'hidden': True, }), ('toy-0.0-multiple.eb', { 'name': 'toy', 'version': '0.0', 'versionsuffix': '-multiple', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, + 'hidden': False, }), ]: # determine full module name self.assertEqual(ActiveMNS().det_full_module_name(dep_spec), ec2mod_map[dep_ec]) - ec = EasyConfig(os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb')) + ec = EasyConfig(os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb'), hidden=True) + self.assertEqual(ec.full_mod_name, ec2mod_map['gzip-1.5-goolf-1.4.10.eb']) self.assertEqual(ec.toolchain.det_short_module_name(), 'goolf/b7515d0efd346970f55e7aa8522e239a70007021') # restore default module naming scheme, and retest @@ -341,49 +446,138 @@ def test_mod_name_validation(self): self.assertTrue(is_valid_module_name('foo-bar/1.2.3')) self.assertTrue(is_valid_module_name('ictce')) + def test_is_short_modname_for(self): + """Test is_short_modname_for method of module naming schemes.""" + test_cases = [ + ('GCC/4.7.2', 'GCC', True), + ('gzip/1.6-gompi-1.4.10', 'gzip', True), + ('OpenMPI/1.6.4-GCC-4.7.2-no-OFED', 'OpenMPI', True), + ('BLACS/1.1-gompi-1.1.0-no-OFED', 'BLACS', True), + ('ScaLAPACK/1.8.0-gompi-1.1.0-no-OFED-ATLAS-3.8.4-LAPACK-3.4.0-BLACS-1.1', 'ScaLAPACK', True), + ('netCDF-C++/4.2-goolf-1.4.10', 'netCDF-C++', True), + ('gcc/4.7.2', 'GCC', False), + ('ScaLAPACK/1.8.0-gompi-1.1.0-no-OFED-ATLAS-3.8.4-LAPACK-3.4.0-BLACS-1.1', 'BLACS', False), + ('apps/blacs/1.1', 'BLACS', False), + ('lib/math/BLACS-stable/1.1', 'BLACS', False), + ] + for modname, softname, res in test_cases: + if res: + errormsg = "%s is recognised as a module for '%s'" % (modname, softname) + else: + errormsg = "%s is NOT recognised as a module for '%s'" % (modname, softname) + self.assertEqual(ActiveMNS().is_short_modname_for(modname, softname), res, errormsg) + def test_hierarchical_mns(self): """Test hierarchical module naming scheme.""" - ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs') + + moduleclasses = ['base', 'compiler', 'mpi', 'numlib', 'system', 'toolchain'] + ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') all_stops = [x[0] for x in EasyBlock.get_steps()] build_options = { 'check_osdeps': False, 'robot_path': [ecs_dir], 'valid_stops': all_stops, 'validate': False, + 'valid_module_classes': moduleclasses, } + + def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, init_modpaths): + """Test whether active module naming scheme returns expected values.""" + ec = EasyConfig(os.path.join(ecs_dir, ecfile)) + self.assertEqual(ActiveMNS().det_full_module_name(ec), os.path.join(mod_subdir, short_modname)) + self.assertEqual(ActiveMNS().det_short_module_name(ec), short_modname) + self.assertEqual(ActiveMNS().det_module_subdir(ec), mod_subdir) + self.assertEqual(ActiveMNS().det_modpath_extensions(ec), modpath_exts) + self.assertEqual(ActiveMNS().det_init_modulepaths(ec), init_modpaths) + os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS' init_config(build_options=build_options) - ec = EasyConfig(os.path.join(ecs_dir, 'GCC-4.7.2.eb')) - self.assertEqual(ActiveMNS().det_full_module_name(ec), 'Core/GCC/4.7.2') - self.assertEqual(ActiveMNS().det_short_module_name(ec), 'GCC/4.7.2') - self.assertEqual(ActiveMNS().det_module_subdir(ec), 'Core') - self.assertEqual(ActiveMNS().det_modpath_extensions(ec), ['Compiler/GCC/4.7.2']) - self.assertEqual(ActiveMNS().det_init_modulepaths(ec), ['Core']) + # format: easyconfig_file: (short_mod_name, mod_subdir, modpath_extensions, init_modpaths) + iccver = '2013.5.192-GCC-4.8.3' + impi_ec = 'impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb' + imkl_ec = 'imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb' + test_ecs = { + 'GCC-4.7.2.eb': ('GCC/4.7.2', 'Core', ['Compiler/GCC/4.7.2'], ['Core']), + 'OpenMPI-1.6.4-GCC-4.7.2.eb': ('OpenMPI/1.6.4', 'Compiler/GCC/4.7.2', ['MPI/GCC/4.7.2/OpenMPI/1.6.4'], ['Core']), + 'gzip-1.5-goolf-1.4.10.eb': ('gzip/1.5', 'MPI/GCC/4.7.2/OpenMPI/1.6.4', [], ['Core']), + 'goolf-1.4.10.eb': ('goolf/1.4.10', 'Core', [], ['Core']), + 'icc-2013.5.192-GCC-4.8.3.eb': ('icc/%s' % iccver, 'Core', ['Compiler/intel/%s' % iccver], ['Core']), + 'ifort-2013.3.163.eb': ('ifort/2013.3.163', 'Core', ['Compiler/intel/2013.3.163'], ['Core']), + 'CUDA-5.5.22-GCC-4.8.2.eb': ('CUDA/5.5.22', 'Compiler/GCC/4.8.2', ['Compiler/GCC-CUDA/4.8.2-5.5.22'], ['Core']), + impi_ec: ('impi/4.1.3.049', 'Compiler/intel/%s' % iccver, ['MPI/intel/%s/impi/4.1.3.049' % iccver], ['Core']), + imkl_ec: ('imkl/11.1.2.144', 'MPI/intel/%s/impi/4.1.3.049' % iccver, [], ['Core']), + } + for ecfile, mns_vals in test_ecs.items(): + test_ec(ecfile, *mns_vals) + + # impi with dummy toolchain, which doesn't make sense in a hierarchical context + ec = EasyConfig(os.path.join(ecs_dir, 'impi-4.1.3.049.eb')) + self.assertErrorRegex(EasyBuildError, 'No compiler available.*MPI lib', ActiveMNS().det_modpath_extensions, ec) - ec = EasyConfig(os.path.join(ecs_dir, 'OpenMPI-1.6.4-GCC-4.7.2.eb')) - self.assertEqual(ActiveMNS().det_full_module_name(ec), 'Compiler/GCC/4.7.2/OpenMPI/1.6.4') - self.assertEqual(ActiveMNS().det_short_module_name(ec), 'OpenMPI/1.6.4') - self.assertEqual(ActiveMNS().det_module_subdir(ec), 'Compiler/GCC/4.7.2') - self.assertEqual(ActiveMNS().det_modpath_extensions(ec), ['MPI/GCC/4.7.2/OpenMPI/1.6.4']) - self.assertEqual(ActiveMNS().det_init_modulepaths(ec), ['Core']) + os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS' + init_config(build_options=build_options) - ec = EasyConfig(os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb')) - self.assertEqual(ActiveMNS().det_full_module_name(ec), 'MPI/GCC/4.7.2/OpenMPI/1.6.4/gzip/1.5') - self.assertEqual(ActiveMNS().det_short_module_name(ec), 'gzip/1.5') - self.assertEqual(ActiveMNS().det_module_subdir(ec), 'MPI/GCC/4.7.2/OpenMPI/1.6.4') - self.assertEqual(ActiveMNS().det_modpath_extensions(ec), []) - self.assertEqual(ActiveMNS().det_init_modulepaths(ec), ['Core']) + # format: easyconfig_file: (short_mod_name, mod_subdir, modpath_extensions) + test_ecs = { + 'GCC-4.7.2.eb': ('GCC/4.7.2', 'Core/compiler', + ['Compiler/GCC/4.7.2/%s' % c for c in moduleclasses]), + 'OpenMPI-1.6.4-GCC-4.7.2.eb': ('OpenMPI/1.6.4', 'Compiler/GCC/4.7.2/mpi', + ['MPI/GCC/4.7.2/OpenMPI/1.6.4/%s' % c for c in moduleclasses]), + 'gzip-1.5-goolf-1.4.10.eb': ('gzip/1.5', 'MPI/GCC/4.7.2/OpenMPI/1.6.4/tools', + []), + 'goolf-1.4.10.eb': ('goolf/1.4.10', 'Core/toolchain', + []), + 'icc-2013.5.192-GCC-4.8.3.eb': ('icc/%s' % iccver, 'Core/compiler', + ['Compiler/intel/%s/%s' % (iccver, c) for c in moduleclasses]), + 'ifort-2013.3.163.eb': ('ifort/2013.3.163', 'Core/compiler', + ['Compiler/intel/2013.3.163/%s' % c for c in moduleclasses]), + 'CUDA-5.5.22-GCC-4.8.2.eb': ('CUDA/5.5.22', 'Compiler/GCC/4.8.2/system', + ['Compiler/GCC-CUDA/4.8.2-5.5.22/%s' % c for c in moduleclasses]), + impi_ec: ('impi/4.1.3.049', 'Compiler/intel/%s/mpi' % iccver, + ['MPI/intel/%s/impi/4.1.3.049/%s' % (iccver, c) for c in moduleclasses]), + imkl_ec: ('imkl/11.1.2.144', 'MPI/intel/%s/impi/4.1.3.049/numlib' % iccver, + []), + } + for ecfile, mns_vals in test_ecs.items(): + test_ec(ecfile, *mns_vals, init_modpaths = ['Core/%s' % c for c in moduleclasses]) + + # impi with dummy toolchain, which doesn't make sense in a hierarchical context + ec = EasyConfig(os.path.join(ecs_dir, 'impi-4.1.3.049.eb')) + self.assertErrorRegex(EasyBuildError, 'No compiler available.*MPI lib', ActiveMNS().det_modpath_extensions, ec) os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme init_config(build_options=build_options) + test_ecs = { + 'GCC-4.7.2.eb': ('GCC/4.7.2', '', [], []), + 'OpenMPI-1.6.4-GCC-4.7.2.eb': ('OpenMPI/1.6.4-GCC-4.7.2', '', [], []), + 'gzip-1.5-goolf-1.4.10.eb': ('gzip/1.5-goolf-1.4.10', '', [], []), + 'goolf-1.4.10.eb': ('goolf/1.4.10', '', [], []), + 'impi-4.1.3.049.eb': ('impi/4.1.3.049', '', [], []), + } + for ecfile, mns_vals in test_ecs.items(): + test_ec(ecfile, *mns_vals) + +class TclModuleGeneratorTest(ModuleGeneratorTest): + """Test for module_generator module for Tcl syntax.""" + MODULE_GENERATOR_CLASS = ModuleGeneratorTcl + + +class LuaModuleGeneratorTest(ModuleGeneratorTest): + """Test for module_generator module for Tcl syntax.""" + MODULE_GENERATOR_CLASS = ModuleGeneratorLua + + def suite(): """ returns all the testcases in this module """ - return TestLoader().loadTestsFromTestCase(ModuleGeneratorTest) + suite = TestSuite() + suite.addTests(TestLoader().loadTestsFromTestCase(TclModuleGeneratorTest)) + suite.addTests(TestLoader().loadTestsFromTestCase(LuaModuleGeneratorTest)) + return suite if __name__ == '__main__': #logToScreen(enable=True) #setLogLevelDebug() - main() + TextTestRunner().run(suite()) diff --git a/test/framework/modules.py b/test/framework/modules.py index f8b96e27b2..3da4f62272 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,12 +37,16 @@ from test.framework.utilities import EnhancedTestCase, init_config from unittest import TestLoader, main +from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.easyconfig.easyconfig import EasyConfig +from easybuild.tools import config from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.modules import get_software_root, get_software_version, get_software_libdir, modules_tool +from easybuild.tools.filetools import mkdir, read_file, write_file +from easybuild.tools.modules import Lmod, get_software_root, get_software_version, get_software_libdir, modules_tool # number of modules included for testing purposes -TEST_MODULES_COUNT = 38 +TEST_MODULES_COUNT = 63 class ModulesTest(EnhancedTestCase): @@ -57,12 +61,7 @@ def init_testmods(self, test_modules_paths=None): """Initialize set of test modules for test.""" if test_modules_paths is None: test_modules_paths = [os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules'))] - mod_paths = self.testmods.mod_paths[:] - for path in test_modules_paths: - self.testmods.prepend_module_path(path) - for path in mod_paths: - if path not in test_modules_paths: - self.testmods.remove_module_path(path) + self.reset_modulepath(test_modules_paths) # for Lmod, this test has to run first, to avoid that it fails; # no modules are found if another test ran before it, but using a (very) long module path works fine interactively @@ -110,20 +109,51 @@ def test_avail(self): def test_exists(self): """Test if testing for module existence works.""" self.init_testmods() - self.assertTrue(self.testmods.exists('OpenMPI/1.6.4-GCC-4.6.4')) - self.assertTrue(not self.testmods.exists(mod_name='foo/1.2.3')) + self.assertEqual(self.testmods.exist(['OpenMPI/1.6.4-GCC-4.6.4']), [True]) + self.assertEqual(self.testmods.exist(['foo/1.2.3']), [False]) + # exists should not return True for incomplete module names + self.assertEqual(self.testmods.exist(['GCC']), [False]) + + # exists works on hidden modules + self.assertEqual(self.testmods.exist(['toy/.0.0-deps']), [True]) + + # exists works on hidden modules in Lua syntax (only with Lmod) + if isinstance(self.testmods, Lmod): + test_modules_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules')) + # make sure only the .lua module file is there, otherwise this test doesn't work as intended + self.assertTrue(os.path.exists(os.path.join(test_modules_path, 'bzip2', '.1.0.6.lua'))) + self.assertFalse(os.path.exists(os.path.join(test_modules_path, 'bzip2', '.1.0.6'))) + self.assertEqual(self.testmods.exist(['bzip2/.1.0.6']), [True]) + + # exists also works on lists of module names + # list should be sufficiently long, since for short lists 'show' is always used + mod_names = ['OpenMPI/1.6.4-GCC-4.6.4', 'foo/1.2.3', 'GCC', + 'ScaLAPACK/1.8.0-gompi-1.1.0-no-OFED', + 'ScaLAPACK/1.8.0-gompi-1.1.0-no-OFED-ATLAS-3.8.4-LAPACK-3.4.0-BLACS-1.1', + 'Compiler/GCC/4.7.2/OpenMPI/1.6.4', 'toy/.0.0-deps'] + self.assertEqual(self.testmods.exist(mod_names), [True, False, False, False, True, True, True]) def test_load(self): """ test if we load one module it is in the loaded_modules """ self.init_testmods() ms = self.testmods.available() - ms = [m for m in ms if not m.startswith('Core/') and not m.startswith('Compiler/')] + # exclude modules not on the top level of a hierarchy + ms = [m for m in ms if not (m.startswith('Core') or m.startswith('Compiler/') or m.startswith('MPI/') or + m.startswith('CategorizedHMNS'))] for m in ms: self.testmods.load([m]) self.assertTrue(m in self.testmods.loaded_modules()) self.testmods.purge() + # trying to load a module not on the top level of a hierarchy should fail + mods = [ + 'Compiler/GCC/4.7.2/OpenMPI/1.6.4', # module use on non-existent dir (Tcl-based env mods), or missing dep (Lmod) + 'MPI/GCC/4.7.2/OpenMPI/1.6.4/ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2', # missing dep + ] + for mod in mods: + self.assertErrorRegex(EasyBuildError, '.*', self.testmods.load, [mod]) + def test_ld_library_path(self): """Make sure LD_LIBRARY_PATH is what it should be when loaded multiple modules.""" self.init_testmods() @@ -217,6 +247,145 @@ def test_wrong_modulepath(self): self.assertEqual(modtool.mod_paths[1], test_modules_path) self.assertTrue(len(modtool.available()) > 0) + def test_path_to_top_of_module_tree(self): + """Test function to determine path to top of the module tree.""" + + modtool = modules_tool() + + path = modtool.path_to_top_of_module_tree([], 'gompi/1.3.12', '', ['GCC/4.6.4', 'OpenMPI/1.6.4-GCC-4.6.4']) + self.assertEqual(path, []) + path = modtool.path_to_top_of_module_tree([], 'toy/.0.0-deps', '', ['gompi/1.3.12']) + self.assertEqual(path, []) + path = modtool.path_to_top_of_module_tree([], 'toy/0.0', '', []) + self.assertEqual(path, []) + + def test_path_to_top_of_module_tree_hierarchical_mns(self): + """Test function to determine path to top of the module tree for a hierarchical module naming scheme.""" + + modtool = modules_tool() + + ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs') + all_stops = [x[0] for x in EasyBlock.get_steps()] + build_options = { + 'check_osdeps': False, + 'robot_path': [ecs_dir], + 'valid_stops': all_stops, + 'validate': False, + } + os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS' + init_config(build_options=build_options) + self.setup_hierarchical_modules() + modtool = modules_tool() + mod_prefix = os.path.join(self.test_installpath, 'modules', 'all') + init_modpaths = [os.path.join(mod_prefix, 'Core')] + + deps = ['GCC/4.7.2', 'OpenMPI/1.6.4', 'FFTW/3.3.3', 'OpenBLAS/0.2.6-LAPACK-3.4.2', + 'ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2'] + path = modtool.path_to_top_of_module_tree(init_modpaths, 'goolf/1.4.10', os.path.join(mod_prefix, 'Core'), deps) + self.assertEqual(path, []) + path = modtool.path_to_top_of_module_tree(init_modpaths, 'GCC/4.7.2', os.path.join(mod_prefix, 'Core'), []) + self.assertEqual(path, []) + full_mod_subdir = os.path.join(mod_prefix, 'Compiler', 'GCC', '4.7.2') + deps = ['GCC/4.7.2', 'hwloc/1.6.2'] + path = modtool.path_to_top_of_module_tree(init_modpaths, 'OpenMPI/1.6.4', full_mod_subdir, deps) + self.assertEqual(path, ['GCC/4.7.2']) + full_mod_subdir = os.path.join(mod_prefix, 'MPI', 'GCC', '4.7.2', 'OpenMPI', '1.6.4') + deps = ['GCC/4.7.2', 'OpenMPI/1.6.4'] + path = modtool.path_to_top_of_module_tree(init_modpaths, 'FFTW/3.3.3', full_mod_subdir, deps) + self.assertEqual(path, ['OpenMPI/1.6.4', 'GCC/4.7.2']) + + def test_path_to_top_of_module_tree_categorized_hmns(self): + """ + Test function to determine path to top of the module tree for a categorized hierarchical module naming + scheme. + """ + + ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs') + all_stops = [x[0] for x in EasyBlock.get_steps()] + build_options = { + 'check_osdeps': False, + 'robot_path': [ecs_dir], + 'valid_stops': all_stops, + 'validate': False, + } + os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS' + init_config(build_options=build_options) + self.setup_categorized_hmns_modules() + modtool = modules_tool() + mod_prefix = os.path.join(self.test_installpath, 'modules', 'all') + init_modpaths = [os.path.join(mod_prefix, 'Core', 'compiler'), os.path.join(mod_prefix, 'Core', 'toolchain')] + + deps = ['GCC/4.7.2', 'OpenMPI/1.6.4', 'FFTW/3.3.3', 'OpenBLAS/0.2.6-LAPACK-3.4.2', + 'ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2'] + path = modtool.path_to_top_of_module_tree(init_modpaths, 'goolf/1.4.10', os.path.join(mod_prefix, 'Core', 'toolchain'), deps) + self.assertEqual(path, []) + path = modtool.path_to_top_of_module_tree(init_modpaths, 'GCC/4.7.2', os.path.join(mod_prefix, 'Core', 'compiler'), []) + self.assertEqual(path, []) + full_mod_subdir = os.path.join(mod_prefix, 'Compiler', 'GCC', '4.7.2', 'mpi') + deps = ['GCC/4.7.2', 'hwloc/1.6.2'] + path = modtool.path_to_top_of_module_tree(init_modpaths, 'OpenMPI/1.6.4', full_mod_subdir, deps) + self.assertEqual(path, ['GCC/4.7.2']) + full_mod_subdir = os.path.join(mod_prefix, 'MPI', 'GCC', '4.7.2', 'OpenMPI', '1.6.4', 'numlib') + deps = ['GCC/4.7.2', 'OpenMPI/1.6.4'] + path = modtool.path_to_top_of_module_tree(init_modpaths, 'FFTW/3.3.3', full_mod_subdir, deps) + self.assertEqual(path, ['OpenMPI/1.6.4', 'GCC/4.7.2']) + + def test_modules_tool_stateless(self): + """Check whether ModulesTool instance is stateless between runs.""" + test_modules_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules') + + # copy test Core/Compiler modules, we need to rewrite the 'module use' statement in the one we're going to load + shutil.copytree(os.path.join(test_modules_path, 'Core'), os.path.join(self.test_prefix, 'Core')) + shutil.copytree(os.path.join(test_modules_path, 'Compiler'), os.path.join(self.test_prefix, 'Compiler')) + + modtxt = read_file(os.path.join(self.test_prefix, 'Core', 'GCC', '4.7.2')) + modpath_extension = os.path.join(self.test_prefix, 'Compiler', 'GCC', '4.7.2') + modtxt = re.sub('module use .*', 'module use %s' % modpath_extension, modtxt, re.M) + write_file(os.path.join(self.test_prefix, 'Core', 'GCC', '4.7.2'), modtxt) + + modtxt = read_file(os.path.join(self.test_prefix, 'Compiler', 'GCC', '4.7.2', 'OpenMPI', '1.6.4')) + modpath_extension = os.path.join(self.test_prefix, 'MPI', 'GCC', '4.7.2', 'OpenMPI', '1.6.4') + mkdir(modpath_extension, parents=True) + modtxt = re.sub('module use .*', 'module use %s' % modpath_extension, modtxt, re.M) + write_file(os.path.join(self.test_prefix, 'Compiler', 'GCC', '4.7.2', 'OpenMPI', '1.6.4'), modtxt) + + # force reset of any singletons by reinitiating config + init_config() + + os.environ['MODULEPATH'] = os.path.join(self.test_prefix, 'Core') + modtool = modules_tool() + + if isinstance(modtool, Lmod): + load_err_msg = "cannot[\s\n]*be[\s\n]*loaded" + else: + load_err_msg = "Unable to locate a modulefile" + + # GCC/4.6.3 is *not* an available Core module + self.assertErrorRegex(EasyBuildError, load_err_msg, modtool.load, ['GCC/4.6.3']) + + # GCC/4.7.2 is one of the available Core modules + modtool.load(['GCC/4.7.2']) + + # OpenMPI/1.6.4 becomes available after loading GCC/4.7.2 module + modtool.load(['OpenMPI/1.6.4']) + modtool.purge() + + # reset $MODULEPATH, obtain new ModulesTool instance, + # which should not remember anything w.r.t. previous $MODULEPATH value + os.environ['MODULEPATH'] = test_modules_path + modtool = modules_tool() + + # GCC/4.6.3 is available + modtool.load(['GCC/4.6.3']) + modtool.purge() + + # GCC/4.7.2 is available (note: also as non-Core module outside of hierarchy) + modtool.load(['GCC/4.7.2']) + + # OpenMPI/1.6.4 is *not* available with current $MODULEPATH (loaded GCC/4.7.2 was not a hierarchical module) + self.assertErrorRegex(EasyBuildError, load_err_msg, modtool.load, ['OpenMPI/1.6.4']) + + def suite(): """ returns all the testcases in this module """ return TestLoader().loadTestsFromTestCase(ModulesTest) diff --git a/test/framework/modules/CategorizedHMNS/Compiler/GCC/4.7.2/mpi/OpenMPI/1.6.4 b/test/framework/modules/CategorizedHMNS/Compiler/GCC/4.7.2/mpi/OpenMPI/1.6.4 new file mode 100644 index 0000000000..a851535416 --- /dev/null +++ b/test/framework/modules/CategorizedHMNS/Compiler/GCC/4.7.2/mpi/OpenMPI/1.6.4 @@ -0,0 +1,30 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { The Open MPI Project is an open source MPI-2 implementation. - Homepage: http://www.open-mpi.org/ + } +} + +module-whatis {Description: The Open MPI Project is an open source MPI-2 implementation. - Homepage: http://www.open-mpi.org/} + +set root /tmp/software/Compiler/GCC/4.7.2/mpi/OpenMPI/1.6.4-no-OFED + +conflict OpenMPI +module use /tmp/modules/all/MPI/GCC/4.7.2/OpenMPI/1.6.4/base +module use /tmp/modules/all/MPI/GCC/4.7.2/OpenMPI/1.6.4/numlib + +if { ![is-loaded hwloc/1.6.2] } { + module load hwloc/1.6.2 +} + +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig + +setenv EBROOTOPENMPI "$root" +setenv EBVERSIONOPENMPI "1.6.4" +setenv EBDEVELOPENMPI "$root/easybuild/Compiler-GCC-4.7.2-OpenMPI-1.6.4-easybuild-devel" + diff --git a/test/framework/modules/CategorizedHMNS/Compiler/GCC/4.7.2/system/hwloc/1.6.2 b/test/framework/modules/CategorizedHMNS/Compiler/GCC/4.7.2/system/hwloc/1.6.2 new file mode 100644 index 0000000000..18dc7a1cc8 --- /dev/null +++ b/test/framework/modules/CategorizedHMNS/Compiler/GCC/4.7.2/system/hwloc/1.6.2 @@ -0,0 +1,34 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { The Portable Hardware Locality (hwloc) software package provides a portable abstraction + (across OS, versions, architectures, ...) of the hierarchical topology of modern architectures, including + NUMA memory nodes, sockets, shared caches, cores and simultaneous multithreading. It also gathers various + system attributes such as cache and memory information as well as the locality of I/O devices such as + network interfaces, InfiniBand HCAs or GPUs. It primarily aims at helping applications with gathering + information about modern computing hardware so as to exploit it accordingly and efficiently. - Homepage: http://www.open-mpi.org/projects/hwloc/ + } +} + +module-whatis {Description: The Portable Hardware Locality (hwloc) software package provides a portable abstraction + (across OS, versions, architectures, ...) of the hierarchical topology of modern architectures, including + NUMA memory nodes, sockets, shared caches, cores and simultaneous multithreading. It also gathers various + system attributes such as cache and memory information as well as the locality of I/O devices such as + network interfaces, InfiniBand HCAs or GPUs. It primarily aims at helping applications with gathering + information about modern computing hardware so as to exploit it accordingly and efficiently. - Homepage: http://www.open-mpi.org/projects/hwloc/} + +set root /tmp/software/Compiler/GCC/4.7.2/system/hwloc/1.6.2 + +conflict hwloc + +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig + +setenv EBROOTHWLOC "$root" +setenv EBVERSIONHWLOC "1.6.2" +setenv EBDEVELHWLOC "$root/easybuild/Compiler-GCC-4.7.2-hwloc-1.6.2-easybuild-devel" + diff --git a/test/framework/modules/CategorizedHMNS/Core/compiler/GCC/4.7.2 b/test/framework/modules/CategorizedHMNS/Core/compiler/GCC/4.7.2 new file mode 100644 index 0000000000..7e6aa96a08 --- /dev/null +++ b/test/framework/modules/CategorizedHMNS/Core/compiler/GCC/4.7.2 @@ -0,0 +1,28 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...). - Homepage: http://gcc.gnu.org/ + } +} + +module-whatis {Description: The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...). - Homepage: http://gcc.gnu.org/} + +set root /tmp/software/Core/compiler/GCC/4.7.2 + +conflict GCC +module use /tmp/modules/all/Compiler/GCC/4.7.2/mpi +module use /tmp/modules/all/Compiler/GCC/4.7.2/system + +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LD_LIBRARY_PATH $root/lib/gcc/x86_64-apple-darwin13.2.0/4.7.2 +prepend-path LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin + +setenv EBROOTGCC "$root" +setenv EBVERSIONGCC "4.7.2" +setenv EBDEVELGCC "$root/easybuild/Core-GCC-4.7.2-easybuild-devel" + diff --git a/test/framework/modules/CategorizedHMNS/Core/toolchain/gompi/1.4.10 b/test/framework/modules/CategorizedHMNS/Core/toolchain/gompi/1.4.10 new file mode 100644 index 0000000000..20bdcdcf44 --- /dev/null +++ b/test/framework/modules/CategorizedHMNS/Core/toolchain/gompi/1.4.10 @@ -0,0 +1,28 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { GNU Compiler Collection (GCC) based compiler toolchain, + including OpenMPI for MPI support. - Homepage: (none) + } +} + +module-whatis {Description: GNU Compiler Collection (GCC) based compiler toolchain, + including OpenMPI for MPI support. - Homepage: (none)} + +set root /tmp/software/Core/toolchain/gompi/1.4.10 + +conflict gompi + +if { ![is-loaded GCC/4.7.2] } { + module load GCC/4.7.2 +} + +if { ![is-loaded OpenMPI/1.6.4] } { + module load OpenMPI/1.6.4 +} + + +setenv EBROOTGOMPI "$root" +setenv EBVERSIONGOMPI "1.4.10" +setenv EBDEVELGOMPI "$root/easybuild/Core-gompi-1.4.10-easybuild-devel" + diff --git a/test/framework/modules/CategorizedHMNS/Core/toolchain/goolf/1.4.10 b/test/framework/modules/CategorizedHMNS/Core/toolchain/goolf/1.4.10 new file mode 100644 index 0000000000..9c3483696f --- /dev/null +++ b/test/framework/modules/CategorizedHMNS/Core/toolchain/goolf/1.4.10 @@ -0,0 +1,39 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { GNU Compiler Collection (GCC) based compiler toolchain, + including OpenMPI for MPI support. - Homepage: (none) + } +} + +module-whatis {Description: GNU Compiler Collection (GCC) based compiler toolchain, + including OpenMPI for MPI support. - Homepage: (none)} + +set root /tmp/software/Core/toolchain/gompi/1.4.10 + +conflict gompi + +if { ![is-loaded GCC/4.7.2] } { + module load GCC/4.7.2 +} + +if { ![is-loaded OpenMPI/1.6.4] } { + module load OpenMPI/1.6.4 +} + +if { ![is-loaded FFTW/3.3.3] } { + module load FFTW/3.3.3 +} + +if { ![is-loaded OpenBLAS/0.2.6-LAPACK-3.4.2] } { + module load OpenBLAS/0.2.6-LAPACK-3.4.2 +} + +if { ![is-loaded ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2] } { + module load ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2 +} + +setenv EBROOTGOMPI "$root" +setenv EBVERSIONGOMPI "1.4.10" +setenv EBDEVELGOMPI "$root/easybuild/Core-gompi-1.4.10-easybuild-devel" + diff --git a/test/framework/modules/CategorizedHMNS/MPI/GCC/4.7.2/OpenMPI/1.6.4/numlib/FFTW/3.3.3 b/test/framework/modules/CategorizedHMNS/MPI/GCC/4.7.2/OpenMPI/1.6.4/numlib/FFTW/3.3.3 new file mode 100644 index 0000000000..9590348bd7 --- /dev/null +++ b/test/framework/modules/CategorizedHMNS/MPI/GCC/4.7.2/OpenMPI/1.6.4/numlib/FFTW/3.3.3 @@ -0,0 +1,27 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data. - Homepage: http://www.fftw.org +} +} + +module-whatis {FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data. - Homepage: http://www.fftw.org} + +set root /tmp/software/MPI/GCC/4.7.2/OpenMPI/1.6.4/numlib/FFTW/3.3.3 + +conflict FFTW + +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig + +setenv EBROOTFFTW "$root" +setenv EBVERSIONFFTW "3.3.3" +setenv EBDEVELFFTW "$root/easybuild/FFTW-3.3.3-gompi-1.4.10-easybuild-devel" + + +# built with EasyBuild version 1.4.0dev diff --git a/test/framework/modules/CategorizedHMNS/MPI/GCC/4.7.2/OpenMPI/1.6.4/numlib/OpenBLAS/0.2.6-LAPACK-3.4.2 b/test/framework/modules/CategorizedHMNS/MPI/GCC/4.7.2/OpenMPI/1.6.4/numlib/OpenBLAS/0.2.6-LAPACK-3.4.2 new file mode 100644 index 0000000000..07b044ec1a --- /dev/null +++ b/test/framework/modules/CategorizedHMNS/MPI/GCC/4.7.2/OpenMPI/1.6.4/numlib/OpenBLAS/0.2.6-LAPACK-3.4.2 @@ -0,0 +1,22 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version. - Homepage: http://xianyi.github.com/OpenBLAS/ +} +} + +module-whatis {OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version. - Homepage: http://xianyi.github.com/OpenBLAS/} + +set root /tmp/software/MPI/GCC/4.7.2/OpenMPI/1.6.4/numlib/OpenBLAS/0.2.6-LAPACK-3.4.2 + +conflict OpenBLAS + +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib + +setenv EBROOTOPENBLAS "$root" +setenv EBVERSIONOPENBLAS "0.2.6" +setenv EBDEVELOPENBLAS "$root/easybuild/OpenBLAS-0.2.6-gompi-1.4.10-LAPACK-3.4.2-easybuild-devel" + + +# built with EasyBuild version 1.4.0dev diff --git a/test/framework/modules/CategorizedHMNS/MPI/GCC/4.7.2/OpenMPI/1.6.4/numlib/ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2 b/test/framework/modules/CategorizedHMNS/MPI/GCC/4.7.2/OpenMPI/1.6.4/numlib/ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2 new file mode 100644 index 0000000000..4a93cd6ef5 --- /dev/null +++ b/test/framework/modules/CategorizedHMNS/MPI/GCC/4.7.2/OpenMPI/1.6.4/numlib/ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2 @@ -0,0 +1,28 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { The ScaLAPACK (or Scalable LAPACK) library includes a subset of LAPACK routines +redesigned for distributed memory MIMD parallel computers. - Homepage: http://www.netlib.org/scalapack/ +} +} + +module-whatis {The ScaLAPACK (or Scalable LAPACK) library includes a subset of LAPACK routines +redesigned for distributed memory MIMD parallel computers. - Homepage: http://www.netlib.org/scalapack/} + +set root /tmp/software/MPI/GCC/4.7.2/OpenMPI/1.6.4/numlib/ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2 + +conflict ScaLAPACK + +if { ![is-loaded OpenBLAS/0.2.6-LAPACK-3.4.2] } { + module load OpenBLAS/0.2.6-LAPACK-3.4.2 +} + +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib + +setenv EBROOTSCALAPACK "$root" +setenv EBVERSIONSCALAPACK "2.0.2" +setenv EBDEVELSCALAPACK "$root/easybuild/ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2-easybuild-devel" + + +# built with EasyBuild version 1.4.0dev diff --git a/test/framework/modules/Compiler/intel/2013.5.192-GCC-4.8.3/impi/4.1.3.049 b/test/framework/modules/Compiler/intel/2013.5.192-GCC-4.8.3/impi/4.1.3.049 new file mode 100644 index 0000000000..6b65daae4f --- /dev/null +++ b/test/framework/modules/Compiler/intel/2013.5.192-GCC-4.8.3/impi/4.1.3.049 @@ -0,0 +1,30 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { The Intel(R) MPI Library for Linux* OS is a multi-fabric message + passing library based on ANL MPICH2 and OSU MVAPICH2. The Intel MPI Library for + Linux OS implements the Message Passing Interface, version 2 (MPI-2) specification. - Homepage: http://software.intel.com/en-us/intel-mpi-library/ + } +} + +module-whatis {Description: The Intel(R) MPI Library for Linux* OS is a multi-fabric message + passing library based on ANL MPICH2 and OSU MVAPICH2. The Intel MPI Library for + Linux OS implements the Message Passing Interface, version 2 (MPI-2) specification. - Homepage: http://software.intel.com/en-us/intel-mpi-library/} + +set root /tmp/software/Compiler/intel/2013.5.192/impi/4.1.3.049 + +conflict impi +module use /tmp/modules/all/MPI/intel/2013.5.192/impi/4.1.3.049 +prepend-path CPATH $root/include64 +prepend-path LD_LIBRARY_PATH $root/lib64 +prepend-path LIBRARY_PATH $root/lib64 +prepend-path PATH $root/bin64 + +setenv EBROOTIMPI "$root" +setenv EBVERSIONIMPI "4.1.3.049" +setenv EBDEVELIMPI "$root/easybuild/Compiler-intel-2013.5.192-impi-4.1.3.049-easybuild-devel" + +prepend-path INTEL_LICENSE_FILE /tmp/license.lic +setenv I_MPI_ROOT $root + +# Built with EasyBuild version 1.16.0dev diff --git a/test/framework/modules/Core/GCC/4.8.3 b/test/framework/modules/Core/GCC/4.8.3 new file mode 100644 index 0000000000..4ddfb7dc18 --- /dev/null +++ b/test/framework/modules/Core/GCC/4.8.3 @@ -0,0 +1,30 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...). - Homepage: http://gcc.gnu.org/ + } +} + +module-whatis {Description: The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...). - Homepage: http://gcc.gnu.org/} + +set root /tmp/software/Core/GCC/4.8.3 + +conflict GCC +module use /tmp/modules/all/Compiler/GCC/4.8.3 +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LD_LIBRARY_PATH $root/lib64 +prepend-path LD_LIBRARY_PATH $root/lib/gcc/x86_64-unknown-linux-gnu/4.8.3 +prepend-path LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib64 +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin + +setenv EBROOTGCC "$root" +setenv EBVERSIONGCC "4.8.3" +setenv EBDEVELGCC "$root/easybuild/Core-GCC-4.8.3-easybuild-devel" + + +# Built with EasyBuild version 1.16.0dev diff --git a/test/framework/modules/Core/goolf/1.4.10 b/test/framework/modules/Core/goolf/1.4.10 new file mode 100644 index 0000000000..b199b678fc --- /dev/null +++ b/test/framework/modules/Core/goolf/1.4.10 @@ -0,0 +1,39 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { GNU Compiler Collection (GCC) based compiler toolchain, + including OpenMPI for MPI support. - Homepage: (none) + } +} + +module-whatis {Description: GNU Compiler Collection (GCC) based compiler toolchain, + including OpenMPI for MPI support. - Homepage: (none)} + +set root /tmp/software/Core/gompi/1.4.10 + +conflict gompi + +if { ![is-loaded GCC/4.7.2] } { + module load GCC/4.7.2 +} + +if { ![is-loaded OpenMPI/1.6.4] } { + module load OpenMPI/1.6.4 +} + +if { ![is-loaded FFTW/3.3.3] } { + module load FFTW/3.3.3 +} + +if { ![is-loaded OpenBLAS/0.2.6-LAPACK-3.4.2] } { + module load OpenBLAS/0.2.6-LAPACK-3.4.2 +} + +if { ![is-loaded ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2] } { + module load ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2 +} + +setenv EBROOTGOMPI "$root" +setenv EBVERSIONGOMPI "1.4.10" +setenv EBDEVELGOMPI "$root/easybuild/Core-gompi-1.4.10-easybuild-devel" + diff --git a/test/framework/modules/Core/icc/2013.5.192-GCC-4.8.3 b/test/framework/modules/Core/icc/2013.5.192-GCC-4.8.3 new file mode 100644 index 0000000000..4428aa9709 --- /dev/null +++ b/test/framework/modules/Core/icc/2013.5.192-GCC-4.8.3 @@ -0,0 +1,34 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { C and C++ compiler from Intel - Homepage: http://software.intel.com/en-us/intel-compilers/ + } +} + +module-whatis {Description: C and C++ compiler from Intel - Homepage: http://software.intel.com/en-us/intel-compilers/} + +set root /tmp/software/Core/icc/2013.5.192-GCC-4.8.3 + +conflict icc +module use /tmp/modules/all/Compiler/intel/2013.5.192-GCC-4.8.3 + +if { ![is-loaded GCC/4.8.3] } { + module load GCC/4.8.3 +} + +prepend-path IDB_HOME $root/bin/intel64 +prepend-path LD_LIBRARY_PATH $root/compiler/lib +prepend-path LD_LIBRARY_PATH $root/compiler/lib/intel64 +prepend-path MANPATH $root/man +prepend-path MANPATH $root/man/en_US +prepend-path PATH $root/bin +prepend-path PATH $root/bin/intel64 + +setenv EBROOTICC "$root" +setenv EBVERSIONICC "2013.5.192" +setenv EBDEVELICC "$root/easybuild/Core-icc-2013.5.192-GCC-4.8.3-easybuild-devel" + +prepend-path INTEL_LICENSE_FILE /tmp/license.lic +prepend-path NLSPATH $root/idb/intel64/locale/%l_%t/%N + +# Built with EasyBuild version 1.16.0dev diff --git a/test/framework/modules/Core/iccifort/2013.5.192-GCC-4.8.3 b/test/framework/modules/Core/iccifort/2013.5.192-GCC-4.8.3 new file mode 100644 index 0000000000..2375404a69 --- /dev/null +++ b/test/framework/modules/Core/iccifort/2013.5.192-GCC-4.8.3 @@ -0,0 +1,28 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { Intel C, C++ and Fortran compilers - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/ + } +} + +module-whatis {Description: Intel C, C++ and Fortran compilers - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/} + +set root /tmp/software/Core/iccifort/2013.5.192-GCC-4.8.3 + +conflict iccifort + +if { ![is-loaded icc/2013.5.192-GCC-4.8.3] } { + module load icc/2013.5.192-GCC-4.8.3 +} + +if { ![is-loaded ifort/2013.5.192-GCC-4.8.3] } { + module load ifort/2013.5.192-GCC-4.8.3 +} + + +setenv EBROOTICCIFORT "$root" +setenv EBVERSIONICCIFORT "2013.5.192" +setenv EBDEVELICCIFORT "$root/easybuild/Core-iccifort-2013.5.192-GCC-4.8.3-easybuild-devel" + + +# Built with EasyBuild version 1.16.0dev diff --git a/test/framework/modules/Core/ifort/2013.5.192-GCC-4.8.3 b/test/framework/modules/Core/ifort/2013.5.192-GCC-4.8.3 new file mode 100644 index 0000000000..750f0a4df9 --- /dev/null +++ b/test/framework/modules/Core/ifort/2013.5.192-GCC-4.8.3 @@ -0,0 +1,34 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { Fortran compiler from Intel - Homepage: http://software.intel.com/en-us/intel-compilers/ + } +} + +module-whatis {Description: Fortran compiler from Intel - Homepage: http://software.intel.com/en-us/intel-compilers/} + +set root /tmp/software/Core/ifort/2013.5.192-GCC-4.8.3 + +conflict ifort +module use /tmp/modules/all/Compiler/intel/2013.5.192-GCC-4.8.3 + +if { ![is-loaded GCC/4.8.3] } { + module load GCC/4.8.3 +} + +prepend-path IDB_HOME $root/bin/intel64 +prepend-path LD_LIBRARY_PATH $root/compiler/lib +prepend-path LD_LIBRARY_PATH $root/compiler/lib/intel64 +prepend-path MANPATH $root/man +prepend-path MANPATH $root/man/en_US +prepend-path PATH $root/bin +prepend-path PATH $root/bin/intel64 + +setenv EBROOTIFORT "$root" +setenv EBVERSIONIFORT "2013.5.192" +setenv EBDEVELIFORT "$root/easybuild/Core-ifort-2013.5.192-GCC-4.8.3-easybuild-devel" + +prepend-path INTEL_LICENSE_FILE /tmp/license.lic +prepend-path NLSPATH $root/idb/intel64/locale/%l_%t/%N + +# Built with EasyBuild version 1.16.0dev diff --git a/test/framework/modules/Core/iimpi/5.5.3-GCC-4.8.3 b/test/framework/modules/Core/iimpi/5.5.3-GCC-4.8.3 new file mode 100644 index 0000000000..3bb0b860cb --- /dev/null +++ b/test/framework/modules/Core/iimpi/5.5.3-GCC-4.8.3 @@ -0,0 +1,32 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { Intel C/C++ and Fortran compilers, alongside Intel MPI. - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/ + } +} + +module-whatis {Description: Intel C/C++ and Fortran compilers, alongside Intel MPI. - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/} + +set root /tmp/software/Core/iimpi/5.5.3-GCC-4.8.3 + +conflict iimpi + +if { ![is-loaded icc/2013.5.192-GCC-4.8.3] } { + module load icc/2013.5.192-GCC-4.8.3 +} + +if { ![is-loaded ifort/2013.5.192-GCC-4.8.3] } { + module load ifort/2013.5.192-GCC-4.8.3 +} + +if { ![is-loaded impi/4.1.3.049] } { + module load impi/4.1.3.049 +} + + +setenv EBROOTIIMPI "$root" +setenv EBVERSIONIIMPI "5.5.3" +setenv EBDEVELIIMPI "$root/easybuild/Core-iimpi-5.5.3-GCC-4.8.3-easybuild-devel" + + +# Built with EasyBuild version 1.16.0dev diff --git a/test/framework/modules/MPI/GCC/4.7.2/OpenMPI/1.6.4/FFTW/3.3.3 b/test/framework/modules/MPI/GCC/4.7.2/OpenMPI/1.6.4/FFTW/3.3.3 new file mode 100644 index 0000000000..f5558adc35 --- /dev/null +++ b/test/framework/modules/MPI/GCC/4.7.2/OpenMPI/1.6.4/FFTW/3.3.3 @@ -0,0 +1,27 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data. - Homepage: http://www.fftw.org +} +} + +module-whatis {FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data. - Homepage: http://www.fftw.org} + +set root /home-2/khoste/.local/easybuild/software/MPI/GCC/4.7.2/OpenMPI/1.6.4/FFTW/3.3.3 + +conflict FFTW + +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig + +setenv EBROOTFFTW "$root" +setenv EBVERSIONFFTW "3.3.3" +setenv EBDEVELFFTW "$root/easybuild/FFTW-3.3.3-gompi-1.4.10-easybuild-devel" + + +# built with EasyBuild version 1.4.0dev diff --git a/test/framework/modules/MPI/GCC/4.7.2/OpenMPI/1.6.4/OpenBLAS/0.2.6-LAPACK-3.4.2 b/test/framework/modules/MPI/GCC/4.7.2/OpenMPI/1.6.4/OpenBLAS/0.2.6-LAPACK-3.4.2 new file mode 100644 index 0000000000..f81a3b5d44 --- /dev/null +++ b/test/framework/modules/MPI/GCC/4.7.2/OpenMPI/1.6.4/OpenBLAS/0.2.6-LAPACK-3.4.2 @@ -0,0 +1,22 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version. - Homepage: http://xianyi.github.com/OpenBLAS/ +} +} + +module-whatis {OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version. - Homepage: http://xianyi.github.com/OpenBLAS/} + +set root /home-2/khoste/.local/easybuild/software/MPI/GCC/4.7.2/OpenMPI/1.6.4/OpenBLAS/0.2.6-LAPACK-3.4.2 + +conflict OpenBLAS + +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib + +setenv EBROOTOPENBLAS "$root" +setenv EBVERSIONOPENBLAS "0.2.6" +setenv EBDEVELOPENBLAS "$root/easybuild/OpenBLAS-0.2.6-gompi-1.4.10-LAPACK-3.4.2-easybuild-devel" + + +# built with EasyBuild version 1.4.0dev diff --git a/test/framework/modules/MPI/GCC/4.7.2/OpenMPI/1.6.4/ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2 b/test/framework/modules/MPI/GCC/4.7.2/OpenMPI/1.6.4/ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2 new file mode 100644 index 0000000000..31166dc2f3 --- /dev/null +++ b/test/framework/modules/MPI/GCC/4.7.2/OpenMPI/1.6.4/ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2 @@ -0,0 +1,28 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { The ScaLAPACK (or Scalable LAPACK) library includes a subset of LAPACK routines +redesigned for distributed memory MIMD parallel computers. - Homepage: http://www.netlib.org/scalapack/ +} +} + +module-whatis {The ScaLAPACK (or Scalable LAPACK) library includes a subset of LAPACK routines +redesigned for distributed memory MIMD parallel computers. - Homepage: http://www.netlib.org/scalapack/} + +set root /home-2/khoste/.local/easybuild/software/MPI/GCC/4.7.2/OpenMPI/1.6.4/ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2 + +conflict ScaLAPACK + +if { ![is-loaded OpenBLAS/0.2.6-LAPACK-3.4.2] } { + module load OpenBLAS/0.2.6-LAPACK-3.4.2 +} + +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib + +setenv EBROOTSCALAPACK "$root" +setenv EBVERSIONSCALAPACK "2.0.2" +setenv EBDEVELSCALAPACK "$root/easybuild/ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2-easybuild-devel" + + +# built with EasyBuild version 1.4.0dev diff --git a/test/framework/modules/bzip2/.1.0.6.lua b/test/framework/modules/bzip2/.1.0.6.lua new file mode 100644 index 0000000000..cad0e55a36 --- /dev/null +++ b/test/framework/modules/bzip2/.1.0.6.lua @@ -0,0 +1,24 @@ +help([[bzip2 is a freely available, patent free, high-quality data compressor. It typically +compresses files to within 10% to 15% of the best available techniques (the PPM family of statistical +compressors), whilst being around twice as fast at compression and six times faster at decompression. - Homepage: http://www.bzip.org/]]) +whatis([[Name: bzip2]]) +whatis([[Version: 1.0.6]]) +whatis([[Description: bzip2 is a freely available, patent free, high-quality data compressor. It typically +compresses files to within 10% to 15% of the best available techniques (the PPM family of statistical +compressors), whilst being around twice as fast at compression and six times faster at decompression. - Homepage: http://www.bzip.org/]]) +whatis([[Homepage: http://www.bzip.org/]]) + +local root = "/Users/example/.local/easybuild/software/bzip2/1.0.6" + +conflict("bzip2") + +prepend_path("CPATH", pathJoin(root, "include")) +prepend_path("LD_LIBRARY_PATH", pathJoin(root, "lib")) +prepend_path("LIBRARY_PATH", pathJoin(root, "lib")) +prepend_path("MANPATH", pathJoin(root, "man")) +prepend_path("PATH", pathJoin(root, "bin")) +setenv("EBROOTBZIP2", root) +setenv("EBVERSIONBZIP2", "1.0.6") +setenv("EBDEVELBZIP2", pathJoin(root, "easybuild/bzip2-1.0.6-easybuild-devel")) + +-- Built with EasyBuild version 2.1.1 diff --git a/test/framework/modules/cgompi/1.1.6 b/test/framework/modules/cgompi/1.1.6 index 22bd608d51..33367e0a50 100644 --- a/test/framework/modules/cgompi/1.1.6 +++ b/test/framework/modules/cgompi/1.1.6 @@ -13,8 +13,12 @@ set root /user/scratch/gent/vsc400/vsc40023/easybuild_REGTEST/SL6/sandybridge conflict cgompi -if { ![is-loaded ClangGCC/1.1.2] } { - module load ClangGCC/1.1.2 +if { ![is-loaded GCC/4.7.2] } { + module load GCC/4.7.2 +} + +if { ![is-loaded Clang/3.2-GCC-4.7.2] } { + module load Clang/3.2-GCC-4.7.2 } if { ![is-loaded OpenMPI/1.6.4-ClangGCC-1.1.2] } { diff --git a/test/framework/modules/cgoolf/1.1.6 b/test/framework/modules/cgoolf/1.1.6 index 477da80d19..7ce5aa2cc0 100644 --- a/test/framework/modules/cgoolf/1.1.6 +++ b/test/framework/modules/cgoolf/1.1.6 @@ -13,8 +13,12 @@ set root /user/scratch/gent/vsc400/vsc40023/easybuild_REGTEST/SL6/sandybridge conflict cgoolf -if { ![is-loaded ClangGCC/1.1.2] } { - module load ClangGCC/1.1.2 +if { ![is-loaded GCC/4.7.2] } { + module load GCC/4.7.2 +} + +if { ![is-loaded Clang/3.2-GCC-4.7.2] } { + module load Clang/3.2-GCC-4.7.2 } if { ![is-loaded OpenMPI/1.6.4-ClangGCC-1.1.2] } { diff --git a/test/framework/modules/goalf/1.1.0-no-OFED-brokenBLACS b/test/framework/modules/goalf/1.1.0-no-OFED-brokenBLACS new file mode 100644 index 0000000000..daaa81cb83 --- /dev/null +++ b/test/framework/modules/goalf/1.1.0-no-OFED-brokenBLACS @@ -0,0 +1,48 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { GNU Compiler Collection (GCC) based compiler toolchain, including +OpenMPI for MPI support, ATLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. - Homepage: (none) +} +} + +module-whatis {GNU Compiler Collection (GCC) based compiler toolchain, including +OpenMPI for MPI support, ATLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. - Homepage: (none)} + +set root /home/kehoste/.local/easybuild/software/goalf/1.1.0-no-OFED-noblacs + +conflict goalf + +if { ![is-loaded GCC/4.6.3] } { + module load GCC/4.6.3 +} + +if { ![is-loaded OpenMPI/1.4.5-GCC-4.6.3-no-OFED] } { + module load OpenMPI/1.4.5-GCC-4.6.3-no-OFED +} + +if { ![is-loaded ATLAS/3.8.4-gompi-1.1.0-no-OFED-LAPACK-3.4.0] } { + module load ATLAS/3.8.4-gompi-1.1.0-no-OFED-LAPACK-3.4.0 +} + +if { ![is-loaded FFTW/3.3.1-gompi-1.1.0-no-OFED] } { + module load FFTW/3.3.1-gompi-1.1.0-no-OFED +} + +# load statement for BLACS purposely omitted, which doesn't match the toolchain definition +# BLACS is optional, but still required here since ScaLAPACK version is < 2.0 +#if { ![is-loaded BLACS/1.1-gompi-1.1.0-no-OFED] } { +# module load BLACS/1.1-gompi-1.1.0-no-OFED +#} + +if { ![is-loaded ScaLAPACK/1.8.0-gompi-1.1.0-no-OFED-ATLAS-3.8.4-LAPACK-3.4.0-BLACS-1.1] } { + module load ScaLAPACK/1.8.0-gompi-1.1.0-no-OFED-ATLAS-3.8.4-LAPACK-3.4.0-BLACS-1.1 +} + + +setenv EBROOTGOALF $root +setenv EBVERSIONGOALF 1.1.0 +setenv EBDEVELGOALF $root/easybuild/goalf-1.1.0-no-OFED-noblacs-easybuild-devel + + +# built with EasyBuild version 0.9dev diff --git a/test/framework/modules/goalf/1.1.0-no-OFED-brokenFFTW b/test/framework/modules/goalf/1.1.0-no-OFED-brokenFFTW new file mode 100644 index 0000000000..07d9cac1b6 --- /dev/null +++ b/test/framework/modules/goalf/1.1.0-no-OFED-brokenFFTW @@ -0,0 +1,47 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { GNU Compiler Collection (GCC) based compiler toolchain, including +OpenMPI for MPI support, ATLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. - Homepage: (none) +} +} + +module-whatis {GNU Compiler Collection (GCC) based compiler toolchain, including +OpenMPI for MPI support, ATLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. - Homepage: (none)} + +set root /home/kehoste/.local/easybuild/software/goalf/1.1.0-no-OFED-broken + +conflict goalf + +if { ![is-loaded GCC/4.6.3] } { + module load GCC/4.6.3 +} + +if { ![is-loaded OpenMPI/1.4.5-GCC-4.6.3-no-OFED] } { + module load OpenMPI/1.4.5-GCC-4.6.3-no-OFED +} + +if { ![is-loaded ATLAS/3.8.4-gompi-1.1.0-no-OFED-LAPACK-3.4.0] } { + module load ATLAS/3.8.4-gompi-1.1.0-no-OFED-LAPACK-3.4.0 +} + +# load statement for FFTW purposely omitted, which doesn't match the toolchain definition +#if { ![is-loaded FFTW/3.3.1-gompi-1.1.0-no-OFED] } { +# module load FFTW/3.3.1-gompi-1.1.0-no-OFED +#} + +if { ![is-loaded BLACS/1.1-gompi-1.1.0-no-OFED] } { + module load BLACS/1.1-gompi-1.1.0-no-OFED +} + +if { ![is-loaded ScaLAPACK/1.8.0-gompi-1.1.0-no-OFED-ATLAS-3.8.4-LAPACK-3.4.0-BLACS-1.1] } { + module load ScaLAPACK/1.8.0-gompi-1.1.0-no-OFED-ATLAS-3.8.4-LAPACK-3.4.0-BLACS-1.1 +} + + +setenv EBROOTGOALF $root +setenv EBVERSIONGOALF 1.1.0 +setenv EBDEVELGOALF $root/easybuild/goalf-1.1.0-no-OFED-broken-easybuild-devel + + +# built with EasyBuild version 0.9dev diff --git a/test/framework/modules/icc/11.1.073 b/test/framework/modules/icc/11.1.073 new file mode 100644 index 0000000000..f78c11207f --- /dev/null +++ b/test/framework/modules/icc/11.1.073 @@ -0,0 +1,7 @@ +#%Module + +set root /tmp/icc/11.1.073 + +setenv EBROOTICC "$root" +setenv EBVERSIONICC "11.1.073" +setenv EBDEVELICC "$root/easybuild/icc-11.1.073-easybuild-devel" diff --git a/test/framework/modules/ictce/3.2.2.u3 b/test/framework/modules/ictce/3.2.2.u3 new file mode 100644 index 0000000000..453b686f4c --- /dev/null +++ b/test/framework/modules/ictce/3.2.2.u3 @@ -0,0 +1,36 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { Intel Cluster Toolkit Compiler Edition provides Intel C/C++ and Fortran compilers, Intel MPI & Intel MKL. - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/ + } +} + +module-whatis {Intel Cluster Toolkit Compiler Edition provides Intel C/C++ and Fortran compilers, Intel MPI & Intel MKL. - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/} + +set root /tmp/ictce/3.2.2.u3 + +conflict ictce + +if { ![is-loaded icc/11.1.073] } { + module load icc/11.1.073 +} + +if { ![is-loaded ifort/11.1.073] } { + module load ifort/11.1.073 +} + +if { ![is-loaded impi/4.0.0.028] } { + module load impi/4.0.0.028 +} + +if { ![is-loaded imkl/10.2.6.038] } { + module load imkl/10.2.6.038 +} + + +setenv EBROOTICTCE "$root" +setenv EBVERSIONICTCE "3.2.2.u3" +setenv EBDEVELICTCE "$root/easybuild/ictce-3.2.2.u3-easybuild-devel" + + +# built with EasyBuild version 1.9.0dev diff --git a/test/framework/modules/ifort/11.1.073 b/test/framework/modules/ifort/11.1.073 new file mode 100644 index 0000000000..66e68eef4e --- /dev/null +++ b/test/framework/modules/ifort/11.1.073 @@ -0,0 +1,7 @@ +#%Module + +set root /tmp/ifort/11.1.073 + +setenv EBROOTIFORT "$root" +setenv EBVERSIONIFORT "11.1.073" +setenv EBDEVELIFORT "$root/easybuild/ifort-11.1.073-easybuild-devel" diff --git a/test/framework/modules/imkl/10.2.6.038 b/test/framework/modules/imkl/10.2.6.038 new file mode 100644 index 0000000000..c539b9e5f7 --- /dev/null +++ b/test/framework/modules/imkl/10.2.6.038 @@ -0,0 +1,7 @@ +#%Module + +set root /var/folders/8s/_frgh9sj6m744mxt5w5lyztr0000gn/T/eb-SGdeX9/tmptwWn9I + +setenv EBROOTIMKL "$root" +setenv EBVERSIONIMKL "10.2.6.038" +setenv EBDEVELIMKL "$root/easybuild/imkl-10.2.6.038-easybuild-devel" diff --git a/test/framework/modules/imkl/10.3.12.361 b/test/framework/modules/imkl/10.3.12.361 index 780b019c78..d63fb61362 100644 --- a/test/framework/modules/imkl/10.3.12.361 +++ b/test/framework/modules/imkl/10.3.12.361 @@ -13,7 +13,7 @@ module-whatis {Intel Math Kernel Library is a library of highly optimized, applications that require maximum performance. Core math functions include BLAS, LAPACK, ScaLAPACK, Sparse Solvers, Fast Fourier Transforms, Vector Math, and more. - Homepage: http://software.intel.com/en-us/intel-mkl/} -set root /var/folders/6y/x4gmwgjn5qz63b7ftg4j_40m0000gn/T/tmpZg_7A8 +set root /tmp/eb-bI0pBy/eb-DmuEpJ/eb-leoYDw/eb-UtJJqp/tmp8P3FOY conflict imkl diff --git a/test/framework/modules/impi/4.0.0.028 b/test/framework/modules/impi/4.0.0.028 new file mode 100644 index 0000000000..ec11aaef29 --- /dev/null +++ b/test/framework/modules/impi/4.0.0.028 @@ -0,0 +1,7 @@ +#%Module + +set root /tmp/impi/4.0.0.028 + +setenv EBROOTIMPI "$root" +setenv EBVERSIONIMPI "4.0.0.028" +setenv EBDEVELIMPI "$root/easybuild/impi-4.0.0.028-easybuild-devel" diff --git a/test/framework/modules/toy/.0.0-deps b/test/framework/modules/toy/.0.0-deps new file mode 100644 index 0000000000..d2bedec555 --- /dev/null +++ b/test/framework/modules/toy/.0.0-deps @@ -0,0 +1,25 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { Toy C program. - Homepage: http://hpcugent.github.com/easybuild + } +} + +module-whatis {Toy C program. - Homepage: http://hpcugent.github.com/easybuild} + +set root /var/folders/6y/x4gmwgjn5qz63b7ftg4j_40m0000gn/T/tmpviG1OT/software/toy/0.0-deps + +conflict toy + +if { ![is-loaded gompi/1.3.12] } { + module load gompi/1.3.12 +} + +prepend-path PATH $root/bin + +setenv EBROOTTOY "$root" +setenv EBVERSIONTOY "0.0-deps" +setenv EBDEVELTOY "$root/easybuild/toy-0.0-deps-easybuild-devel" + + +# built with EasyBuild version 1.8.0dev diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index ad8c28de27..4eadb8d0d1 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2014 Ghent University +# Copyright 2014-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,6 +29,7 @@ """ import os import re +import stat import tempfile from vsc.utils import fancylogger @@ -41,7 +42,7 @@ from easybuild.tools import config, modules from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option -from easybuild.tools.filetools import which +from easybuild.tools.filetools import which, write_file from easybuild.tools.modules import modules_tool, Lmod from test.framework.utilities import init_config @@ -76,7 +77,7 @@ def test_mock(self): os.environ['module'] = "() { eval `/bin/echo $*`\n}" # ue empty mod_path list, otherwise the install_path is called - mmt = MockModulesTool(mod_paths=[]) + mmt = MockModulesTool(mod_paths=[], testing=True) # the version of the MMT is the commandline option self.assertEqual(mmt.version, StrictVersion(MockModulesTool.VERSION_OPTION)) @@ -91,16 +92,16 @@ def test_environment_command(self): os.environ['module'] = "() { %s $*\n}" % BrokenMockModulesTool.COMMAND try: - bmmt = BrokenMockModulesTool(mod_paths=[]) + bmmt = BrokenMockModulesTool(mod_paths=[], testing=True) # should never get here self.assertTrue(False, 'BrokenMockModulesTool should fail') except EasyBuildError, err: - self.assertTrue('command is not available' in str(err)) + err_msg = "command is not available" + self.assertTrue(err_msg in str(err), "'%s' found in: %s" % (err_msg, err)) os.environ[BrokenMockModulesTool.COMMAND_ENVIRONMENT] = MockModulesTool.COMMAND os.environ['module'] = "() { /bin/echo $*\n}" - BrokenMockModulesTool._instances.pop(BrokenMockModulesTool, None) - bmmt = BrokenMockModulesTool(mod_paths=[]) + bmmt = BrokenMockModulesTool(mod_paths=[], testing=True) cmd_abspath = which(MockModulesTool.COMMAND) self.assertEqual(bmmt.version, StrictVersion(MockModulesTool.VERSION_OPTION)) @@ -112,8 +113,9 @@ def test_environment_command(self): def test_module_mismatch(self): """Test whether mismatch detection between modules tool and 'module' function works.""" # redefine 'module' function (deliberate mismatch with used module command in MockModulesTool) - os.environ['module'] = "() { eval `/Users/kehoste/Modules/$MODULE_VERSION/bin/modulecmd bash $*`\n}" - self.assertErrorRegex(EasyBuildError, ".*pattern .* not found in defined 'module' function", MockModulesTool) + os.environ['module'] = "() { eval `/tmp/Modules/$MODULE_VERSION/bin/modulecmd bash $*`\n}" + error_regex = ".*pattern .* not found in defined 'module' function" + self.assertErrorRegex(EasyBuildError, error_regex, MockModulesTool, testing=True) # check whether escaping error by allowing mismatch via build options works build_options = { @@ -123,7 +125,7 @@ def test_module_mismatch(self): fancylogger.logToFile(self.logfile) - mt = MockModulesTool() + mt = MockModulesTool(testing=True) f = open(self.logfile, 'r') logtxt = f.read() f.close() @@ -132,14 +134,12 @@ def test_module_mismatch(self): # redefine 'module' function with correct module command os.environ['module'] = "() { eval `/bin/echo $*`\n}" - MockModulesTool._instances.pop(MockModulesTool) - mt = MockModulesTool() + mt = MockModulesTool(testing=True) self.assertTrue(isinstance(mt.loaded_modules(), list)) # dummy usage # a warning should be logged if the 'module' function is undefined del os.environ['module'] - MockModulesTool._instances.pop(MockModulesTool) - mt = MockModulesTool() + mt = MockModulesTool(testing=True) f = open(self.logfile, 'r') logtxt = f.read() f.close() @@ -155,9 +155,13 @@ def test_lmod_specific(self): if lmod_abspath is not None: build_options = { 'allow_modules_tool_mismatch': True, + 'update_modules_tool_cache': True, } init_config(build_options=build_options) + lmod = Lmod(testing=True) + self.assertEqual(lmod.cmd, lmod_abspath) + # drop any location where 'lmod' or 'spider' can be found from $PATH paths = os.environ.get('PATH', '').split(os.pathsep) new_paths = [] @@ -171,10 +175,19 @@ def test_lmod_specific(self): # make sure $MODULEPATH contains path that provides some modules os.environ['MODULEPATH'] = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules')) - # initialize Lmod modules tool, pass full path to 'lmod' via $LMOD_CMD + # initialize Lmod modules tool, pass (fake) full path to 'lmod' via $LMOD_CMD + fake_path = os.path.join(self.test_installpath, 'lmod') + write_file(fake_path, '#!/bin/bash\necho "Modules based on Lua: Version %s " >&2' % Lmod.REQ_VERSION) + os.chmod(fake_path, stat.S_IRUSR|stat.S_IXUSR) + os.environ['LMOD_CMD'] = fake_path + init_config(build_options=build_options) + lmod = Lmod(testing=True) + self.assertEqual(lmod.cmd, fake_path) + + # use correct full path for 'lmod' via $LMOD_CMD os.environ['LMOD_CMD'] = lmod_abspath - lmod = Lmod() - lmod.testing = True + init_config(build_options=build_options) + lmod = Lmod(testing=True) # obtain list of availabe modules, should be non-empty self.assertTrue(lmod.available(), "List of available modules obtained using Lmod is non-empty") diff --git a/test/framework/options.py b/test/framework/options.py index 67d580a476..7f10003976 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -33,26 +33,42 @@ import shutil import sys import tempfile -from test.framework.utilities import EnhancedTestCase +from test.framework.utilities import EnhancedTestCase, init_config from unittest import TestLoader from unittest import main as unittestmain +from urllib2 import URLError import easybuild.tools.build_log +import easybuild.tools.options +import easybuild.tools.toolchain from easybuild.framework.easyconfig import BUILD, CUSTOM, DEPENDENCIES, EXTENSIONS, FILEMANAGEMENT, LICENSE from easybuild.framework.easyconfig import MANDATORY, MODULES, OTHER, TOOLCHAIN from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import DEFAULT_MODULECLASSES, get_module_syntax from easybuild.tools.environment import modify_env -from easybuild.tools.filetools import read_file, write_file +from easybuild.tools.filetools import mkdir, read_file, write_file +from easybuild.tools.github import fetch_github_token from easybuild.tools.modules import modules_tool from easybuild.tools.options import EasyBuildOptions +from easybuild.tools.toolchain.utilities import TC_CONST_PREFIX from easybuild.tools.version import VERSION from vsc.utils import fancylogger + +# test account, for which a token is available +GITHUB_TEST_ACCOUNT = 'easybuild_test' + + class CommandLineOptionsTest(EnhancedTestCase): """Testcases for command line options.""" logfile = None + def setUp(self): + """Set up test.""" + super(CommandLineOptionsTest, self).setUp() + self.github_token = fetch_github_token(GITHUB_TEST_ACCOUNT) + def test_help_short(self, txt=None): """Test short help message.""" @@ -87,7 +103,7 @@ def test_help_long(self): ) outtxt = topt.parser.help_to_file.getvalue() - self.assertTrue(re.search("-H, --help", outtxt), "Long documentation expanded in long help") + self.assertTrue(re.search("-H OUTPUT_FORMAT, --help=OUTPUT_FORMAT", outtxt), "Long documentation expanded in long help") self.assertTrue(re.search("show short help message and exit", outtxt), "Documentation included in long help") self.assertTrue(re.search("Software search and build options", outtxt), "Not all option groups included in short help (1)") self.assertTrue(re.search("Regression test options", outtxt), "Not all option groups included in short help (2)") @@ -97,33 +113,29 @@ def test_no_args(self): outtxt = self.eb_main([]) - error_msg = "ERROR .* Please provide one or multiple easyconfig files," + error_msg = "ERROR Please provide one or multiple easyconfig files," error_msg += " or use software build options to make EasyBuild search for easyconfigs" self.assertTrue(re.search(error_msg, outtxt), "Error message when eb is run without arguments") def test_debug(self): """Test enabling debug logging.""" - for debug_arg in ['-d', '--debug']: args = [ - '--software-name=somethingrandom', - debug_arg, - ] + 'nosuchfile.eb', + debug_arg, + ] outtxt = self.eb_main(args) for log_msg_type in ['DEBUG', 'INFO', 'ERROR']: res = re.search(' %s ' % log_msg_type, outtxt) self.assertTrue(res, "%s log messages are included when using %s: %s" % (log_msg_type, debug_arg, outtxt)) - modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None - def test_info(self): """Test enabling info logging.""" for info_arg in ['--info']: args = [ - '--software-name=somethingrandom', + 'nosuchfile.eb', info_arg, ] outtxt = self.eb_main(args) @@ -136,29 +148,24 @@ def test_info(self): res = re.search(' %s ' % log_msg_type, outtxt) self.assertTrue(not res, "%s log messages are *not* included when using %s" % (log_msg_type, info_arg)) - modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None - def test_quiet(self): """Test enabling quiet logging (errors only).""" - for quiet_arg in ['--quiet']: args = [ - '--software-name=somethingrandom', + 'nosuchfile.eb', quiet_arg, ] outtxt = self.eb_main(args) for log_msg_type in ['ERROR']: res = re.search(' %s ' % log_msg_type, outtxt) - self.assertTrue(res, "%s log messages are included when using %s (outtxt: %s)" % (log_msg_type, quiet_arg, outtxt)) + msg = "%s log messages are included when using %s (outtxt: %s)" % (log_msg_type, quiet_arg, outtxt) + self.assertTrue(res, msg) for log_msg_type in ['DEBUG', 'INFO']: res = re.search(' %s ' % log_msg_type, outtxt) - self.assertTrue(not res, "%s log messages are *not* included when using %s (outtxt: %s)" % (log_msg_type, quiet_arg, outtxt)) - - modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None + msg = "%s log messages are *not* included when using %s (outtxt: %s)" % (log_msg_type, quiet_arg, outtxt) + self.assertTrue(not res, msg) def test_force(self): """Test forcing installation even if the module is already available.""" @@ -178,10 +185,8 @@ def test_force(self): already_msg = "GCC/4.6.3 is already installed" self.assertTrue(re.search(already_msg, outtxt), "Already installed message without --force, outtxt: %s" % outtxt) - # clear log file, clean up environment + # clear log file write_file(self.logfile, '') - modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None # check that --force works args = [ @@ -196,20 +201,15 @@ def test_force(self): def test_skip(self): """Test skipping installation of module (--skip, -k).""" - # use temporary paths for build/install paths, make sure sources can be found - buildpath = tempfile.mkdtemp() - installpath = tempfile.mkdtemp() - sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'sources') - # use toy-0.0.eb easyconfig file that comes with the tests - eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb') + eb_file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'toy-0.0.eb') # check log message with --skip for existing module args = [ eb_file, - '--sourcepath=%s' % sourcepath, - '--buildpath=%s' % buildpath, - '--installpath=%s' % installpath, + '--sourcepath=%s' % self.test_sourcepath, + '--buildpath=%s' % self.test_buildpath, + '--installpath=%s' % self.test_installpath, '--force', '--debug', ] @@ -228,17 +228,15 @@ def test_skip(self): os.chdir(self.cwd) modules_tool().purge() # reinitialize modules tool with original $MODULEPATH, to avoid problems with future tests - modify_env(os.environ, self.orig_environ) os.environ['MODULEPATH'] = '' modules_tool() - tempfile.tempdir = None # check log message with --skip for non-existing module args = [ eb_file, - '--sourcepath=%s' % sourcepath, - '--buildpath=%s' % buildpath, - '--installpath=%s' % installpath, + '--sourcepath=%s' % self.test_sourcepath, + '--buildpath=%s' % self.test_buildpath, + '--installpath=%s' % self.test_installpath, '--try-software-version=1.2.3.4.5.6.7.8.9', '--try-amend=sources=toy-0.0.tar.gz,toy-0.0.tar.gz', # hackish, but fine '--force', @@ -260,21 +258,16 @@ def test_skip(self): modify_env(os.environ, self.orig_environ) modules_tool() - # cleanup - shutil.rmtree(buildpath) - shutil.rmtree(installpath) - def test_job(self): """Test submitting build as a job.""" # use gzip-1.4.eb easyconfig file that comes with the tests eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.4.eb') - # check log message with --job - for job_args in [ # options passed are reordered, so order here matters to make tests pass - ['--debug'], - ['--debug', '--stop=configure', '--try-software-name=foo'], - ]: + def check_args(job_args, passed_args=None): + """Check whether specified args yield expected result.""" + if passed_args is None: + passed_args = job_args[:] # clear log file write_file(self.logfile, '') @@ -285,12 +278,17 @@ def test_job(self): ] + job_args outtxt = self.eb_main(args) - job_msg = "INFO.* Command template for jobs: .* && eb %%\(spec\)s.* %s.*\n" % ' .*'.join(job_args) - assertmsg = "Info log message with job command template when using --job (job_msg: %s, outtxt: %s)" % (job_msg, outtxt) + job_msg = "INFO.* Command template for jobs: .* && eb %%\(spec\)s.* %s.*\n" % ' .*'.join(passed_args) + assertmsg = "Info log msg with job command template for --job (job_msg: %s, outtxt: %s)" % (job_msg, outtxt) self.assertTrue(re.search(job_msg, outtxt), assertmsg) - modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None + # options passed are reordered, so order here matters to make tests pass + check_args(['--debug']) + check_args(['--debug', '--stop=configure', '--try-software-name=foo']) + check_args(['--debug', '--robot-paths=/tmp/foo:/tmp/bar']) + # --robot has preference over --robot-paths, --robot is not passed down + check_args(['--debug', '--robot-paths=/tmp/foo', '--robot=/tmp/bar'], + passed_args=['--debug', '--robot-paths=/tmp/bar:/tmp/foo']) # 'zzz' prefix in the test name is intentional to make this test run last, # since it fiddles with the logging infrastructure which may break things @@ -327,38 +325,49 @@ def test_zzz_logtostdout(self): # cleanup os.remove(fn) - modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None + + stdoutorig = sys.stdout + sys.stdout = open("/dev/null", 'w') + + toy_ecfile = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'toy-0.0.eb') + self.logfile = None + out = self.eb_main([toy_ecfile, '--debug', '-l', '--force'], raise_error=True) if os.path.exists(dummylogfn): os.remove(dummylogfn) - fancylogger.logToFile(self.logfile) + + sys.stdout.close() + sys.stdout = stdoutorig def test_avail_easyconfig_params(self): """Test listing available easyconfig parameters.""" - def run_test(custom=None, extra_params=[]): + def run_test(custom=None, extra_params=[], fmt=None): """Inner function to run actual test in current setting.""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) - for avail_arg in [ - '-a', - '--avail-easyconfig-params', - ]: + avail_args = [ + '-a', + '--avail-easyconfig-params', + ] + for avail_arg in avail_args: # clear log write_file(self.logfile, '') args = [ - avail_arg, - '--unittest-file=%s' % self.logfile, - ] + '--unittest-file=%s' % self.logfile, + avail_arg, + ] + if fmt is not None: + args.append(fmt) if custom is not None: args.extend(['-e', custom]) outtxt = self.eb_main(args, logfile=dummylogfn, verbose=True) + logtxt = read_file(self.logfile) # check whether all parameter types are listed par_types = [BUILD, DEPENDENCIES, EXTENSIONS, FILEMANAGEMENT, @@ -367,27 +376,29 @@ def run_test(custom=None, extra_params=[]): par_types.append(CUSTOM) for param_type in [x[1] for x in par_types]: - self.assertTrue(re.search("%s\n%s" % (param_type.upper(), '-' * len(param_type)), outtxt), - "Parameter type %s is featured in output of eb %s (args: %s): %s" % - (param_type, avail_arg, args, outtxt)) + # regex for parameter group title, matches both txt and rst formats + regex = re.compile("%s.*\n%s" % (param_type, '-' * len(param_type)), re.I) + tup = (param_type, avail_arg, args, logtxt) + msg = "Parameter type %s is featured in output of eb %s (args: %s): %s" % tup + self.assertTrue(regex.search(logtxt), msg) # check a couple of easyconfig parameters for param in ["name", "version", "toolchain", "versionsuffix", "buildopts", "sources", "start_dir", "dependencies", "group", "exts_list", "moduleclass", "buildstats"] + extra_params: - self.assertTrue(re.search("%s(?:\(\*\))?:\s*\w.*" % param, outtxt), - "Parameter %s is listed with help in output of eb %s (args: %s): %s" % - (param, avail_arg, args, outtxt) - ) - - modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None + # regex for parameter name (with optional '*') & description, matches both txt and rst formats + regex = re.compile("^[`]*%s(?:\*)?[`]*\s+\w+" % param, re.M) + tup = (param, avail_arg, args, regex.pattern, logtxt) + msg = "Parameter %s is listed with help in output of eb %s (args: %s, regex: %s): %s" % tup + self.assertTrue(regex.search(logtxt), msg) if os.path.exists(dummylogfn): os.remove(dummylogfn) - run_test(custom='EB_foo', extra_params=['foo_extra1', 'foo_extra2']) - run_test(custom='bar', extra_params=['bar_extra1', 'bar_extra2']) - run_test(custom='EB_foofoo', extra_params=['foofoo_extra1', 'foofoo_extra2']) + for fmt in [None, 'txt', 'rst']: + run_test(fmt=fmt) + run_test(custom='EB_foo', extra_params=['foo_extra1', 'foo_extra2'], fmt=fmt) + run_test(custom='bar', extra_params=['bar_extra1', 'bar_extra2'], fmt=fmt) + run_test(custom='EB_foofoo', extra_params=['foofoo_extra1', 'foofoo_extra2'], fmt=fmt) # double underscore to make sure it runs first, which is required to detect certain types of bugs, # e.g. running with non-initialized EasyBuild config (truly mimicing 'eb --list-toolchains') @@ -404,13 +415,22 @@ def test__list_toolchains(self): outtxt = self.eb_main(args, logfile=dummylogfn) info_msg = r"INFO List of known toolchains \(toolchainname: module\[,module\.\.\.\]\):" - self.assertTrue(re.search(info_msg, outtxt), "Info message with list of known compiler toolchains") - for tc in ["dummy", "goalf", "ictce"]: - res = re.findall("^\s*%s: " % tc, outtxt, re.M) + logtxt = read_file(self.logfile) + self.assertTrue(re.search(info_msg, logtxt), "Info message with list of known compiler toolchains") + # toolchain elements should be in alphabetical order + tcs = { + 'dummy': [], + 'goalf': ['ATLAS', 'BLACS', 'FFTW', 'GCC', 'OpenMPI', 'ScaLAPACK'], + 'ictce': ['icc', 'ifort', 'imkl', 'impi'], + } + for tc, tcelems in tcs.items(): + res = re.findall("^\s*%s: .*" % tc, logtxt, re.M) self.assertTrue(res, "Toolchain %s is included in list of known compiler toolchains" % tc) # every toolchain should only be mentioned once n = len(res) self.assertEqual(n, 1, "Toolchain %s is only mentioned once (count: %d)" % (tc, n)) + # make sure definition is correct (each element only named once, in alphabetical order) + self.assertEqual("\t%s: %s" % (tc, ', '.join(tcelems)), res[0]) if os.path.exists(dummylogfn): os.remove(dummylogfn) @@ -423,7 +443,7 @@ def test_avail_lists(self): name_items = { 'modules-tools': ['EnvironmentModulesC', 'Lmod'], - 'module-naming-schemes': ['EasyBuildMNS', 'HierarchicalMNS'], + 'module-naming-schemes': ['EasyBuildMNS', 'HierarchicalMNS', 'CategorizedHMNS'], } for (name, items) in name_items.items(): args = [ @@ -431,22 +451,54 @@ def test_avail_lists(self): '--unittest-file=%s' % self.logfile, ] outtxt = self.eb_main(args, logfile=dummylogfn) + logtxt = read_file(self.logfile) words = name.replace('-', ' ') info_msg = r"INFO List of supported %s:" % words - self.assertTrue(re.search(info_msg, outtxt), "Info message with list of available %s" % words) + self.assertTrue(re.search(info_msg, logtxt), "Info message with list of available %s" % words) for item in items: - res = re.findall("^\s*%s" % item, outtxt, re.M) + res = re.findall("^\s*%s" % item, logtxt, re.M) self.assertTrue(res, "%s is included in list of available %s" % (item, words)) # every item should only be mentioned once n = len(res) self.assertEqual(n, 1, "%s is only mentioned once (count: %d)" % (item, n)) - modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None + if os.path.exists(dummylogfn): + os.remove(dummylogfn) + + def test_avail_cfgfile_constants(self): + """Test --avail-cfgfile-constants.""" + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory + # to check whether easyconfigs install path is auto-included in robot path + tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') + mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) + + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + shutil.copytree(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) + + orig_sys_path = sys.path[:] + sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs + + args = [ + '--avail-cfgfile-constants', + '--unittest-file=%s' % self.logfile, + ] + outtxt = self.eb_main(args, logfile=dummylogfn) + logtxt = read_file(self.logfile) + cfgfile_constants = { + 'DEFAULT_ROBOT_PATHS': os.path.join(tmpdir, 'easybuild', 'easyconfigs'), + } + for cst_name, cst_value in cfgfile_constants.items(): + cst_regex = re.compile(r"^\*\s%s:\s.*\s\[value: .*%s.*\]" % (cst_name, cst_value), re.M) + tup = (cst_regex.pattern, logtxt) + self.assertTrue(cst_regex.search(logtxt), "Pattern '%s' in --avail-cfgfile_constants output: %s" % tup) if os.path.exists(dummylogfn): os.remove(dummylogfn) + sys.path[:] = orig_sys_path def test_list_easyblocks(self): """Test listing easyblock hierarchy.""" @@ -454,18 +506,6 @@ def test_list_easyblocks(self): fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) - # adjust PYTHONPATH such that test easyblocks are found - - import easybuild - eb_blocks_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'sandbox')) - if not eb_blocks_path in sys.path: - sys.path.append(eb_blocks_path) - easybuild = reload(easybuild) - - import easybuild.easyblocks - reload(easybuild.easyblocks) - reload(easybuild.tools.module_naming_scheme) # required to run options unit tests stand-alone - # simple view for list_arg in ['--list-easyblocks', '--list-easyblocks=simple']: @@ -476,7 +516,8 @@ def test_list_easyblocks(self): list_arg, '--unittest-file=%s' % self.logfile, ] - outtxt = self.eb_main(args, logfile=dummylogfn) + self.eb_main(args, logfile=dummylogfn) + logtxt = read_file(self.logfile) for pat in [ r"EasyBlock\n", @@ -484,10 +525,8 @@ def test_list_easyblocks(self): r"|--\s+bar\n", ]: - self.assertTrue(re.search(pat, outtxt), "Pattern '%s' is found in output of --list-easyblocks: %s" % (pat, outtxt)) - - modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None + msg = "Pattern '%s' is found in output of --list-easyblocks: %s" % (pat, logtxt) + self.assertTrue(re.search(pat, logtxt), msg) # clear log write_file(self.logfile, '') @@ -497,7 +536,8 @@ def test_list_easyblocks(self): '--list-easyblocks=detailed', '--unittest-file=%s' % self.logfile, ] - outtxt = self.eb_main(args, logfile=dummylogfn) + self.eb_main(args, logfile=dummylogfn) + logtxt = read_file(self.logfile) for pat in [ r"EasyBlock\s+\(easybuild.framework.easyblock\)\n", @@ -505,7 +545,8 @@ def test_list_easyblocks(self): r"|--\s+bar\s+\(easybuild.easyblocks.generic.bar\)\n", ]: - self.assertTrue(re.search(pat, outtxt), "Pattern '%s' is found in output of --list-easyblocks: %s" % (pat, outtxt)) + msg = "Pattern '%s' is found in output of --list-easyblocks: %s" % (pat, logtxt) + self.assertTrue(re.search(pat, logtxt), msg) if os.path.exists(dummylogfn): os.remove(dummylogfn) @@ -521,65 +562,108 @@ def test_search(self): '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), '--unittest-file=%s' % self.logfile, ] - outtxt = self.eb_main(args, logfile=dummylogfn) + self.eb_main(args, logfile=dummylogfn) + logtxt = read_file(self.logfile) info_msg = r"Searching \(case-insensitive\) for 'gzip' in" - self.assertTrue(re.search(info_msg, outtxt), "Info message when searching for easyconfigs in '%s'" % outtxt) + self.assertTrue(re.search(info_msg, logtxt), "Info message when searching for easyconfigs in '%s'" % logtxt) for ec in ["gzip-1.4.eb", "gzip-1.4-GCC-4.6.3.eb"]: - self.assertTrue(re.search(" \* \S*%s$" % ec, outtxt, re.M), "Found easyconfig %s in '%s'" % (ec, outtxt)) + self.assertTrue(re.search(r" \* \S*%s$" % ec, logtxt, re.M), "Found easyconfig %s in '%s'" % (ec, logtxt)) + + if os.path.exists(dummylogfn): + os.remove(dummylogfn) + + write_file(self.logfile, '') + + args = [ + '--search=^gcc.*2.eb', + '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + '--unittest-file=%s' % self.logfile, + ] + self.eb_main(args, logfile=dummylogfn) + logtxt = read_file(self.logfile) + + info_msg = r"Searching \(case-insensitive\) for '\^gcc.\*2.eb' in" + self.assertTrue(re.search(info_msg, logtxt), "Info message when searching for easyconfigs in '%s'" % logtxt) + for ec in ['GCC-4.7.2.eb', 'GCC-4.8.2.eb', 'GCC-4.9.2.eb']: + self.assertTrue(re.search(r" \* \S*%s$" % ec, logtxt, re.M), "Found easyconfig %s in '%s'" % (ec, logtxt)) if os.path.exists(dummylogfn): os.remove(dummylogfn) + write_file(self.logfile, '') + for search_arg in ['-S', '--search-short']: open(self.logfile, 'w').write('') args = [ search_arg, 'toy-0.0', - '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + '-r', + os.path.join(os.path.dirname(__file__), 'easyconfigs'), '--unittest-file=%s' % self.logfile, ] - outtxt = self.eb_main(args, logfile=dummylogfn) + self.eb_main(args, logfile=dummylogfn, raise_error=True, verbose=True) + logtxt = read_file(self.logfile) info_msg = r"Searching \(case-insensitive\) for 'toy-0.0' in" - self.assertTrue(re.search(info_msg, outtxt), "Info message when searching for easyconfigs in '%s'" % outtxt) - self.assertTrue(re.search('INFO CFGS\d+=', outtxt), "CFGS line message found in '%s'" % outtxt) + self.assertTrue(re.search(info_msg, logtxt), "Info message when searching for easyconfigs in '%s'" % logtxt) + self.assertTrue(re.search('INFO CFGS\d+=', logtxt), "CFGS line message found in '%s'" % logtxt) for ec in ["toy-0.0.eb", "toy-0.0-multiple.eb"]: - self.assertTrue(re.search(" \* \$CFGS\d+/*%s" % ec, outtxt), "Found easyconfig %s in '%s'" % (ec, outtxt)) + self.assertTrue(re.search(" \* \$CFGS\d+/*%s" % ec, logtxt), "Found easyconfig %s in '%s'" % (ec, logtxt)) if os.path.exists(dummylogfn): os.remove(dummylogfn) def test_dry_run(self): - """Test dry runs.""" - + """Test dry run (long format).""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) args = [ os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.4-GCC-4.6.3.eb'), - '--dry-run', - '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + '--dry-run', # implies enabling dependency resolution '--unittest-file=%s' % self.logfile, + '--robot-paths=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), ] - outtxt = self.eb_main(args, logfile=dummylogfn) + self.eb_main(args, logfile=dummylogfn) + logtxt = read_file(self.logfile) info_msg = r"Dry run: printing build status of easyconfigs and dependencies" - self.assertTrue(re.search(info_msg, outtxt, re.M), "Info message dry running in '%s'" % outtxt) + self.assertTrue(re.search(info_msg, logtxt, re.M), "Info message dry running in '%s'" % logtxt) ecs_mods = [ - ("gzip-1.4-GCC-4.6.3.eb", "gzip/1.4-GCC-4.6.3"), - ("GCC-4.6.3.eb", "GCC/4.6.3"), + ("gzip-1.4-GCC-4.6.3.eb", "gzip/1.4-GCC-4.6.3", ' '), + ("GCC-4.6.3.eb", "GCC/4.6.3", 'x'), ] - for ec, mod in ecs_mods: - regex = re.compile(r" \* \[.\] \S+%s \(module: %s\)" % (ec, mod), re.M) - self.assertTrue(regex.search(outtxt), "Found match for pattern %s in '%s'" % (regex.pattern, outtxt)) + for ec, mod, mark in ecs_mods: + regex = re.compile(r" \* \[%s\] \S+%s \(module: %s\)" % (mark, ec, mod), re.M) + self.assertTrue(regex.search(logtxt), "Found match for pattern %s in '%s'" % (regex.pattern, logtxt)) + + def test_dry_run_short(self): + """Test dry run (short format).""" + # unset $EASYBUILD_ROBOT_PATHS that was defined in setUp + del os.environ['EASYBUILD_ROBOT_PATHS'] + + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory + # to check whether easyconfigs install path is auto-included in robot path + tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') + mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) + + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + shutil.copytree(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) + + orig_sys_path = sys.path[:] + sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs for dry_run_arg in ['-D', '--dry-run-short']: open(self.logfile, 'w').write('') args = [ - os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.4-GCC-4.6.3.eb'), + os.path.join(tmpdir, 'easybuild', 'easyconfigs', 'gzip-1.4-GCC-4.6.3.eb'), dry_run_arg, - '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + # purposely specifying senseless dir, to test auto-inclusion of easyconfigs pkg path in robot path + '--robot=%s' % os.path.join(tmpdir, 'robot_decoy'), '--unittest-file=%s' % self.logfile, ] outtxt = self.eb_main(args, logfile=dummylogfn) @@ -588,50 +672,140 @@ def test_dry_run(self): self.assertTrue(re.search(info_msg, outtxt, re.M), "Info message dry running in '%s'" % outtxt) self.assertTrue(re.search('CFGS=', outtxt), "CFGS line message found in '%s'" % outtxt) ecs_mods = [ - ("gzip-1.4-GCC-4.6.3.eb", "gzip/1.4-GCC-4.6.3"), - ("GCC-4.6.3.eb", "GCC/4.6.3"), + ("gzip-1.4-GCC-4.6.3.eb", "gzip/1.4-GCC-4.6.3", ' '), + ("GCC-4.6.3.eb", "GCC/4.6.3", 'x'), ] - for ec, mod in ecs_mods: - regex = re.compile(r" \* \[.\] \$CFGS\S+%s \(module: %s\)" % (ec, mod), re.M) + for ec, mod, mark in ecs_mods: + regex = re.compile(r" \* \[%s\] \$CFGS\S+%s \(module: %s\)" % (mark, ec, mod), re.M) self.assertTrue(regex.search(outtxt), "Found match for pattern %s in '%s'" % (regex.pattern, outtxt)) if os.path.exists(dummylogfn): os.remove(dummylogfn) + # cleanup + shutil.rmtree(tmpdir) + sys.path[:] = orig_sys_path + + def test_try_robot_force(self): + """ + Test correct behavior for combination of --try-toolchain --robot --force. + Only the listed easyconfigs should be forced, resolved dependencies should not (even if tweaked). + """ + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + # use toy-0.0.eb easyconfig file that comes with the tests + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + eb_file1 = os.path.join(test_ecs_dir, 'FFTW-3.3.3-gompi-1.4.10.eb') + eb_file2 = os.path.join(test_ecs_dir, 'ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb') + + # check log message with --skip for existing module + args = [ + eb_file1, + eb_file2, + '--sourcepath=%s' % self.test_sourcepath, + '--buildpath=%s' % self.test_buildpath, + '--installpath=%s' % self.test_installpath, + '--debug', + '--force', + '--robot=%s' % test_ecs_dir, + '--try-toolchain=gompi,1.3.12', + '--dry-run', + '--unittest-file=%s' % self.logfile, + ] + outtxt = self.eb_main(args, logfile=dummylogfn) + + scalapack_ver = '2.0.2-gompi-1.3.12-OpenBLAS-0.2.6-LAPACK-3.4.2' + ecs_mods = [ + # GCC/OpenMPI dependencies are there, but part of toolchain => 'x' + ("GCC-4.6.4.eb", "GCC/4.6.4", 'x'), + ("OpenMPI-1.6.4-GCC-4.6.4.eb", "OpenMPI/1.6.4-GCC-4.6.4", 'x'), + # OpenBLAS dependency is there, but not listed => 'x' + ("OpenBLAS-0.2.6-gompi-1.3.12-LAPACK-3.4.2.eb", "OpenBLAS/0.2.6-gompi-1.3.12-LAPACK-3.4.2", 'x'), + # both FFTW and ScaLAPACK are listed => 'F' + ("ScaLAPACK-%s.eb" % scalapack_ver, "ScaLAPACK/%s" % scalapack_ver, 'F'), + ("FFTW-3.3.3-gompi-1.3.12.eb", "FFTW/3.3.3-gompi-1.3.12", 'F'), + ] + for ec, mod, mark in ecs_mods: + regex = re.compile("^ \* \[%s\] \S+%s \(module: %s\)$" % (mark, ec, mod), re.M) + self.assertTrue(regex.search(outtxt), "Found match for pattern %s in '%s'" % (regex.pattern, outtxt)) + def test_dry_run_hierarchical(self): """Test dry run using a hierarchical module naming scheme.""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) + test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') args = [ - os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb'), + os.path.join(test_ecs, 'gzip-1.5-goolf-1.4.10.eb'), + os.path.join(test_ecs, 'OpenMPI-1.6.4-GCC-4.7.2.eb'), '--dry-run', '--unittest-file=%s' % self.logfile, '--module-naming-scheme=HierarchicalMNS', '--ignore-osdeps', + '--force', + '--debug', + '--robot-paths=%s' % os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs'), ] - errmsg = r"No robot path specified, which is required when looking for easyconfigs \(use --robot\)" - self.assertErrorRegex(EasyBuildError, errmsg, self.eb_main, args, logfile=dummylogfn, raise_error=True) - - args.append('--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs')) outtxt = self.eb_main(args, logfile=dummylogfn, verbose=True, raise_error=True) ecs_mods = [ # easyconfig, module subdir, (short) module name - ("GCC-4.7.2.eb", "Core", "GCC/4.7.2"), - ("hwloc-1.6.2-GCC-4.7.2.eb", "Compiler/GCC/4.7.2", "hwloc/1.6.2"), - ("OpenMPI-1.6.4-GCC-4.7.2.eb", "Compiler/GCC/4.7.2", "OpenMPI/1.6.4"), - ("gompi-1.4.10.eb", "Core", "gompi/1.4.10"), + ("GCC-4.7.2.eb", "Core", "GCC/4.7.2", 'x'), # already present but not listed, so 'x' + ("hwloc-1.6.2-GCC-4.7.2.eb", "Compiler/GCC/4.7.2", "hwloc/1.6.2", 'x'), + ("OpenMPI-1.6.4-GCC-4.7.2.eb", "Compiler/GCC/4.7.2", "OpenMPI/1.6.4", 'F'), # already present and listed, so 'F' + ("gompi-1.4.10.eb", "Core", "gompi/1.4.10", 'x'), ("OpenBLAS-0.2.6-gompi-1.4.10-LAPACK-3.4.2.eb", "MPI/GCC/4.7.2/OpenMPI/1.6.4", - "OpenBLAS/0.2.6-LAPACK-3.4.2"), - ("FFTW-3.3.3-gompi-1.4.10.eb", "MPI/GCC/4.7.2/OpenMPI/1.6.4", "FFTW/3.3.3"), + "OpenBLAS/0.2.6-LAPACK-3.4.2", 'x'), + ("FFTW-3.3.3-gompi-1.4.10.eb", "MPI/GCC/4.7.2/OpenMPI/1.6.4", "FFTW/3.3.3", 'x'), ("ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb", "MPI/GCC/4.7.2/OpenMPI/1.6.4", - "ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2"), - ("goolf-1.4.10.eb", "Core", "goolf/1.4.10"), - ("gzip-1.5-goolf-1.4.10.eb", "MPI/GCC/4.8.2/OpenMPI/1.6.5", "gzip/1.5"), + "ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2", 'x'), + ("goolf-1.4.10.eb", "Core", "goolf/1.4.10", 'x'), + ("gzip-1.5-goolf-1.4.10.eb", "MPI/GCC/4.7.2/OpenMPI/1.6.4", "gzip/1.5", ' '), # listed but not there: ' ' + ] + for ec, mod_subdir, mod_name, mark in ecs_mods: + regex = re.compile("^ \* \[%s\] \S+%s \(module: %s \| %s\)$" % (mark, ec, mod_subdir, mod_name), re.M) + self.assertTrue(regex.search(outtxt), "Found match for pattern %s in '%s'" % (regex.pattern, outtxt)) + + if os.path.exists(dummylogfn): + os.remove(dummylogfn) + + def test_dry_run_categorized(self): + """Test dry run using a categorized hierarchical module naming scheme.""" + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + self.setup_categorized_hmns_modules() + test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + args = [ + os.path.join(test_ecs, 'gzip-1.5-goolf-1.4.10.eb'), + os.path.join(test_ecs, 'OpenMPI-1.6.4-GCC-4.7.2.eb'), + '--dry-run', + '--unittest-file=%s' % self.logfile, + '--module-naming-scheme=CategorizedHMNS', + '--ignore-osdeps', + '--force', + '--debug', + '--robot-paths=%s' % os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs'), ] - for ec, mod_subdir, mod_name in ecs_mods: - regex = re.compile(r" \* \[.\] \S+%s \(module: %s | %s\)" % (ec, mod_subdir, mod_name), re.M) + outtxt = self.eb_main(args, logfile=dummylogfn, verbose=True, raise_error=True) + + ecs_mods = [ + # easyconfig, module subdir, (short) module name, mark + ("GCC-4.7.2.eb", "Core/compiler", "GCC/4.7.2", 'x'), # already present but not listed, so 'x' + ("hwloc-1.6.2-GCC-4.7.2.eb", "Compiler/GCC/4.7.2/system", "hwloc/1.6.2", 'x'), + ("OpenMPI-1.6.4-GCC-4.7.2.eb", "Compiler/GCC/4.7.2/mpi", "OpenMPI/1.6.4", 'F'), # already present and listed, so 'F' + ("gompi-1.4.10.eb", "Core/toolchain", "gompi/1.4.10", 'x'), + ("OpenBLAS-0.2.6-gompi-1.4.10-LAPACK-3.4.2.eb", "MPI/GCC/4.7.2/OpenMPI/1.6.4/numlib", + "OpenBLAS/0.2.6-LAPACK-3.4.2", 'x'), + ("FFTW-3.3.3-gompi-1.4.10.eb", "MPI/GCC/4.7.2/OpenMPI/1.6.4/numlib", "FFTW/3.3.3", 'x'), + ("ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb", "MPI/GCC/4.7.2/OpenMPI/1.6.4/numlib", + "ScaLAPACK/2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2", 'x'), + ("goolf-1.4.10.eb", "Core/toolchain", "goolf/1.4.10", 'x'), + ("gzip-1.5-goolf-1.4.10.eb", "MPI/GCC/4.7.2/OpenMPI/1.6.4/tools", "gzip/1.5", ' '), # listed but not there: ' ' + ] + for ec, mod_subdir, mod_name, mark in ecs_mods: + regex = re.compile("^ \* \[%s\] \S+%s \(module: %s \| %s\)$" % (mark, ec, mod_subdir, mod_name), re.M) self.assertTrue(regex.search(outtxt), "Found match for pattern %s in '%s'" % (regex.pattern, outtxt)) if os.path.exists(dummylogfn): @@ -639,31 +813,110 @@ def test_dry_run_hierarchical(self): def test_from_pr(self): """Test fetching easyconfigs from a PR.""" + if self.github_token is None: + print "Skipping test_from_pr, no GitHub token available?" + return + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) + tmpdir = tempfile.mkdtemp() args = [ - # PR for ictce/6.2.5, see https://github.com/hpcugent/easybuild-easyconfigs/pull/726/files - '--from-pr=726', + # PR for foss/2015a, see https://github.com/hpcugent/easybuild-easyconfigs/pull/1239/files + '--from-pr=1239', '--dry-run', + # an argument must be specified to --robot, since easybuild-easyconfigs may not be installed '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), '--unittest-file=%s' % self.logfile, - '--github-user=easybuild_test', # a GitHub token should be available for this user - ] - outtxt = self.eb_main(args, logfile=dummylogfn, verbose=True) - - modules = [ - 'icc/2013_sp1.2.144', - 'ifort/2013_sp1.2.144', - 'impi/4.1.3.049', - 'imkl/11.1.2.144', - 'ictce/6.2.5', - 'gzip/1.6-ictce-6.2.5', + '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user + '--tmpdir=%s' % tmpdir, ] - for module in modules: - ec_fn = "%s.eb" % '-'.join(module.split('/')) - regex = re.compile(r"^ \* \[.\] .*/%s \(module: %s\)$" % (ec_fn, module), re.M) + try: + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) + modules = [ + (tmpdir, 'FFTW/3.3.4-gompi-2015a'), + (tmpdir, 'foss/2015a'), + ('.*', 'GCC/4.9.2'), # not included in PR + (tmpdir, 'gompi/2015a'), + (tmpdir, 'HPL/2.1-foss-2015a'), + (tmpdir, 'hwloc/1.10.0-GCC-4.9.2'), + (tmpdir, 'numactl/2.0.10-GCC-4.9.2'), + (tmpdir, 'OpenBLAS/0.2.13-GCC-4.9.2-LAPACK-3.5.0'), + (tmpdir, 'OpenMPI/1.8.3-GCC-4.9.2'), + (tmpdir, 'OpenMPI/1.8.4-GCC-4.9.2'), + (tmpdir, 'ScaLAPACK/2.0.2-gompi-2015a-OpenBLAS-0.2.13-LAPACK-3.5.0'), + ] + for path_prefix, module in modules: + ec_fn = "%s.eb" % '-'.join(module.split('/')) + regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path_prefix, ec_fn, module), re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + + # make sure that *only* these modules are listed, no others + regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M) + self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules)) + + pr_tmpdir = os.path.join(tmpdir, 'eb-\S{6}', 'files_pr1239') + regex = re.compile("Prepended list of robot search paths with %s:" % pr_tmpdir, re.M) self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + except URLError, err: + print "Ignoring URLError '%s' in test_from_pr" % err + shutil.rmtree(tmpdir) + + def test_from_pr_listed_ecs(self): + """Test --from-pr in combination with specifying easyconfigs on the command line.""" + if self.github_token is None: + print "Skipping test_from_pr, no GitHub token available?" + return + + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory + test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + ecstmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') + mkdir(os.path.join(ecstmpdir, 'easybuild'), parents=True) + shutil.copytree(test_ecs_path, os.path.join(ecstmpdir, 'easybuild', 'easyconfigs')) + + # inject path to test easyconfigs into head of Python search path + sys.path.insert(0, ecstmpdir) + + tmpdir = tempfile.mkdtemp() + args = [ + 'toy-0.0.eb', + 'gompi-2015a.eb', # also pulls in GCC, OpenMPI (which pulls in hwloc and numactl) + 'GCC-4.6.3.eb', + # PR for foss/2015a, see https://github.com/hpcugent/easybuild-easyconfigs/pull/1239/files + '--from-pr=1239', + '--dry-run', + # an argument must be specified to --robot, since easybuild-easyconfigs may not be installed + '--robot=%s' % test_ecs_path, + '--unittest-file=%s' % self.logfile, + '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user + '--tmpdir=%s' % tmpdir, + ] + try: + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) + modules = [ + (test_ecs_path, 'toy/0.0'), # not included in PR + (test_ecs_path, 'GCC/4.9.2'), # not included in PR + (tmpdir, 'hwloc/1.10.0-GCC-4.9.2'), + (tmpdir, 'numactl/2.0.10-GCC-4.9.2'), + (tmpdir, 'OpenMPI/1.8.4-GCC-4.9.2'), + (tmpdir, 'gompi/2015a'), + (test_ecs_path, 'GCC/4.6.3'), # not included in PR + ] + for path_prefix, module in modules: + ec_fn = "%s.eb" % '-'.join(module.split('/')) + regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path_prefix, ec_fn, module), re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + + # make sure that *only* these modules are listed, no others + regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M) + self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules)) + + except URLError, err: + print "Ignoring URLError '%s' in test_from_pr" % err + shutil.rmtree(tmpdir) def test_no_such_software(self): """Test using no arguments.""" @@ -676,30 +929,33 @@ def test_no_such_software(self): outtxt = self.eb_main(args) # error message when template is not found - error_msg1 = "ERROR .* No easyconfig files found for software nosuchsoftware, and no templates available. I'm all out of ideas." + error_msg1 = "ERROR No easyconfig files found for software nosuchsoftware, and no templates available. " + error_msg1 += "I'm all out of ideas." # error message when template is found - error_msg2 = "ERROR .* Unable to find an easyconfig for the given specifications" + error_msg2 = "ERROR Unable to find an easyconfig for the given specifications" msg = "Error message when eb can't find software with specified name (outtxt: %s)" % outtxt self.assertTrue(re.search(error_msg1, outtxt) or re.search(error_msg2, outtxt), msg) def test_footer(self): """Test specifying a module footer.""" - # use temporary paths for build/install paths, make sure sources can be found - buildpath = tempfile.mkdtemp() - installpath = tempfile.mkdtemp() - tmpdir = tempfile.mkdtemp() - sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'sources') # create file containing modules footer - module_footer_txt = '\n'.join([ - "# test footer", - "setenv SITE_SPECIFIC_ENV_VAR foobar", - ]) + if get_module_syntax() == 'Tcl': + module_footer_txt = '\n'.join([ + "# test footer", + "setenv SITE_SPECIFIC_ENV_VAR foobar", + ]) + elif get_module_syntax() == 'Lua': + module_footer_txt = '\n'.join([ + "-- test footer", + 'setenv("SITE_SPECIFIC_ENV_VAR", "foobar")', + ]) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + fd, modules_footer = tempfile.mkstemp(prefix='modules-footer-') os.close(fd) - f = open(modules_footer, 'w') - f.write(module_footer_txt) - f.close() + write_file(modules_footer, module_footer_txt) # use toy-0.0.eb easyconfig file that comes with the tests eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb') @@ -707,69 +963,56 @@ def test_footer(self): # check log message with --skip for existing module args = [ eb_file, - '--sourcepath=%s' % sourcepath, - '--buildpath=%s' % buildpath, - '--installpath=%s' % installpath, + '--sourcepath=%s' % self.test_sourcepath, + '--buildpath=%s' % self.test_buildpath, + '--installpath=%s' % self.test_installpath, '--debug', '--force', '--modules-footer=%s' % modules_footer, ] self.eb_main(args, do_build=True) - toy_module = os.path.join(installpath, 'modules', 'all', 'toy', '0.0') + toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_module += '.lua' toy_module_txt = read_file(toy_module) - footer_regex = re.compile(r'%s$' % module_footer_txt, re.M) + footer_regex = re.compile(r'%s$' % module_footer_txt.replace('(', '\\(').replace(')', '\\)'), re.M) msg = "modules footer '%s' is present in '%s'" % (module_footer_txt, toy_module_txt) self.assertTrue(footer_regex.search(toy_module_txt), msg) # cleanup - shutil.rmtree(buildpath) - shutil.rmtree(installpath) - shutil.rmtree(tmpdir) os.remove(modules_footer) def test_recursive_module_unload(self): """Test generating recursively unloading modules.""" - # use temporary paths for build/install paths, make sure sources can be found - buildpath = tempfile.mkdtemp() - installpath = tempfile.mkdtemp() - tmpdir = tempfile.mkdtemp() - sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'sources') - # use toy-0.0.eb easyconfig file that comes with the tests eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0-deps.eb') # check log message with --skip for existing module args = [ eb_file, - '--sourcepath=%s' % sourcepath, - '--buildpath=%s' % buildpath, - '--installpath=%s' % installpath, + '--sourcepath=%s' % self.test_sourcepath, + '--buildpath=%s' % self.test_buildpath, + '--installpath=%s' % self.test_installpath, '--debug', '--force', '--recursive-module-unload', ] self.eb_main(args, do_build=True, verbose=True) - toy_module = os.path.join(installpath, 'modules', 'all', 'toy', '0.0-deps') + toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-deps') + if get_module_syntax() == 'Lua': + toy_module += '.lua' toy_module_txt = read_file(toy_module) is_loaded_regex = re.compile(r"if { !\[is-loaded gompi/1.3.12\] }", re.M) self.assertFalse(is_loaded_regex.search(toy_module_txt), "Recursive unloading is used: %s" % toy_module_txt) - # cleanup - shutil.rmtree(buildpath) - shutil.rmtree(installpath) - shutil.rmtree(tmpdir) - def test_tmpdir(self): """Test setting temporary directory to use by EasyBuild.""" # use temporary paths for build/install paths, make sure sources can be found - buildpath = tempfile.mkdtemp() - installpath = tempfile.mkdtemp() tmpdir = tempfile.mkdtemp() - sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'sources') # use toy-0.0.eb easyconfig file that comes with the tests eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb') @@ -777,35 +1020,34 @@ def test_tmpdir(self): # check log message with --skip for existing module args = [ eb_file, - '--sourcepath=%s' % sourcepath, - '--buildpath=%s' % buildpath, - '--installpath=%s' % installpath, + '--sourcepath=%s' % self.test_sourcepath, + '--buildpath=%s' % self.test_buildpath, + '--installpath=%s' % self.test_installpath, '--debug', '--tmpdir=%s' % tmpdir, ] - outtxt = self.eb_main(args, do_build=True) + outtxt = self.eb_main(args, do_build=True, reset_env=False) - tmpdir_msg = r"Using %s\S+ as temporary directory" % os.path.join(tmpdir, 'easybuild-') + tmpdir_msg = r"Using %s\S+ as temporary directory" % os.path.join(tmpdir, 'eb-') found = re.search(tmpdir_msg, outtxt, re.M) self.assertTrue(found, "Log message for tmpdir found in outtxt: %s" % outtxt) for var in ['TMPDIR', 'TEMP', 'TMP']: - self.assertTrue(os.environ[var].startswith(os.path.join(tmpdir, 'easybuild-'))) - self.assertTrue(tempfile.gettempdir().startswith(os.path.join(tmpdir, 'easybuild-'))) + self.assertTrue(os.environ[var].startswith(os.path.join(tmpdir, 'eb-'))) + self.assertTrue(tempfile.gettempdir().startswith(os.path.join(tmpdir, 'eb-'))) tempfile_tmpdir = tempfile.mkdtemp() - self.assertTrue(tempfile_tmpdir.startswith(os.path.join(tmpdir, 'easybuild-'))) + self.assertTrue(tempfile_tmpdir.startswith(os.path.join(tmpdir, 'eb-'))) fd, tempfile_tmpfile = tempfile.mkstemp() - self.assertTrue(tempfile_tmpfile.startswith(os.path.join(tmpdir, 'easybuild-'))) + self.assertTrue(tempfile_tmpfile.startswith(os.path.join(tmpdir, 'eb-'))) # cleanup - shutil.rmtree(buildpath) - shutil.rmtree(installpath) os.close(fd) shutil.rmtree(tmpdir) def test_ignore_osdeps(self): """Test ignoring of listed OS dependencies.""" txt = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -886,6 +1128,10 @@ def test_experimental(self): def test_deprecated(self): """Test the deprecated option""" + if 'EASYBUILD_DEPRECATED' in os.environ: + os.environ['EASYBUILD_DEPRECATED'] = str(VERSION) + init_config() + orig_value = easybuild.tools.build_log.CURRENT_VERSION # make sure it's off by default @@ -913,7 +1159,7 @@ def test_deprecated(self): except easybuild.tools.build_log.EasyBuildError, err2: self.assertTrue('DEPRECATED' in str(err2)) - # force higher version by prefixing it with 1, which should result in deprecation + # force higher version by prefixing it with 1, which should result in deprecation errors topt = EasyBuildOptions( go_args=['--deprecated=1%s' % orig_value], ) @@ -942,6 +1188,7 @@ def test_allow_modules_tool_mismatch(self): args = [ ec_file, '--modules-tool=MockModulesTool', + '--module-syntax=Tcl', # Lua would require Lmod ] self.eb_main(args, do_build=True) outtxt = read_file(self.logfile) @@ -953,6 +1200,7 @@ def test_allow_modules_tool_mismatch(self): args = [ ec_file, '--modules-tool=MockModulesTool', + '--module-syntax=Tcl', # Lua would require Lmod '--allow-modules-tool-mismatch', ] self.eb_main(args, do_build=True) @@ -965,6 +1213,7 @@ def test_allow_modules_tool_mismatch(self): args = [ ec_file, '--modules-tool=MockModulesTool', + '--module-syntax=Tcl', # Lua would require Lmod '--debug', ] self.eb_main(args, do_build=True) @@ -978,13 +1227,71 @@ def test_allow_modules_tool_mismatch(self): else: del os.environ['module'] + def test_try(self): + """Test whether --try options are taken into account.""" + ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + tweaked_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') + shutil.copy2(os.path.join(ecs_path, 'toy-0.0.eb'), tweaked_toy_ec) + f = open(tweaked_toy_ec, 'a') + f.write("easyblock = 'ConfigureMake'") + f.close() + + args = [ + tweaked_toy_ec, + '--sourcepath=%s' % self.test_sourcepath, + '--buildpath=%s' % self.test_buildpath, + '--installpath=%s' % self.test_installpath, + '--dry-run', + '--robot=%s' % ecs_path, + ] + + test_cases = [ + ([], 'toy/0.0'), + (['--try-software=foo,1.2.3', '--try-toolchain=gompi,1.4.10'], 'foo/1.2.3-gompi-1.4.10'), + (['--try-toolchain-name=gompi', '--try-toolchain-version=1.4.10'], 'toy/0.0-gompi-1.4.10'), + # --try-toolchain is overridden by --toolchain + (['--try-toolchain=gompi,1.3.12', '--toolchain=dummy,dummy'], 'toy/0.0'), + (['--try-software-name=foo', '--try-software-version=1.2.3'], 'foo/1.2.3'), + (['--try-toolchain-name=gompi', '--try-toolchain-version=1.4.10'], 'toy/0.0-gompi-1.4.10'), + (['--try-software-version=1.2.3', '--try-toolchain=gompi,1.4.10'], 'toy/1.2.3-gompi-1.4.10'), + (['--try-amend=versionsuffix=-test'], 'toy/0.0-test'), + # --try-amend is overridden by --amend + (['--amend=versionsuffix=', '--try-amend=versionsuffix=-test'], 'toy/0.0'), + (['--try-toolchain=gompi,1.3.12', '--toolchain=dummy,dummy'], 'toy/0.0'), + # tweak existing list-typed value (patches) + (['--try-amend=versionsuffix=-test2', '--try-amend=patches=1.patch,2.patch'], 'toy/0.0-test2'), + # append to existing list-typed value (patches) + (['--try-amend=versionsuffix=-test3', '--try-amend=patches=,extra.patch'], 'toy/0.0-test3'), + # prepend to existing list-typed value (patches) + (['--try-amend=versionsuffix=-test4', '--try-amend=patches=extra.patch,'], 'toy/0.0-test4'), + # define extra list-typed parameter + (['--try-amend=versionsuffix=-test5', '--try-amend=exts_list=1,2,3'], 'toy/0.0-test5'), + # only --try causes other build specs to be included too + (['--try-software=foo,1.2.3', '--toolchain=gompi,1.4.10'], 'foo/1.2.3-gompi-1.4.10'), + (['--software=foo,1.2.3', '--try-toolchain=gompi,1.4.10'], 'foo/1.2.3-gompi-1.4.10'), + (['--software=foo,1.2.3', '--try-amend=versionsuffix=-test'], 'foo/1.2.3-test'), + ] + + for extra_args, mod in test_cases: + outtxt = self.eb_main(args + extra_args, verbose=True, raise_error=True) + mod_regex = re.compile("\(module: %s\)$" % mod, re.M) + self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) + + for extra_arg in ['--try-software=foo', '--try-toolchain=gompi', '--try-toolchain=gomp,1.4.10,-no-OFED']: + allargs = args + [extra_arg] + self.assertErrorRegex(EasyBuildError, "problems validating the options", self.eb_main, allargs, raise_error=True) + + # no --try used, so no tweaked easyconfig files are generated + allargs = args + ['--software-version=1.2.3', '--toolchain=gompi,1.4.10'] + self.assertErrorRegex(EasyBuildError, "version .* not available", self.eb_main, allargs, raise_error=True) + def test_recursive_try(self): """Test whether recursive --try-X works.""" - ecs_path = os.path.join(os.path.dirname(__file__), 'easyconfigs') + ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') tweaked_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') shutil.copy2(os.path.join(ecs_path, 'toy-0.0.eb'), tweaked_toy_ec) f = open(tweaked_toy_ec, 'a') - f.write("dependencies = [('gzip', '1.4')]") # add fictious dependency + f.write("dependencies = [('gzip', '1.4')]\n") # add fictious dependency f.close() sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'sources') @@ -998,18 +1305,44 @@ def test_recursive_try(self): '--ignore-osdeps', '--dry-run', ] - outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True) - # toolchain gompi/1.4.10 should be listed - tc_regex = re.compile("^\s*\*\s*\[.\]\s*\S*%s/gompi-1.4.10.eb\s\(module: gompi/1.4.10\)\s*$" % ecs_path, re.M) - self.assertTrue(tc_regex.search(outtxt), "Pattern %s found in %s" % (tc_regex.pattern, outtxt)) + for extra_args in [[], ['--module-naming-scheme=HierarchicalMNS']]: + + outtxt = self.eb_main(args + extra_args, verbose=True, raise_error=True) + + # toolchain gompi/1.4.10 should be listed (but not present yet) + if extra_args: + mark = 'x' + else: + mark = ' ' + tc_regex = re.compile("^ \* \[%s\] %s/gompi-1.4.10.eb \(module: .*gompi/1.4.10\)$" % (mark, ecs_path), re.M) + self.assertTrue(tc_regex.search(outtxt), "Pattern %s found in %s" % (tc_regex.pattern, outtxt)) + + # both toy and gzip dependency should be listed with gompi/1.4.10 toolchain + for ec_name in ['gzip-1.4', 'toy-0.0']: + ec = '%s-gompi-1.4.10.eb' % ec_name + if extra_args: + mod = ec_name.replace('-', '/') + else: + mod = '%s-gompi-1.4.10' % ec_name.replace('-', '/') + mod_regex = re.compile("^ \* \[ \] \S+/eb-\S+/%s \(module: .*%s\)$" % (ec, mod), re.M) + #mod_regex = re.compile("%s \(module: .*%s\)$" % (ec, mod), re.M) + self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) + + # clear fictious dependency + f = open(tweaked_toy_ec, 'a') + f.write("dependencies = []\n") + f.close() - # both toy and gzip dependency should be listed with gompi/1.4.10 toolchain - for ec_name in ['gzip-1.4', 'toy-0.0']: - ec = '%s-gompi-1.4.10.eb' % ec_name - mod = '%s-gompi-1.4.10' % ec_name.replace('-', '/') - mod_regex = re.compile("^\s*\*\s*\[.\]\s*\S*/easybuild-\S*/%s\s\(module: %s\)\s*$" % (ec, mod), re.M) - self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) + # no recursive try if --(try-)software(-X) is involved + for extra_args in [['--try-software-version=1.2.3'], ['--software-version=1.2.3']]: + outtxt = self.eb_main(args + extra_args, raise_error=True) + for mod in ['toy/1.2.3-gompi-1.4.10', 'gompi/1.4.10', 'GCC/4.7.2']: + mod_regex = re.compile("\(module: %s\)$" % mod, re.M) + self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) + for mod in ['gompi/1.2.3', 'GCC/1.2.3']: + mod_regex = re.compile("\(module: %s\)$" % mod, re.M) + self.assertFalse(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) def test_cleanup_builddir(self): """Test cleaning up of build dir and --disable-cleanup-builddir.""" @@ -1034,7 +1367,7 @@ def test_cleanup_builddir(self): args = [ toy_ec, '--force', - '--try-amend=premakeopts=nosuchcommand &&', + '--try-amend=prebuildopts=nosuchcommand &&', ] self.eb_main(args, do_build=True) self.assertTrue(os.path.exists(toy_buildpath), "Build dir %s is retained after failed build" % toy_buildpath) @@ -1065,22 +1398,55 @@ def test_filter_deps(self): self.assertFalse(re.search('module: FFTW/3.3.3-gompi', outtxt)) self.assertFalse(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt)) self.assertFalse(re.search('module: zlib', outtxt)) - + + def test_hide_deps(self): + """Test use of --hide-deps.""" + test_dir = os.path.dirname(os.path.abspath(__file__)) + ec_file = os.path.join(test_dir, 'easyconfigs', 'goolf-1.4.10.eb') + os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules') + args = [ + ec_file, + '--buildpath=%s' % self.test_buildpath, + '--installpath=%s' % self.test_installpath, + '--robot=%s' % os.path.join(test_dir, 'easyconfigs'), + '--dry-run', + ] + outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True) + self.assertTrue(re.search('module: GCC/4.7.2', outtxt)) + self.assertTrue(re.search('module: OpenMPI/1.6.4-GCC-4.7.2', outtxt)) + self.assertTrue(re.search('module: OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', outtxt)) + self.assertTrue(re.search('module: FFTW/3.3.3-gompi', outtxt)) + self.assertTrue(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt)) + # zlib is not a dep at all + self.assertFalse(re.search('module: zlib', outtxt)) + + # clear log file + open(self.logfile, 'w').write('') + + # filter deps (including a non-existing dep, i.e. zlib) + args.append('--hide-deps=FFTW,ScaLAPACK,zlib') + outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True) + self.assertTrue(re.search('module: GCC/4.7.2', outtxt)) + self.assertTrue(re.search('module: OpenMPI/1.6.4-GCC-4.7.2', outtxt)) + self.assertTrue(re.search('module: OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', outtxt)) + self.assertFalse(re.search(r'module: FFTW/3\.3\.3-gompi', outtxt)) + self.assertTrue(re.search(r'module: FFTW/\.3\.3\.3-gompi', outtxt)) + self.assertFalse(re.search(r'module: ScaLAPACK/2\.0\.2-gompi', outtxt)) + self.assertTrue(re.search(r'module: ScaLAPACK/\.2\.0\.2-gompi', outtxt)) + # zlib is not a dep at all + self.assertFalse(re.search(r'module: zlib', outtxt)) + def test_test_report_env_filter(self): """Test use of --test-report-env-filter.""" - sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'sources') - def toy(extra_args=None): """Build & install toy, return contents of test report.""" - buildpath = tempfile.mkdtemp() - installpath = tempfile.mkdtemp() eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb') args = [ eb_file, - '--sourcepath=%s' % sourcepath, - '--buildpath=%s' % buildpath, - '--installpath=%s' % installpath, + '--sourcepath=%s' % self.test_sourcepath, + '--buildpath=%s' % self.test_buildpath, + '--installpath=%s' % self.test_installpath, '--force', '--debug', ] @@ -1088,7 +1454,7 @@ def toy(extra_args=None): args.extend(extra_args) self.eb_main(args, do_build=True, raise_error=True, verbose=True) - software_path = os.path.join(installpath, 'software', 'toy', '0.0') + software_path = os.path.join(self.test_installpath, 'software', 'toy', '0.0') test_report_path_pattern = os.path.join(software_path, 'easybuild', 'easybuild-toy-0.0*test_report.md') f = open(glob.glob(test_report_path_pattern)[0], 'r') test_report_txt = f.read() @@ -1119,6 +1485,420 @@ def toy(extra_args=None): tup = (filter_arg_regex.pattern, test_report_txt) self.assertTrue(filter_arg_regex.search(test_report_txt), "%s in %s" % tup) + def test_robot(self): + """Test --robot and --robot-paths command line options.""" + # unset $EASYBUILD_ROBOT_PATHS that was defined in setUp + os.environ['EASYBUILD_ROBOT_PATHS'] = self.test_prefix + + test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + eb_file = os.path.join(test_ecs_path, 'gzip-1.4-GCC-4.6.3.eb') # includes 'toy/.0.0-deps' as a dependency + + # hide test modules + self.reset_modulepath([]) + + # dependency resolution is disabled by default, even if required paths are available + args = [ + eb_file, + '--robot-paths=%s' % test_ecs_path, + ] + error_regex = 'no module .* found for dependency' + self.assertErrorRegex(EasyBuildError, error_regex, self.eb_main, args, raise_error=True, do_build=True) + + # enable robot, but without passing path required to resolve toy dependency => FAIL + args = [ + eb_file, + '--robot', + '--dry-run', + ] + self.assertErrorRegex(EasyBuildError, 'Irresolvable dependencies', self.eb_main, args, raise_error=True) + + # add path to test easyconfigs to robot paths, so dependencies can be resolved + self.eb_main(args + ['--robot-paths=%s' % test_ecs_path], raise_error=True) + + # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory + # to check whether easyconfigs install path is auto-included in robot path + tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') + mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) + shutil.copytree(test_ecs_path, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) + + # prepend path to test easyconfigs into Python search path, so it gets picked up as --robot-paths default + del os.environ['EASYBUILD_ROBOT_PATHS'] + orig_sys_path = sys.path[:] + sys.path.insert(0, tmpdir) + self.eb_main(args, raise_error=True) + + shutil.rmtree(tmpdir) + sys.path[:] = orig_sys_path + + # make sure that paths specified to --robot get preference over --robot-paths + args = [ + eb_file, + '--robot=%s' % test_ecs_path, + '--robot-paths=%s' % os.path.join(tmpdir, 'easybuild', 'easyconfigs'), + '--dry-run', + ] + outtxt = self.eb_main(args, raise_error=True) + + for ecfile in ['GCC-4.6.3.eb', 'ictce-4.1.13.eb', 'toy-0.0-deps.eb', 'gzip-1.4-GCC-4.6.3.eb']: + ec_regex = re.compile(r'^\s\*\s\[[xF ]\]\s%s' % os.path.join(test_ecs_path, ecfile), re.M) + self.assertTrue(ec_regex.search(outtxt), "Pattern %s found in %s" % (ec_regex.pattern, outtxt)) + + def test_missing_cfgfile(self): + """Test behaviour when non-existing config file is specified.""" + args = ['--configfiles=/no/such/cfgfile.foo'] + error_regex = "parseconfigfiles: configfile .* not found" + self.assertErrorRegex(EasyBuildError, error_regex, self.eb_main, args, raise_error=True) + + def test_show_default_moduleclasses(self): + """Test --show-default-moduleclasses.""" + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + args = [ + '--unittest-file=%s' % self.logfile, + '--show-default-moduleclasses', + ] + write_file(self.logfile, '') + self.eb_main(args, logfile=dummylogfn, verbose=True) + logtxt = read_file(self.logfile) + + lst = ["\t%s:[ ]*%s" % (c, d.replace('(', '\\(').replace(')', '\\)')) for (c, d) in DEFAULT_MODULECLASSES] + regex = re.compile("Default available module classes:\n\n" + '\n'.join(lst), re.M) + + self.assertTrue(regex.search(logtxt), "Pattern '%s' found in %s" % (regex.pattern, logtxt)) + + def test_show_default_configfiles(self): + """Test --show-default-configfiles.""" + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + home = os.environ['HOME'] + for envvar in ['XDG_CONFIG_DIRS', 'XDG_CONFIG_HOME']: + if envvar in os.environ: + del os.environ[envvar] + reload(easybuild.tools.options) + + args = [ + '--unittest-file=%s' % self.logfile, + '--show-default-configfiles', + ] + + cfgtxt = '\n'.join([ + '[config]', + 'prefix = %s' % self.test_prefix, + ]) + + expected_tmpl = '\n'.join([ + "Default list of configuration files:", + '', + "[with $XDG_CONFIG_HOME: %s, $XDG_CONFIG_DIRS: %s]", + '', + "* user-level: ${XDG_CONFIG_HOME:-$HOME/.config}/easybuild/config.cfg", + " -> %s", + "* system-level: ${XDG_CONFIG_DIRS:-/etc}/easybuild.d/*.cfg", + " -> %s/easybuild.d/*.cfg => ", + ]) + + write_file(self.logfile, '') + self.eb_main(args, logfile=dummylogfn, verbose=True) + logtxt = read_file(self.logfile) + + homecfgfile = os.path.join(os.environ['HOME'], '.config', 'easybuild', 'config.cfg') + homecfgfile_str = homecfgfile + if os.path.exists(homecfgfile): + homecfgfile_str += " => found" + else: + homecfgfile_str += " => not found" + expected = expected_tmpl % ('(not set)', '(not set)', homecfgfile_str, '{/etc}') + self.assertTrue(expected in logtxt) + + # to predict the full output, we need to take control over $HOME and $XDG_CONFIG_DIRS + os.environ['HOME'] = self.test_prefix + xdg_config_dirs = os.path.join(self.test_prefix, 'etc') + os.environ['XDG_CONFIG_DIRS'] = xdg_config_dirs + + expected_tmpl += '\n'.join([ + "%s", + '', + "Default list of existing configuration files (%d): %s", + ]) + + # put dummy cfgfile in place in $HOME (to predict last line of output which only lists *existing* files) + mkdir(os.path.join(self.test_prefix, '.config', 'easybuild'), parents=True) + homecfgfile = os.path.join(self.test_prefix, '.config', 'easybuild', 'config.cfg') + write_file(homecfgfile, cfgtxt) + + reload(easybuild.tools.options) + write_file(self.logfile, '') + self.eb_main(args, logfile=dummylogfn, verbose=True) + logtxt = read_file(self.logfile) + expected = expected_tmpl % ('(not set)', xdg_config_dirs, "%s => found" % homecfgfile, '{%s}' % xdg_config_dirs, + '(no matches)', 1, homecfgfile) + self.assertTrue(expected in logtxt) + + xdg_config_home = os.path.join(self.test_prefix, 'home') + os.environ['XDG_CONFIG_HOME'] = xdg_config_home + xdg_config_dirs = [os.path.join(self.test_prefix, 'etc'), os.path.join(self.test_prefix, 'moaretc')] + os.environ['XDG_CONFIG_DIRS'] = os.pathsep.join(xdg_config_dirs) + + # put various dummy cfgfiles in place + cfgfiles = [ + os.path.join(self.test_prefix, 'etc', 'easybuild.d', 'config.cfg'), + os.path.join(self.test_prefix, 'moaretc', 'easybuild.d', 'bar.cfg'), + os.path.join(self.test_prefix, 'moaretc', 'easybuild.d', 'foo.cfg'), + os.path.join(xdg_config_home, 'easybuild', 'config.cfg'), + ] + for cfgfile in cfgfiles: + mkdir(os.path.dirname(cfgfile), parents=True) + write_file(cfgfile, cfgtxt) + reload(easybuild.tools.options) + + write_file(self.logfile, '') + self.eb_main(args, logfile=dummylogfn, verbose=True) + logtxt = read_file(self.logfile) + expected = expected_tmpl % (xdg_config_home, os.pathsep.join(xdg_config_dirs), + "%s => found" % os.path.join(xdg_config_home, 'easybuild', 'config.cfg'), + '{' + ', '.join(xdg_config_dirs) + '}', + ', '.join(cfgfiles[:-1]), 4, ', '.join(cfgfiles)) + self.assertTrue(expected in logtxt) + + del os.environ['XDG_CONFIG_DIRS'] + del os.environ['XDG_CONFIG_HOME'] + os.environ['HOME'] = home + reload(easybuild.tools.options) + + def test_generate_cmd_line(self): + """Test for generate_cmd_line.""" + ebopts = EasyBuildOptions() + self.assertEqual(ebopts.generate_cmd_line(), []) + + ebopts = EasyBuildOptions(go_args=['--force']) + self.assertEqual(ebopts.generate_cmd_line(), ['--force']) + + ebopts = EasyBuildOptions(go_args=['--search=bar', '--search', 'foobar']) + self.assertEqual(ebopts.generate_cmd_line(), ['--search=foobar']) + + def test_include_easyblocks(self): + """Test --include-easyblocks.""" + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + # clear log + write_file(self.logfile, '') + + # existing test EB_foo easyblock found without include a custom one + args = [ + '--list-easyblocks=detailed', + '--unittest-file=%s' % self.logfile, + ] + self.eb_main(args, logfile=dummylogfn, raise_error=True) + logtxt = read_file(self.logfile) + + test_easyblocks = os.path.dirname(os.path.abspath(__file__)) + path_pattern = os.path.join(test_easyblocks, 'sandbox', 'easybuild', 'easyblocks', 'f', 'foo.py') + foo_regex = re.compile(r"^\|-- EB_foo \(easybuild.easyblocks.foo @ %s\)" % path_pattern, re.M) + self.assertTrue(foo_regex.search(logtxt), "Pattern '%s' found in: %s" % (foo_regex.pattern, logtxt)) + + # 'undo' import of foo easyblock + del sys.modules['easybuild.easyblocks.foo'] + + # include extra test easyblocks + foo_txt = '\n'.join([ + 'from easybuild.framework.easyblock import EasyBlock', + 'class EB_foo(EasyBlock):', + ' pass', + '' + ]) + write_file(os.path.join(self.test_prefix, 'foo.py'), foo_txt) + + # clear log + write_file(self.logfile, '') + + args = [ + '--include-easyblocks=%s/*.py' % self.test_prefix, + '--list-easyblocks=detailed', + '--unittest-file=%s' % self.logfile, + ] + self.eb_main(args, logfile=dummylogfn, raise_error=True) + logtxt = read_file(self.logfile) + + path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks', 'easybuild', 'easyblocks', 'foo.py') + foo_regex = re.compile(r"^\|-- EB_foo \(easybuild.easyblocks.foo @ %s\)" % path_pattern, re.M) + self.assertTrue(foo_regex.search(logtxt), "Pattern '%s' found in: %s" % (foo_regex.pattern, logtxt)) + + # 'undo' import of foo easyblock + del sys.modules['easybuild.easyblocks.foo'] + + def test_include_module_naming_schemes(self): + """Test --include-module-naming-schemes.""" + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + # clear log + write_file(self.logfile, '') + + mns_regex = re.compile(r'^\s*TestIncludedMNS', re.M) + + # TestIncludedMNS module naming scheme is not available by default + args = [ + '--avail-module-naming-schemes', + '--unittest-file=%s' % self.logfile, + ] + self.eb_main(args, logfile=dummylogfn, raise_error=True) + logtxt = read_file(self.logfile) + self.assertFalse(mns_regex.search(logtxt), "Unexpected pattern '%s' found in: %s" % (mns_regex.pattern, logtxt)) + + # include extra test MNS + mns_txt = '\n'.join([ + 'from easybuild.tools.module_naming_scheme import ModuleNamingScheme', + 'class TestIncludedMNS(ModuleNamingScheme):', + ' pass', + ]) + write_file(os.path.join(self.test_prefix, 'test_mns.py'), mns_txt) + + # clear log + write_file(self.logfile, '') + + args = [ + '--avail-module-naming-schemes', + '--include-module-naming-schemes=%s/*.py' % self.test_prefix, + '--unittest-file=%s' % self.logfile, + ] + self.eb_main(args, logfile=dummylogfn, raise_error=True) + logtxt = read_file(self.logfile) + self.assertTrue(mns_regex.search(logtxt), "Pattern '%s' *not* found in: %s" % (mns_regex.pattern, logtxt)) + + # undo successful import + del sys.modules['easybuild.tools.module_naming_scheme.test_mns'] + + def test_use_included_module_naming_scheme(self): + """Test using an included module naming scheme.""" + # try selecting the added module naming scheme + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + # include extra test MNS + mns_txt = '\n'.join([ + 'import os', + 'from easybuild.tools.module_naming_scheme import ModuleNamingScheme', + 'class AnotherTestIncludedMNS(ModuleNamingScheme):', + ' def det_full_module_name(self, ec):', + " return os.path.join(ec['name'], ec['version'])", + ]) + write_file(os.path.join(self.test_prefix, 'test_mns.py'), mns_txt) + + eb_file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'toy-0.0.eb') + args = [ + '--unittest-file=%s' % self.logfile, + '--module-naming-scheme=AnotherTestIncludedMNS', + '--force', + eb_file, + ] + + # selecting a module naming scheme that doesn't exist leads to 'invalid choice' + error_regex = "Selected module naming scheme \'AnotherTestIncludedMNS\' is unknown" + self.assertErrorRegex(EasyBuildError, error_regex, self.eb_main, args, logfile=dummylogfn, + raise_error=True, raise_systemexit=True) + + args.append('--include-module-naming-schemes=%s/*.py' % self.test_prefix) + self.eb_main(args, logfile=dummylogfn, do_build=True, raise_error=True, raise_systemexit=True, verbose=True) + toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_mod += '.lua' + self.assertTrue(os.path.exists(toy_mod), "Found %s" % toy_mod) + + def test_include_toolchains(self): + """Test --include-toolchains.""" + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + # clear log + write_file(self.logfile, '') + + # set processed attribute to false, to trigger rescan in search_toolchain + setattr(easybuild.tools.toolchain, '%s_PROCESSED' % TC_CONST_PREFIX, False) + + tc_regex = re.compile(r'^\s*test_included_toolchain: TestIncludedCompiler', re.M) + + # TestIncludedCompiler is not available by default + args = [ + '--list-toolchains', + '--unittest-file=%s' % self.logfile, + ] + self.eb_main(args, logfile=dummylogfn, raise_error=True) + logtxt = read_file(self.logfile) + self.assertFalse(tc_regex.search(logtxt), "Pattern '%s' *not* found in: %s" % (tc_regex.pattern, logtxt)) + + # include extra test toolchain + comp_txt = '\n'.join([ + 'from easybuild.tools.toolchain.compiler import Compiler', + 'class TestIncludedCompiler(Compiler):', + " COMPILER_MODULE_NAME = ['TestIncludedCompiler']", + ]) + mkdir(os.path.join(self.test_prefix, 'compiler')) + write_file(os.path.join(self.test_prefix, 'compiler', 'test_comp.py'), comp_txt) + + tc_txt = '\n'.join([ + 'from easybuild.toolchains.compiler.test_comp import TestIncludedCompiler', + 'class TestIncludedToolchain(TestIncludedCompiler):', + " NAME = 'test_included_toolchain'", + ]) + write_file(os.path.join(self.test_prefix, 'test_tc.py'), tc_txt) + + args = [ + '--include-toolchains=%s/*.py,%s/*/*.py' % (self.test_prefix, self.test_prefix), + '--list-toolchains', + '--unittest-file=%s' % self.logfile, + ] + self.eb_main(args, logfile=dummylogfn, raise_error=True) + logtxt = read_file(self.logfile) + self.assertTrue(tc_regex.search(logtxt), "Pattern '%s' *not* found in: %s" % (tc_regex.pattern, logtxt)) + + # undo successful import + del sys.modules['easybuild.toolchains.compiler.test_comp'] + del sys.modules['easybuild.toolchains.test_tc'] + + def test_cleanup_tmpdir(self): + """Test --cleanup-tmpdir.""" + args = [ + os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'toy-0.0.eb'), + '--dry-run', + '--try-software-version=1.0', # so we get a tweaked easyconfig + ] + + tmpdir = tempfile.gettempdir() + # just making sure this is empty before we get started + self.assertEqual(os.listdir(tmpdir), []) + + # force silence (since we're not using testing mode) + self.mock_stdout(True) + + # default: cleanup tmpdir & logfile + self.eb_main(args, raise_error=True, testing=False) + self.assertEqual(os.listdir(tmpdir), []) + self.assertFalse(os.path.exists(self.logfile)) + + # disable cleaning up tmpdir + args.append('--disable-cleanup-tmpdir') + self.eb_main(args, raise_error=True, testing=False) + tmpdir_files = os.listdir(tmpdir) + # tmpdir and logfile are still there \o/ + self.assertTrue(len(tmpdir_files) == 1) + self.assertTrue(os.path.exists(self.logfile)) + # tweaked easyconfigs is still there \o/ + tweaked_dir = os.path.join(tmpdir, tmpdir_files[0], 'tweaked_easyconfigs') + self.assertTrue(os.path.exists(os.path.join(tweaked_dir, 'toy-1.0.eb'))) + + def test_review_pr(self): + """Test --review-pr.""" + self.mock_stdout(True) + # PR for zlib 1.2.8 easyconfig, see https://github.com/hpcugent/easybuild-easyconfigs/pull/1484 + self.eb_main(['--review-pr=1484', '--disable-color'], raise_error=True) + txt = self.get_stdout() + self.mock_stdout(False) + self.assertTrue(re.search(r"^Comparing zlib-1.2.8\S* with zlib-1.2.8", txt)) + + def suite(): """ returns all the testcases in this module """ return TestLoader().loadTestsFromTestCase(CommandLineOptionsTest) diff --git a/test/framework/package.py b/test/framework/package.py new file mode 100644 index 0000000000..2ce61a5ce2 --- /dev/null +++ b/test/framework/package.py @@ -0,0 +1,157 @@ +# # +# Copyright 2015-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Unit tests for packaging support. + +@author: Kenneth Hoste (Ghent University) +""" +import os +import re +import stat + +from test.framework.utilities import EnhancedTestCase, init_config +from unittest import TestLoader +from unittest import main as unittestmain + +import easybuild.tools.build_log +from easybuild.framework.easyconfig.easyconfig import EasyConfig +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import adjust_permissions, read_file, write_file +from easybuild.tools.package.utilities import ActivePNS, avail_package_naming_schemes, check_pkg_support, package +from easybuild.tools.version import VERSION as EASYBUILD_VERSION + + +MOCKED_FPM = """#!/bin/bash +# only parse what we need to spit out the expected package file, ignore the rest +workdir=`echo $@ | sed 's/--workdir \([^ ]*\).*/\\1/g'` +name=`echo $@ | sed 's/.* --name \([^ ]*\).*/\\1/g'` +version=`echo $@ | sed 's/.*--version \([^ ]*\).*/\\1/g'` +iteration=`echo $@ | sed 's/.*--iteration \([^ ]*\).*/\\1/g'` +target=`echo $@ | sed 's/.*-t \([^ ]*\).*/\\1/g'` + +args=`echo $@ | sed 's/-[^ ]* [^ ]* //g'` +installdir=`echo $args | cut -d' ' -f1` +modulefile=`echo $args | cut -d' ' -f2` + +pkgfile=${workdir}/${name}-${version}.${iteration}.${target} +echo "thisisan$target" > $pkgfile +echo $@ >> $pkgfile +echo "Contents of installdir $installdir:" >> $pkgfile +ls $installdir >> $pkgfile +echo "Contents of module file $modulefile:" >> $pkgfile +cat $modulefile >> $pkgfile +""" + + +def mock_fpm(tmpdir): + """Put mocked version of fpm command in place in specified tmpdir.""" + # put mocked 'fpm' command in place, just for testing purposes + fpm = os.path.join(tmpdir, 'fpm') + write_file(fpm, MOCKED_FPM) + adjust_permissions(fpm, stat.S_IXUSR, add=True) + + # also put mocked rpmbuild in place + rpmbuild = os.path.join(tmpdir, 'rpmbuild') + write_file(rpmbuild, '#!/bin/bash') # only needs to be there, doesn't need to actually do something... + adjust_permissions(rpmbuild, stat.S_IXUSR, add=True) + + os.environ['PATH'] = '%s:%s' % (tmpdir, os.environ['PATH']) + + +class PackageTest(EnhancedTestCase): + """Tests for packaging support.""" + + def test_avail_package_naming_schemes(self): + """Test avail_package_naming_schemes()""" + self.assertEqual(sorted(avail_package_naming_schemes().keys()), ['EasyBuildPNS']) + + def test_check_pkg_support(self): + """Test check_pkg_support().""" + # hard enable experimental + orig_experimental = easybuild.tools.build_log.EXPERIMENTAL + easybuild.tools.build_log.EXPERIMENTAL = True + + # clear $PATH to make sure fpm/rpmbuild can not be found + os.environ['PATH'] = '' + + self.assertErrorRegex(EasyBuildError, "Selected packaging tool 'fpm' not found", check_pkg_support) + + for binary in ['fpm', 'rpmbuild']: + binpath = os.path.join(self.test_prefix, binary) + write_file(binpath, '#!/bin/bash') + adjust_permissions(binpath, stat.S_IXUSR, add=True) + os.environ['PATH'] = self.test_prefix + + # no errors => support check passes + check_pkg_support() + + # restore + easybuild.tools.build_log.EXPERIMENTAL = orig_experimental + + def test_active_pns(self): + """Test use of ActivePNS.""" + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + ec = EasyConfig(os.path.join(test_easyconfigs, 'OpenMPI-1.6.4-GCC-4.6.4.eb'), validate=False) + + pns = ActivePNS() + + # default: EasyBuild package naming scheme, pkg release 1 + self.assertEqual(pns.name(ec), 'OpenMPI-1.6.4-GCC-4.6.4') + self.assertEqual(pns.version(ec), 'eb-%s' % EASYBUILD_VERSION) + self.assertEqual(pns.release(ec), '1') + + def test_package(self): + """Test package function.""" + init_config(build_options={'silent': True}) + + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + ec = EasyConfig(os.path.join(test_easyconfigs, 'toy-0.0-gompi-1.3.12-test.eb'), validate=False) + + mock_fpm(self.test_prefix) + + # import needs to be done here, since test easyblocks are only included later + from easybuild.easyblocks.toy import EB_toy + easyblock = EB_toy(ec) + + # build & install first + easyblock.run_all_steps(False) + + # package using default packaging configuration (FPM to build RPM packages) + pkgdir = package(easyblock) + + pkgfile = os.path.join(pkgdir, 'toy-0.0-gompi-1.3.12-test-eb-%s.1.rpm' % EASYBUILD_VERSION) + self.assertTrue(os.path.isfile(pkgfile), "Found %s" % pkgfile) + + pkgtxt = read_file(pkgfile) + pkgtxt_regex = re.compile("Contents of installdir %s" % easyblock.installdir) + self.assertTrue(pkgtxt_regex.search(pkgtxt), "Pattern '%s' found in: %s" % (pkgtxt_regex.pattern, pkgtxt)) + +def suite(): + """ returns all the testcases in this module """ + return TestLoader().loadTestsFromTestCase(PackageTest) + + +if __name__ == '__main__': + unittestmain() diff --git a/test/framework/parallelbuild.py b/test/framework/parallelbuild.py index 395b4c127a..e679347c20 100644 --- a/test/framework/parallelbuild.py +++ b/test/framework/parallelbuild.py @@ -28,14 +28,38 @@ @author: Kenneth Hoste (Ghent University) """ import os +import re +import stat from test.framework.utilities import EnhancedTestCase, init_config from unittest import TestLoader, main from vsc.utils.fancylogger import setLogLevelDebug, logToScreen -from easybuild.framework.easyconfig.tools import process_easyconfig, resolve_dependencies -from easybuild.tools import config, parallelbuild -from easybuild.tools.parallelbuild import PbsJob, build_easyconfigs_in_parallel - +from easybuild.framework.easyconfig.tools import process_easyconfig +from easybuild.tools import config +from easybuild.tools.filetools import adjust_permissions, mkdir, which, write_file +from easybuild.tools.job import pbs_python +from easybuild.tools.job.pbs_python import PbsPython +from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel, submit_jobs +from easybuild.tools.robot import resolve_dependencies + + +# test GC3Pie configuration with large resource specs +GC3PIE_LOCAL_CONFIGURATION = """[resource/ebtestlocalhost] +enabled = yes +type = shellcmd +frontend = localhost +transport = local +max_cores_per_job = 1 +max_memory_per_core = 1000GiB +max_walltime = 1000 hours +# this doubles as "maximum concurrent jobs" +max_cores = 1000 +architecture = x86_64 +auth = none +override = no +resourcedir = %(resourcedir)s +time_cmd = %(time)s +""" def mock(*args, **kwargs): """Function used for mocking several functions imported in parallelbuild module.""" @@ -48,9 +72,10 @@ def __init__(self, *args, **kwargs): self.deps = [] self.jobid = None self.clean_conn = None + self.script = args[1] - def add_dependencies(self, *args, **kwargs): - pass + def add_dependencies(self, jobs): + self.deps.extend(jobs) def cleanup(self, *args, **kwargs): pass @@ -58,34 +83,132 @@ def cleanup(self, *args, **kwargs): def has_holds(self, *args, **kwargs): pass - def submit(self, *args, **kwargs): + def _submit(self, *args, **kwargs): pass class ParallelBuildTest(EnhancedTestCase): """ Testcase for run module """ - def setUp(self): - """Set up testcase.""" - super(ParallelBuildTest, self).setUp() + def test_build_easyconfigs_in_parallel_pbs_python(self): + """Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job.""" + # put mocked functions in place + PbsPython__init__ = PbsPython.__init__ + PbsPython_check_version = PbsPython._check_version + PbsPython_complete = PbsPython.complete + PbsPython_connect_to_server = PbsPython.connect_to_server + PbsPython_ppn = PbsPython.ppn + pbs_python_PbsJob = pbs_python.PbsJob + + PbsPython.__init__ = lambda self: PbsPython__init__(self, pbs_server='localhost') + PbsPython._check_version = lambda _: True + PbsPython.complete = mock + PbsPython.connect_to_server = mock + PbsPython.ppn = mock + pbs_python.PbsJob = MockPbsJob + build_options = { + 'external_modules_metadata': {}, 'robot_path': os.path.join(os.path.dirname(__file__), 'easyconfigs'), 'valid_module_classes': config.module_classes(), + 'validate': False, } - init_config(build_options=build_options) + init_config(args=['--job-backend=PbsPython'], build_options=build_options) - # put mocked functions in place - parallelbuild.connect_to_server = mock - parallelbuild.disconnect_from_server = mock - parallelbuild.get_ppn = mock - parallelbuild.PbsJob = MockPbsJob - - def test_build_easyconfigs_in_parallel(self): - """Basic test for build_easyconfigs_in_parallel function.""" - easyconfig_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb') - easyconfigs = process_easyconfig(easyconfig_file, validate=False) + ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb') + easyconfigs = process_easyconfig(ec_file) + ordered_ecs = resolve_dependencies(easyconfigs) + jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False) + self.assertEqual(len(jobs), 8) + regex = re.compile("echo '.*/gzip-1.5-goolf-1.4.10.eb'") + self.assertTrue(regex.search(jobs[-1].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[-1].script)) + + ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'gzip-1.4-GCC-4.6.3.eb') + ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), retain_all_deps=True) + jobs = submit_jobs(ordered_ecs, '', testing=False, prepare_first=False) + + # make sure command is correct, and that --hidden is there when it needs to be + for i, ec in enumerate(ordered_ecs): + if ec['hidden']: + regex = re.compile("eb %s.* --hidden" % ec['spec']) + else: + regex = re.compile("eb %s" % ec['spec']) + self.assertTrue(regex.search(jobs[i].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[i].script)) + + # no deps for GCC/4.6.3 (toolchain) and ictce/4.1.13 (test easyconfig with 'fake' deps) + self.assertEqual(len(jobs[0].deps), 0) + self.assertEqual(len(jobs[1].deps), 0) + + # only dependency for toy/0.0-deps is ictce/4.1.13 (dep marked as external module is filtered out) + self.assertTrue('toy-0.0-deps.eb' in jobs[2].script) + self.assertEqual(len(jobs[2].deps), 1) + self.assertTrue('ictce-4.1.13.eb' in jobs[2].deps[0].script) + + # dependencies for gzip/1.4-GCC-4.6.3: GCC/4.6.3 (toolchain) + toy/.0.0-deps + self.assertTrue('gzip-1.4-GCC-4.6.3.eb' in jobs[3].script) + self.assertEqual(len(jobs[3].deps), 2) + regex = re.compile('toy-0.0-deps.eb\s* --hidden') + self.assertTrue(regex.search(jobs[3].deps[0].script)) + self.assertTrue('GCC-4.6.3.eb' in jobs[3].deps[1].script) + + # restore mocked stuff + PbsPython.__init__ = PbsPython__init__ + PbsPython._check_version = PbsPython_check_version + PbsPython.complete = PbsPython_complete + PbsPython.connect_to_server = PbsPython_connect_to_server + PbsPython.ppn = PbsPython_ppn + pbs_python.PbsJob = pbs_python_PbsJob + + def test_build_easyconfigs_in_parallel_gc3pie(self): + """Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job.""" + try: + import gc3libs + except ImportError: + print "GC3Pie not available, skipping test" + return + + # put GC3Pie config in place to use local host and fork/exec + resourcedir = os.path.join(self.test_prefix, 'gc3pie') + gc3pie_cfgfile = os.path.join(self.test_prefix, 'gc3pie_local.ini') + gc3pie_cfgtxt = GC3PIE_LOCAL_CONFIGURATION % { + 'resourcedir': resourcedir, + 'time': which('time'), + } + write_file(gc3pie_cfgfile, gc3pie_cfgtxt) + + output_dir = os.path.join(self.test_prefix, 'subdir', 'gc3pie_output_dir') + # purposely pre-create output dir, and put a file in it (to check whether GC3Pie tries to rename the output dir) + mkdir(output_dir, parents=True) + write_file(os.path.join(output_dir, 'foo'), 'bar') + # remove write permissions on parent dir of specified output dir, + # to check that GC3Pie does not try to rename the (already existing) output directory... + adjust_permissions(os.path.dirname(output_dir), stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, + add=False, recursive=False) + + build_options = { + 'job_backend_config': gc3pie_cfgfile, + 'job_max_walltime': 24, + 'job_output_dir': output_dir, + 'job_polling_interval': 0.2, # quick polling + 'job_target_resource': 'ebtestlocalhost', + 'robot_path': os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs'), + 'silent': True, + 'valid_module_classes': config.module_classes(), + 'validate': False, + } + options = init_config(args=['--job-backend=GC3Pie'], build_options=build_options) + + ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb') + easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs) - build_easyconfigs_in_parallel("echo %(spec)s", ordered_ecs) + topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + test_easyblocks_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox') + cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (topdir, test_easyblocks_path) + jobs = build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False) + + self.assertTrue(os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')) + self.assertTrue(os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy')) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/repository.py b/test/framework/repository.py index dbb86d9d1f..b1730dc522 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,16 +28,20 @@ @author: Toon Willems (Ghent University) """ import os +import re import shutil import tempfile from test.framework.utilities import EnhancedTestCase from unittest import TestLoader, main +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.repository.filerepo import FileRepository from easybuild.tools.repository.gitrepo import GitRepository +from easybuild.tools.repository.hgrepo import HgRepository from easybuild.tools.repository.svnrepo import SvnRepository from easybuild.tools.repository.repository import init_repository from easybuild.tools.run import run_cmd +from easybuild.tools.version import VERSION class RepositoryTest(EnhancedTestCase): @@ -75,10 +79,14 @@ def test_gitrepo(self): # URL repo = GitRepository(test_repo_url) - repo.init() - self.assertEqual(os.path.basename(repo.wc), 'testrepository') - self.assertTrue(os.path.exists(os.path.join(repo.wc, 'README.md'))) - shutil.rmtree(repo.wc) + try: + repo.init() + self.assertEqual(os.path.basename(repo.wc), 'testrepository') + self.assertTrue(os.path.exists(os.path.join(repo.wc, 'README.md'))) + shutil.rmtree(repo.wc) + except EasyBuildError, err: + print "ignoring failed subtest in test_gitrepo, testing offline?" + self.assertTrue(re.search("pull in working copy .* went wrong", str(err))) # filepath tmpdir = tempfile.mkdtemp() @@ -91,6 +99,11 @@ def test_gitrepo(self): repo.init() toy_ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb') repo.add_easyconfig(toy_ec_file, 'test', '1.0', {}, False) + repo.commit("toy/0.0") + + log_regex = re.compile(r"toy/0.0 with EasyBuild v%s @ .* \(time: .*, user: .*\)" % VERSION, re.M) + logmsg = repo.client.log('HEAD^!') + self.assertTrue(log_regex.search(logmsg), "Pattern '%s' found in %s" % (log_regex.pattern, logmsg)) shutil.rmtree(repo.wc) shutil.rmtree(tmpdir) @@ -112,6 +125,23 @@ def test_svnrepo(self): self.assertTrue(os.path.exists(os.path.join(repo.wc, 'trunk', 'README.md'))) shutil.rmtree(repo.wc) + def test_hgrepo(self): + """Test using HgRepository.""" + # only run this test if pysvn Python module is available + try: + import hglib + except ImportError: + print "(skipping HgRepository test)" + return + + # GitHub also supports SVN + test_repo_url = 'https://kehoste@bitbucket.org/kehoste/testrepository' + + repo = HgRepository(test_repo_url) + repo.init() + self.assertTrue(os.path.exists(os.path.join(repo.wc, 'README'))) + shutil.rmtree(repo.wc) + def test_init_repository(self): """Test use of init_repository function.""" repo = init_repository('FileRepository', self.path) diff --git a/test/framework/robot.py b/test/framework/robot.py index 532d39450b..beb16f219a 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,19 +29,32 @@ """ import os +import re +import shutil +import tempfile from copy import deepcopy from test.framework.utilities import EnhancedTestCase, init_config from unittest import TestLoader from unittest import main as unittestmain import easybuild.framework.easyconfig.tools as ectools -from easybuild.framework.easyconfig.tools import resolve_dependencies, skip_available +import easybuild.tools.robot as robot +from easybuild.framework.easyconfig.tools import skip_available from easybuild.tools import config, modules from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.configobj import ConfigObj +from easybuild.tools.filetools import write_file +from easybuild.tools.github import fetch_github_token +from easybuild.tools.robot import resolve_dependencies from test.framework.utilities import find_full_path + +# test account, for which a token is available +GITHUB_TEST_ACCOUNT = 'easybuild_test' + ORIG_MODULES_TOOL = modules.modules_tool -ORIG_MAIN_MODULES_TOOL = ectools.modules_tool +ORIG_ECTOOLS_MODULES_TOOL = ectools.modules_tool +ORIG_ROBOT_MODULES_TOOL = robot.modules_tool ORIG_MODULE_FUNCTION = os.environ.get('module', None) @@ -56,9 +69,17 @@ class MockModule(modules.ModulesTool): avail_modules = [] def available(self, *args): - """ no module should be available """ + """Dummy implementation of available.""" return self.avail_modules + def show(self, modname): + """Dummy implementation of show, which includes full path to (available or hidden) module files.""" + if modname in self.avail_modules or os.path.basename(modname).startswith('.'): + txt = ' %s:' % os.path.join('/tmp', modname) + else: + txt = 'Module %s not found' % modname + return txt + def mock_module(mod_paths=None): """Get mock module instance.""" return MockModule(mod_paths=mod_paths) @@ -68,19 +89,22 @@ class RobotTest(EnhancedTestCase): """ Testcase for the robot dependency resolution """ def setUp(self): - """Set up everything for a unit test.""" + """Set up test.""" super(RobotTest, self).setUp() + self.github_token = fetch_github_token(GITHUB_TEST_ACCOUNT) + + def test_resolve_dependencies(self): + """ Test with some basic testcases (also check if he can find dependencies inside the given directory """ # replace Modules class with something we have control over config.modules_tool = mock_module ectools.modules_tool = mock_module + robot.modules_tool = mock_module os.environ['module'] = "() { eval `/bin/echo $*`\n}" - self.base_easyconfig_dir = find_full_path(os.path.join("test", "framework", "easyconfigs")) - self.assertTrue(self.base_easyconfig_dir) + base_easyconfig_dir = find_full_path(os.path.join("test", "framework", "easyconfigs")) + self.assertTrue(base_easyconfig_dir) - def test_resolve_dependencies(self): - """ Test with some basic testcases (also check if he can find dependencies inside the given directory """ easyconfig = { 'spec': '_', 'full_mod_name': 'name/version', @@ -89,6 +113,7 @@ def test_resolve_dependencies(self): } build_options = { 'allow_modules_tool_mismatch': True, + 'external_modules_metadata': ConfigObj(), 'robot_path': None, 'validate': False, } @@ -112,10 +137,11 @@ def test_resolve_dependencies(self): 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, + 'hidden': False, }], 'parsed': True, } - build_options.update({'robot_path': self.base_easyconfig_dir}) + build_options.update({'robot': True, 'robot_path': base_easyconfig_dir}) init_config(build_options=build_options) res = resolve_dependencies([deepcopy(easyconfig_dep)]) # dependency should be found, order should be correct @@ -123,7 +149,34 @@ def test_resolve_dependencies(self): self.assertEqual('gzip/1.4', res[0]['full_mod_name']) self.assertEqual('foo/1.2.3', res[-1]['full_mod_name']) - # here we have include a Dependency in the easyconfig list + # hidden dependencies are found too, but only retained if they're not available (or forced to be retained + hidden_dep = { + 'name': 'toy', + 'version': '0.0', + 'versionsuffix': '-deps', + 'toolchain': {'name': 'dummy', 'version': 'dummy'}, + 'dummy': True, + 'hidden': True, + } + easyconfig_moredeps = deepcopy(easyconfig_dep) + easyconfig_moredeps['dependencies'].append(hidden_dep) + easyconfig_moredeps['hiddendependencies'] = [hidden_dep] + + # toy/.0.0-deps is available and thus should be omitted + res = resolve_dependencies([deepcopy(easyconfig_moredeps)]) + self.assertEqual(len(res), 2) + full_mod_names = [ec['full_mod_name'] for ec in res] + self.assertFalse('toy/.0.0-deps' in full_mod_names) + + res = resolve_dependencies([deepcopy(easyconfig_moredeps)], retain_all_deps=True) + self.assertEqual(len(res), 4) # hidden dep toy/.0.0-deps (+1) depends on (fake) ictce/4.1.13 (+1) + self.assertEqual('gzip/1.4', res[0]['full_mod_name']) + self.assertEqual('foo/1.2.3', res[-1]['full_mod_name']) + full_mod_names = [ec['full_mod_name'] for ec in res] + self.assertTrue('toy/.0.0-deps' in full_mod_names) + self.assertTrue('ictce/4.1.13' in full_mod_names) + + # here we have included a dependency in the easyconfig list easyconfig['full_mod_name'] = 'gzip/1.4' ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)] @@ -145,9 +198,10 @@ def test_resolve_dependencies(self): 'versionsuffix': '', 'toolchain': {'name': 'GCC', 'version': '4.6.3'}, 'dummy': True, + 'hidden': False, }] ecs = [deepcopy(easyconfig_dep)] - build_options.update({'robot_path': self.base_easyconfig_dir}) + build_options.update({'robot_path': base_easyconfig_dir}) init_config(build_options=build_options) res = resolve_dependencies([deepcopy(easyconfig_dep)]) @@ -171,6 +225,7 @@ def test_resolve_dependencies(self): 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, + 'hidden': False, }] ecs = [deepcopy(easyconfig_dep)] res = resolve_dependencies(ecs) @@ -199,7 +254,7 @@ def test_resolve_dependencies(self): # build that are listed but already have a module available are not retained without force build_options.update({'force': False}) init_config(build_options=build_options) - newecs = skip_available(ecs, testing=True) # skip available builds since force is not enabled + newecs = skip_available(ecs) # skip available builds since force is not enabled res = resolve_dependencies(newecs) self.assertEqual(len(res), 2) self.assertEqual('goolf/1.4.10', res[0]['full_mod_name']) @@ -209,7 +264,7 @@ def test_resolve_dependencies(self): build_options.update({'retain_all_deps': True}) init_config(build_options=build_options) ecs = [deepcopy(easyconfig_dep)] - newecs = skip_available(ecs, testing=True) # skip available builds since force is not enabled + newecs = skip_available(ecs) # skip available builds since force is not enabled res = resolve_dependencies(newecs) self.assertEqual(len(res), 9) self.assertEqual('GCC/4.7.2', res[0]['full_mod_name']) @@ -233,6 +288,7 @@ def test_resolve_dependencies(self): 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, + 'hidden': False, }] ecs = [deepcopy(easyconfig_dep)] res = resolve_dependencies([deepcopy(easyconfig_dep)]) @@ -245,18 +301,112 @@ def test_resolve_dependencies(self): self.assertEqual('goolf/1.4.10', res[2]['full_mod_name']) self.assertEqual('foo/1.2.3', res[3]['full_mod_name']) - def tearDown(self): - """ reset the Modules back to its original """ - super(RobotTest, self).tearDown() - config.modules_tool = ORIG_MODULES_TOOL - ectools.modules_tool = ORIG_MAIN_MODULES_TOOL + ectools.modules_tool = ORIG_ECTOOLS_MODULES_TOOL + robot.modules_tool = ORIG_ROBOT_MODULES_TOOL if ORIG_MODULE_FUNCTION is not None: os.environ['module'] = ORIG_MODULE_FUNCTION else: if 'module' in os.environ: del os.environ['module'] + def test_det_easyconfig_paths(self): + """Test det_easyconfig_paths function (without --from-pr).""" + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + + test_ec = 'toy-0.0-deps.eb' + shutil.copy2(os.path.join(test_ecs_path, test_ec), self.test_prefix) + shutil.copy2(os.path.join(test_ecs_path, 'ictce-4.1.13.eb'), self.test_prefix) + self.assertFalse(os.path.exists(test_ec)) + + args = [ + os.path.join(test_ecs_path, 'toy-0.0.eb'), + test_ec, # relative path, should be resolved via robot search path + # PR for foss/2015a, see https://github.com/hpcugent/easybuild-easyconfigs/pull/1239/files + #'--from-pr=1239', + '--dry-run', + '--debug', + '--robot', + '--robot-paths=%s' % self.test_prefix, # override $EASYBUILD_ROBOT_PATHS + '--unittest-file=%s' % self.logfile, + '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user + '--tmpdir=%s' % self.test_prefix, + ] + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) + + modules = [ + (test_ecs_path, 'toy/0.0'), # specified easyconfigs, available at given location + (self.test_prefix, 'ictce/4.1.13'), # dependency, found in robot search path + (self.test_prefix, 'toy/0.0-deps'), # specified easyconfig, found in robot search path + ] + for path_prefix, module in modules: + ec_fn = "%s.eb" % '-'.join(module.split('/')) + regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path_prefix, ec_fn, module), re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + + def test_det_easyconfig_paths_from_pr(self): + """Test det_easyconfig_paths function, with --from-pr enabled as well.""" + if self.github_token is None: + print "Skipping test_from_pr, no GitHub token available?" + return + + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + + test_ec = 'toy-0.0-deps.eb' + shutil.copy2(os.path.join(test_ecs_path, test_ec), self.test_prefix) + shutil.copy2(os.path.join(test_ecs_path, 'ictce-4.1.13.eb'), self.test_prefix) + self.assertFalse(os.path.exists(test_ec)) + + gompi_2015a_txt = '\n'.join([ + "easyblock = 'Toolchain'", + "name = 'gompi'", + "version = '2015a'", + "versionsuffix = '-test'", + "homepage = 'foo'", + "description = 'bar'", + "toolchain = {'name': 'dummy', 'version': 'dummy'}", + ]) + write_file(os.path.join(self.test_prefix, 'gompi-2015a-test.eb'), gompi_2015a_txt) + # put gompi-2015a.eb easyconfig in place that shouldn't be considered (paths via --from-pr have precedence) + write_file(os.path.join(self.test_prefix, 'gompi-2015a.eb'), gompi_2015a_txt) + + args = [ + os.path.join(test_ecs_path, 'toy-0.0.eb'), + test_ec, # relative path, should be resolved via robot search path + # PR for foss/2015a, see https://github.com/hpcugent/easybuild-easyconfigs/pull/1239/files + '--from-pr=1239', + 'FFTW-3.3.4-gompi-2015a.eb', + 'gompi-2015a-test.eb', # relative path, available in robot search path + '--dry-run', + '--robot', + '--robot=%s' % self.test_prefix, + '--unittest-file=%s' % self.logfile, + '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user + '--tmpdir=%s' % self.test_prefix, + ] + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) + + from_pr_prefix = os.path.join(self.test_prefix, '.*', 'files_pr1239') + modules = [ + (test_ecs_path, 'toy/0.0'), # specified easyconfigs, available at given location + (self.test_prefix, 'ictce/4.1.13'), # dependency, found in robot search path + (self.test_prefix, 'toy/0.0-deps'), # specified easyconfig, found in robot search path + (self.test_prefix, 'gompi/2015a-test'), # specified easyconfig, found in robot search path + (from_pr_prefix, 'FFTW/3.3.4-gompi-2015a'), # part of PR easyconfigs + (from_pr_prefix, 'gompi/2015a'), # part of PR easyconfigs + (test_ecs_path, 'GCC/4.9.2'), # dependency for PR easyconfigs, found in robot search path + ] + for path_prefix, module in modules: + ec_fn = "%s.eb" % '-'.join(module.split('/')) + regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path_prefix, ec_fn, module), re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/run.py b/test/framework/run.py index ba8cb3c491..4d46392ddf 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -95,30 +95,6 @@ def test_parse_log_error(self): errors = parse_log_for_error("error failed", True) self.assertEqual(len(errors), 1) - def test_run_cmd_suse(self): - """Test run_cmd on SuSE systems, which have $PROFILEREAD set.""" - # avoid warning messages - run_log_level = run_log.getEffectiveLevel() - run_log.setLevel('ERROR') - - # run_cmd should also work if $PROFILEREAD is set (very relevant for SuSE systems) - profileread = os.environ.get('PROFILEREAD', None) - os.environ['PROFILEREAD'] = 'profilereadxxx' - try: - (out, ec) = run_cmd("echo hello") - except Exception, err: - out, ec = "ERROR: %s" % err, 1 - - # make sure it's restored again before we can fail the test - if profileread is not None: - os.environ['PROFILEREAD'] = profileread - else: - del os.environ['PROFILEREAD'] - - self.assertEqual(out, "hello\n") - self.assertEqual(ec, 0) - run_log.setLevel(run_log_level) - def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/sandbox/easybuild/easyblocks/__init__.py b/test/framework/sandbox/easybuild/easyblocks/__init__.py index 5b5529273b..a2bc4d6715 100644 --- a/test/framework/sandbox/easybuild/easyblocks/__init__.py +++ b/test/framework/sandbox/easybuild/easyblocks/__init__.py @@ -1,4 +1,12 @@ from pkgutil import extend_path -# we're not the only ones in this namespace -__path__ = extend_path(__path__, __name__) #@ReservedAssignment +# Extend path so python finds our easyblocks in the subdirectories where they are located +subdirs = [chr(l) for l in range(ord('a'), ord('z') + 1)] + ['0'] +for subdir in subdirs: + __path__ = extend_path(__path__, '%s.%s' % (__name__, subdir)) + +# And let python know this is not the only place to look for them, so we can have multiple +# easybuild/easyblock paths in your python search path, next to the official easyblocks distribution +__path__ = extend_path(__path__, __name__) # @ReservedAssignment + +del subdir, subdirs, l diff --git a/vsc/install/__init__.py b/test/framework/sandbox/easybuild/easyblocks/f/__init__.py similarity index 100% rename from vsc/install/__init__.py rename to test/framework/sandbox/easybuild/easyblocks/f/__init__.py diff --git a/test/framework/sandbox/easybuild/easyblocks/foo.py b/test/framework/sandbox/easybuild/easyblocks/f/foo.py similarity index 97% rename from test/framework/sandbox/easybuild/easyblocks/foo.py rename to test/framework/sandbox/easybuild/easyblocks/f/foo.py index 769f2a7e5a..8f1145fe72 100644 --- a/test/framework/sandbox/easybuild/easyblocks/foo.py +++ b/test/framework/sandbox/easybuild/easyblocks/f/foo.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/foofoo.py b/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py similarity index 97% rename from test/framework/sandbox/easybuild/easyblocks/foofoo.py rename to test/framework/sandbox/easybuild/easyblocks/f/foofoo.py index 511953a0fe..94ec86ef89 100644 --- a/test/framework/sandbox/easybuild/easyblocks/foofoo.py +++ b/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/bar.py b/test/framework/sandbox/easybuild/easyblocks/generic/bar.py index 482ee15cc1..6d62238e39 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/bar.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/bar.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -38,9 +38,8 @@ class bar(EasyBlock): @staticmethod def extra_options(): """Custom easyconfig parameters for bar.""" - - extra_vars = [ - ('bar_extra1', [None, "first bar-specific easyconfig parameter (mandatory)", MANDATORY]), - ('bar_extra2', ['BAR', "second bar-specific easyconfig parameter", CUSTOM]), - ] + extra_vars = { + 'bar_extra1': [None, "first bar-specific easyconfig parameter (mandatory)", MANDATORY], + 'bar_extra2': ['BAR', "second bar-specific easyconfig parameter", CUSTOM], + } return EasyBlock.extra_options(extra_vars) diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py b/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py index 8304b9e42c..7fba52ca5b 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py new file mode 100644 index 0000000000..59d96675e1 --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py @@ -0,0 +1,34 @@ +## +# Copyright 2009-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing dummy extensions, implemented as an easyblock + +@author: Kenneth Hoste (Ghent University) +""" + +from easybuild.framework.extensioneasyblock import ExtensionEasyBlock + +class DummyExtension(ExtensionEasyBlock): + """Support for building/installing dummy extensions.""" diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py b/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py index db67440c90..f8212fdf96 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -31,4 +31,11 @@ class Toolchain(EasyBlock): """Dummy support for toolchains.""" - pass + def configure_step(self): + pass + def build_step(self): + pass + def install_step(self): + pass + def sanity_check_step(self): + pass diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index b6e15d6347..f41a63c580 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/h/__init__.py b/test/framework/sandbox/easybuild/easyblocks/h/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/framework/sandbox/easybuild/easyblocks/h/hpl.py b/test/framework/sandbox/easybuild/easyblocks/h/hpl.py new file mode 100644 index 0000000000..ba6bfe42f7 --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/h/hpl.py @@ -0,0 +1,33 @@ +## +# Copyright 2009-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Dummy easyblock for HPL + +@author: Kenneth Hoste (Ghent University) +""" +from easybuild.framework.easyblock import EasyBlock + +class EB_HPL(EasyBlock): + pass diff --git a/test/framework/sandbox/easybuild/easyblocks/s/__init__.py b/test/framework/sandbox/easybuild/easyblocks/s/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py b/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py new file mode 100644 index 0000000000..3ed062e7d0 --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py @@ -0,0 +1,33 @@ +## +# Copyright 2009-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Dummy easyblock for ScaLAPACK + +@author: Kenneth Hoste (Ghent University) +""" +from easybuild.framework.easyblock import EasyBlock + +class EB_ScaLAPACK(EasyBlock): + pass diff --git a/test/framework/sandbox/easybuild/easyblocks/t/__init__.py b/test/framework/sandbox/easybuild/easyblocks/t/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/framework/sandbox/easybuild/easyblocks/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py similarity index 87% rename from test/framework/sandbox/easybuild/easyblocks/toy.py rename to test/framework/sandbox/easybuild/easyblocks/t/toy.py index 897d248dfe..f0c02aa99f 100644 --- a/test/framework/sandbox/easybuild/easyblocks/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -33,8 +33,11 @@ import shutil from easybuild.framework.easyblock import EasyBlock -from easybuild.tools.filetools import mkdir, run_cmd +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import mkdir from easybuild.tools.modules import get_software_root, get_software_version +from easybuild.tools.run import run_cmd + class EB_toy(EasyBlock): """Support for building/installing toy.""" @@ -54,16 +57,16 @@ def configure_step(self, name=None): # make sure Python system dep is handled correctly when specified if self.cfg['allow_system_deps']: if get_software_root('Python') != 'Python' or get_software_version('Python') != platform.python_version(): - self.log.error("Sanity check on allowed Python system dep failed.") + raise EasyBuildError("Sanity check on allowed Python system dep failed.") os.rename('%s.source' % name, '%s.c' % name) def build_step(self, name=None): """Build toy.""" if name is None: name = self.name - run_cmd('%(premakeopts)s gcc %(name)s.c -o %(name)s' % { + run_cmd('%(prebuildopts)s gcc %(name)s.c -o %(name)s' % { 'name': name, - 'premakeopts': self.cfg['premakeopts'], + 'prebuildopts': self.cfg['prebuildopts'], }) def install_step(self, name=None): diff --git a/test/framework/sandbox/easybuild/easyblocks/toy_buggy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py similarity index 97% rename from test/framework/sandbox/easybuild/easyblocks/toy_buggy.py rename to test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py index 07658b64fd..74df658f3d 100644 --- a/test/framework/sandbox/easybuild/easyblocks/toy_buggy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/__init__.py b/test/framework/sandbox/easybuild/tools/__init__.py index 15512e2f9c..6badf9fb34 100644 --- a/test/framework/sandbox/easybuild/tools/__init__.py +++ b/test/framework/sandbox/easybuild/tools/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py index 1a66de17e0..5a2060c2d0 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2014 Ghent University +# Copyright 2011-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py index 2b682a72b1..ccd03960ca 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -61,3 +61,9 @@ def det_module_symlink_paths(self, ec): Determine list of paths in which symlinks to module files must be created. """ return [ec['moduleclass'].upper(), ec['name'].lower()[0]] + + def is_short_modname_for(self, modname, name): + """ + Determine whether the specified (short) module name is a module for software with the specified name. + """ + return modname.find('%s' % name)!= -1 diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py index d7ba785468..b8b95a9dda 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -70,3 +70,9 @@ def det_full_module_name(self, ec): ec_sha1 = sha1(res).hexdigest() _log.debug("SHA1 for string '%s' obtained for %s: %s" % (res, ec, ec_sha1)) return os.path.join(ec['name'], ec_sha1) + + def is_short_modname_for(self, modname, name): + """ + Determine whether the specified (short) module name is a module for software with the specified name. + """ + return modname.startswith(name) diff --git a/test/framework/sandbox/sources/g/gzip/gzip-1.4.tar.gz b/test/framework/sandbox/sources/g/gzip/gzip-1.4.tar.gz new file mode 100644 index 0000000000..92d6ae4e08 Binary files /dev/null and b/test/framework/sandbox/sources/g/gzip/gzip-1.4.tar.gz differ diff --git a/test/framework/sandbox/sources/toy/extensions/barbar-0.0.tar.gz b/test/framework/sandbox/sources/toy/extensions/barbar-0.0.tar.gz new file mode 100644 index 0000000000..d665916415 Binary files /dev/null and b/test/framework/sandbox/sources/toy/extensions/barbar-0.0.tar.gz differ diff --git a/test/framework/sandbox/sources/toy/toy-extra.txt b/test/framework/sandbox/sources/toy/toy-extra.txt new file mode 100644 index 0000000000..1286a1af21 --- /dev/null +++ b/test/framework/sandbox/sources/toy/toy-extra.txt @@ -0,0 +1 @@ +moar! diff --git a/test/framework/scripts.py b/test/framework/scripts.py index 27b75128b5..1613e4128d 100644 --- a/test/framework/scripts.py +++ b/test/framework/scripts.py @@ -35,19 +35,35 @@ from test.framework.utilities import EnhancedTestCase from unittest import TestLoader, main +import vsc + +import easybuild.framework +from easybuild.framework.easyconfig.easyconfig import EasyConfig +from easybuild.tools.filetools import read_file, write_file from easybuild.tools.run import run_cmd class ScriptsTest(EnhancedTestCase): """ Testcase for run module """ + def setUp(self): + """Test setup.""" + super(ScriptsTest, self).setUp() + + # make sure both vsc-base and easybuild-framework are included in $PYTHONPATH (so scripts can pick it up) + vsc_loc = os.path.dirname(os.path.dirname(os.path.abspath(vsc.__file__))) + framework_loc = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(easybuild.framework.__file__)))) + pythonpath = os.environ.get('PYTHONPATH', '') + os.environ['PYTHONPATH'] = os.pathsep.join([vsc_loc, framework_loc, pythonpath]) + def test_generate_software_list(self): """Test for generate_software_list.py script.""" # adjust $PYTHONPATH such that test easyblocks are found by the script - eb_blocks_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'sandbox')) - pythonpath = os.environ['PYTHONPATH'] - os.environ['PYTHONPATH'] = "%s:%s" % (pythonpath, eb_blocks_path) + test_dir = os.path.abspath(os.path.dirname(__file__)) + eb_blocks_path = os.path.join(test_dir, 'sandbox') + pythonpath = os.environ.get('PYTHONPATH', os.path.dirname(test_dir)) + os.environ['PYTHONPATH'] = os.pathsep.join([pythonpath, eb_blocks_path]) testdir = os.path.dirname(__file__) topdir = os.path.dirname(os.path.dirname(testdir)) @@ -59,19 +75,21 @@ def test_generate_software_list(self): for root, subfolders, files in os.walk(easyconfigs_dir): if 'v2.0' in subfolders: subfolders.remove('v2.0') - for ec_file in files: + for ec_file in [f for f in files if 'broken' not in os.path.basename(f)]: shutil.copy2(os.path.join(root, ec_file), tmpdir) - cmd = "python %s --local --quiet --path %s" % (script, tmpdir) + cmd = "%s %s --local --quiet --path %s" % (sys.executable, script, tmpdir) out, ec = run_cmd(cmd, simple=False) # make sure output is kind of what we expect it to be - self.assertTrue(re.search(r"Supported Packages \(11", out)) + regex = r"Supported Packages \(18 " + self.assertTrue(re.search(regex, out), "Pattern '%s' found in output: %s" % (regex, out)) per_letter = { + 'C': '1', # CUDA 'F': '1', # FFTW 'G': '4', # GCC, gompi, goolf, gzip 'H': '1', # hwloc - 'I': '1', # ictce + 'I': '7', # icc, iccifort, ictce, ifort, iimpi, imkl, impi 'O': '2', # OpenMPI, OpenBLAS 'S': '1', # ScaLAPACK 'T': '1', # toy @@ -89,6 +107,111 @@ def test_generate_software_list(self): shutil.rmtree(tmpdir) os.environ['PYTHONPATH'] = pythonpath + def test_fix_broken_easyconfig(self): + """Test fix_broken_easyconfigs.py script.""" + testdir = os.path.dirname(__file__) + topdir = os.path.dirname(os.path.dirname(testdir)) + script = os.path.join(topdir, 'easybuild', 'scripts', 'fix_broken_easyconfigs.py') + test_easyblocks = os.path.join(testdir, 'sandbox') + + broken_ec_txt_tmpl = '\n'.join([ + "# licenseheader", + "%sname = '%s'", + "version = '1.2.3'", + '', + "description = 'foo'", + "homepage = 'http://example.com'", + '', + "toolchain = {'name': 'GCC', 'version': '4.8.2'}", + '', + "premakeopts = 'FOO=libfoo.%%s' %% shared_lib_ext", + "makeopts = 'CC=gcc'", + '', + "license = 'foo.lic'", + ]) + fixed_ec_txt_tmpl = '\n'.join([ + "# licenseheader", + "%sname = '%s'", + "version = '1.2.3'", + '', + "description = 'foo'", + "homepage = 'http://example.com'", + '', + "toolchain = {'name': 'GCC', 'version': '4.8.2'}", + '', + "prebuildopts = 'FOO=libfoo.%%s' %% SHLIB_EXT", + "buildopts = 'CC=gcc'", + '', + "license_file = 'foo.lic'", + ]) + broken_ec_tmpl = os.path.join(self.test_prefix, '%s.eb') + script_cmd_tmpl = "PYTHONPATH=%s:$PYTHONPATH:%s %s %%s" % (topdir, test_easyblocks, script) + + # don't change it if it isn't broken + broken_ec = broken_ec_tmpl % 'notbroken' + script_cmd = script_cmd_tmpl % broken_ec + fixed_ec_txt = fixed_ec_txt_tmpl % ("easyblock = 'ConfigureMake'\n\n", 'foo') + + write_file(broken_ec, fixed_ec_txt) + # (dummy) ConfigureMake easyblock is available in test sandbox + script_cmd = script_cmd_tmpl % broken_ec + new_ec_txt = read_file(broken_ec) + self.assertEqual(new_ec_txt, fixed_ec_txt) + self.assertTrue(EasyConfig(None, rawtxt=new_ec_txt)) + self.assertFalse(os.path.exists('%s.bk' % broken_ec)) # no backup created if nothing was fixed + + broken_ec = broken_ec_tmpl % 'nosuchsoftware' + script_cmd = script_cmd_tmpl % broken_ec + broken_ec_txt = broken_ec_txt_tmpl % ('', 'nosuchsoftware') + fixed_ec_txt = fixed_ec_txt_tmpl % ("easyblock = 'ConfigureMake'\n\n", 'nosuchsoftware') + + # broken easyconfig is fixed in place, original file is backed up + write_file(broken_ec, broken_ec_txt) + run_cmd(script_cmd) + new_ec_txt = read_file(broken_ec) + self.assertEqual(new_ec_txt, fixed_ec_txt) + self.assertTrue(EasyConfig(None, rawtxt=new_ec_txt)) + self.assertEqual(read_file('%s.bk' % broken_ec), broken_ec_txt) + self.assertFalse(os.path.exists('%s.bk1' % broken_ec)) + + # broken easyconfig is fixed in place, original file is backed up, existing backup is not overwritten + write_file(broken_ec, broken_ec_txt) + write_file('%s.bk' % broken_ec, 'thisshouldnot\nbechanged') + run_cmd(script_cmd) + new_ec_txt = read_file(broken_ec) + self.assertEqual(new_ec_txt, fixed_ec_txt) + self.assertTrue(EasyConfig(None, rawtxt=new_ec_txt)) + self.assertEqual(read_file('%s.bk' % broken_ec), 'thisshouldnot\nbechanged') + self.assertEqual(read_file('%s.bk1' % broken_ec), broken_ec_txt) + + # if easyblock is specified, that part is left untouched + broken_ec = broken_ec_tmpl % 'footoy' + script_cmd = script_cmd_tmpl % broken_ec + broken_ec_txt = broken_ec_txt_tmpl % ("easyblock = 'EB_toy'\n\n", 'foo') + fixed_ec_txt = fixed_ec_txt_tmpl % ("easyblock = 'EB_toy'\n\n", 'foo') + + write_file(broken_ec, broken_ec_txt) + run_cmd(script_cmd) + new_ec_txt = read_file(broken_ec) + self.assertEqual(new_ec_txt, fixed_ec_txt) + self.assertTrue(EasyConfig(None, rawtxt=new_ec_txt)) + self.assertEqual(read_file('%s.bk' % broken_ec), broken_ec_txt) + + # for existing easyblocks, "easyblock = 'ConfigureMake'" should *not* be added + # EB_toy easyblock is available in test sandbox + test_easyblocks = os.path.join(testdir, 'sandbox') + broken_ec = broken_ec_tmpl % 'toy' + # path to test easyblocks must be *appended* to PYTHONPATH (due to flattening in easybuild-easyblocks repo) + script_cmd = script_cmd_tmpl % broken_ec + broken_ec_txt = broken_ec_txt_tmpl % ('', 'toy') + fixed_ec_txt = fixed_ec_txt_tmpl % ('', 'toy') + write_file(broken_ec, broken_ec_txt) + run_cmd(script_cmd) + new_ec_txt = read_file(broken_ec) + self.assertEqual(new_ec_txt, fixed_ec_txt) + self.assertTrue(EasyConfig(None, rawtxt=new_ec_txt)) + self.assertEqual(read_file('%s.bk' % broken_ec), broken_ec_txt) + def suite(): """ returns all the testcases in this module """ return TestLoader().loadTestsFromTestCase(ScriptsTest) diff --git a/test/framework/suite.py b/test/framework/suite.py old mode 100644 new mode 100755 index 61b3ac30b4..7207a56447 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -1,6 +1,6 @@ #!/usr/bin/python # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -32,19 +32,29 @@ """ import glob import os -import shutil import sys import tempfile import unittest from vsc.utils import fancylogger +# initialize EasyBuild logging, so we disable it +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import set_tmpdir + +# set plain text key ring to be used, so a GitHub token stored in it can be obtained without having to provide a password +try: + import keyring + keyring.set_keyring(keyring.backends.file.PlaintextKeyring()) +except ImportError: + pass + # disable all logging to significantly speed up tests -import easybuild.tools.build_log # initialize EasyBuild logging, so we disable it fancylogger.disableDefaultHandlers() fancylogger.setLogLevelError() # toolkit should be first to allow hacks to work import test.framework.asyncprocess as a +import test.framework.build_log as bl import test.framework.config as c import test.framework.easyblock as b import test.framework.easyconfig as e @@ -52,15 +62,19 @@ import test.framework.easyconfigformat as ef import test.framework.ebconfigobj as ebco import test.framework.easyconfigversion as ev +import test.framework.docs as d import test.framework.filetools as f import test.framework.format_convert as f_c +import test.framework.general as gen import test.framework.github as g +import test.framework.include as i import test.framework.license as l import test.framework.module_generator as mg import test.framework.modules as m import test.framework.modulestool as mt import test.framework.options as o import test.framework.parallelbuild as p +import test.framework.package as pkg import test.framework.repository as r import test.framework.robot as robot import test.framework.run as run @@ -69,22 +83,17 @@ import test.framework.toolchain as tc import test.framework.toolchainvariables as tcv import test.framework.toy_build as t +import test.framework.type_checking as et +import test.framework.tweak as tw import test.framework.variables as v # make sure temporary files can be created/used -fd, fn = tempfile.mkstemp() -os.close(fd) -os.remove(fn) -testdir = tempfile.mkdtemp() -for test_fn in [fn, os.path.join(testdir, 'test')]: - try: - open(fn, 'w').write('test') - except IOError, err: - sys.stderr.write("ERROR: Can't write to temporary file %s, set $TMPDIR to a writeable directory (%s)" % (fn, err)) - sys.exit(1) -os.remove(fn) -shutil.rmtree(testdir) +try: + set_tmpdir(raise_error=True) +except EasyBuildError, err: + sys.stderr.write("No execution rights on temporary files, specify another location via $TMPDIR: %s\n" % err) + sys.exit(1) # initialize logger for all the unit tests fd, log_fn = tempfile.mkstemp(prefix='easybuild-tests-', suffix='.log') @@ -95,7 +104,8 @@ # call suite() for each module and then run them all # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config -tests = [o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, l, f_c, sc] +tests = [gen, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, l, f_c, sc, tw, + p, i, pkg, d, et] SUITE = unittest.TestSuite([x.suite() for x in tests]) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 8f9e8a4cf2..5c03e4bd3c 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -27,52 +27,325 @@ @author: Kenneth hoste (Ghent University) """ +import re +from os.path import exists as orig_os_path_exists from test.framework.utilities import EnhancedTestCase from unittest import TestLoader, main -from easybuild.tools.systemtools import AMD, ARM, DARWIN, INTEL, LINUX, UNKNOWN -from easybuild.tools.systemtools import get_avail_core_count, get_core_count +import easybuild.tools.systemtools as st +from easybuild.tools.filetools import read_file +from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import CPU_FAMILIES, ARM, DARWIN, IBM, INTEL, LINUX, POWER, UNKNOWN, VENDORS +from easybuild.tools.systemtools import det_parallelism, get_avail_core_count, get_cpu_family from easybuild.tools.systemtools import get_cpu_model, get_cpu_speed, get_cpu_vendor, get_glibc_version from easybuild.tools.systemtools import get_os_type, get_os_name, get_os_version, get_platform_name, get_shared_lib_ext from easybuild.tools.systemtools import get_system_info +MAX_FREQ_FP = '/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq' +PROC_CPUINFO_FP = '/proc/cpuinfo' + +PROC_CPUINFO_TXT = None +PROC_CPUINFO_TXT_ARM = """processor : 0 +model name : ARMv7 Processor rev 5 (v7l) +BogoMIPS : 57.60 +Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 idiva idivt vfpd32 lpae evtstrm +CPU implementer : 0x41 +CPU architecture: 7 +CPU variant : 0x0 +CPU part : 0xc07 +CPU revision : 5 + +processor : 1 +model name : ARMv7 Processor rev 5 (v7l) +BogoMIPS : 57.60 +Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 idiva idivt vfpd32 lpae evtstrm +CPU implementer : 0x41 +CPU architecture: 7 +CPU variant : 0x0 +CPU part : 0xc07 +CPU revision : 5 +""" +PROC_CPUINFO_TXT_POWER = """processor : 0 +cpu : POWER7 (architected), altivec supported +clock : 3550.000000MHz +revision : 2.3 (pvr 003f 0203) + +processor : 13 +cpu : POWER7 (architected), altivec supported +clock : 3550.000000MHz +revision : 2.3 (pvr 003f 0203) + +timebase : 512000000 +platform : pSeries +model : IBM,8205-E6C +machine : CHRP IBM,8205-E6C +""" +PROC_CPUINFO_TXT_X86 = """processor : 0 +vendor_id : GenuineIntel +cpu family : 6 +model : 45 +model name : Intel(R) Xeon(R) CPU E5-2670 0 @ 2.60GHz +stepping : 7 +microcode : 1808 +cpu MHz : 2600.075 +cache size : 20480 KB +physical id : 0 +siblings : 8 +core id : 0 +cpu cores : 8 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 13 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm ida arat xsaveopt pln pts dts tpr_shadow vnmi flexpriority ept vpid +bogomips : 5200.15 +clflush size : 64 +cache_alignment : 64 +address sizes : 46 bits physical, 48 bits virtual +power management: + +processor : 1 +vendor_id : GenuineIntel +cpu family : 6 +model : 45 +model name : Intel(R) Xeon(R) CPU E5-2670 0 @ 2.60GHz +stepping : 7 +microcode : 1808 +cpu MHz : 2600.075 +cache size : 20480 KB +physical id : 1 +siblings : 8 +core id : 0 +cpu cores : 8 +apicid : 32 +initial apicid : 32 +fpu : yes +fpu_exception : yes +cpuid level : 13 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm ida arat xsaveopt pln pts dts tpr_shadow vnmi flexpriority ept vpid +bogomips : 5200.04 +clflush size : 64 +cache_alignment : 64 +address sizes : 46 bits physical, 48 bits virtual +power management: +""" + + +def mocked_read_file(fp): + """Mocked version of read_file, with specified contents for known filenames.""" + known_fps = { + MAX_FREQ_FP: '2850000', + PROC_CPUINFO_FP: PROC_CPUINFO_TXT, + } + if fp in known_fps: + return known_fps[fp] + else: + return read_file(fp) + +def mocked_os_path_exists(mocked_fp, fp): + """Mocked version of os.path.exists, returns True for a particular specified filepath.""" + return fp == mocked_fp + +def mocked_run_cmd(cmd, **kwargs): + """Mocked version of run_cmd, with specified output for known commands.""" + known_cmds = { + "ldd --version" : "ldd (GNU libc) 2.12", + "sysctl -n hw.cpufrequency_max": "2400000000", + "sysctl -n hw.ncpu": '10', + "sysctl -n machdep.cpu.brand_string": "Intel(R) Core(TM) i5-4258U CPU @ 2.40GHz", + "sysctl -n machdep.cpu.vendor": 'GenuineIntel', + "ulimit -u": '40', + } + if cmd in known_cmds: + if 'simple' in kwargs and kwargs['simple']: + return True + else: + return (known_cmds[cmd], 0) + else: + return run_cmd(cmd, **kwargs) + + class SystemToolsTest(EnhancedTestCase): """ very basis FileRepository test, we don't want git / svn dependency """ - def test_core_count(self): + def setUp(self): + """Set up systemtools test.""" + super(SystemToolsTest, self).setUp() + self.orig_get_os_type = st.get_os_type + self.orig_os_path_exists = st.os.path.exists + self.orig_read_file = st.read_file + self.orig_run_cmd = st.run_cmd + + def tearDown(self): + """Cleanup after systemtools test.""" + st.os.path.exists = self.orig_os_path_exists + st.read_file = self.orig_read_file + st.get_os_type = self.orig_get_os_type + st.run_cmd = self.orig_run_cmd + super(SystemToolsTest, self).tearDown() + + def test_avail_core_count_native(self): """Test getting core count.""" - for core_count in [get_avail_core_count(), get_core_count()]: - self.assertTrue(isinstance(core_count, int), "core_count has type int: %s, %s" % (core_count, type(core_count))) - self.assertTrue(core_count > 0, "core_count %d > 0" % core_count) + core_count = get_avail_core_count() + self.assertTrue(isinstance(core_count, int), "core_count has type int: %s, %s" % (core_count, type(core_count))) + self.assertTrue(core_count > 0, "core_count %d > 0" % core_count) - def test_cpu_model(self): + def test_avail_core_count_linux(self): + """Test getting core count (mocked for Linux).""" + st.get_os_type = lambda: st.LINUX + orig_sched_getaffinity = st.sched_getaffinity + class MockedSchedGetaffinity(object): + cpus = [1L, 1L, 0L, 0L, 1L, 1L, 0L, 0L, 1L, 1L, 0L, 0L] + st.sched_getaffinity = lambda: MockedSchedGetaffinity() + self.assertEqual(get_avail_core_count(), 6) + st.sched_getaffinity = orig_sched_getaffinity + + def test_avail_core_count_darwin(self): + """Test getting core count (mocked for Darwin).""" + st.get_os_type = lambda: st.DARWIN + st.run_cmd = mocked_run_cmd + self.assertEqual(get_avail_core_count(), 10) + + def test_cpu_model_native(self): """Test getting CPU model.""" cpu_model = get_cpu_model() self.assertTrue(isinstance(cpu_model, basestring)) - def test_cpu_speed(self): + def test_cpu_model_linux(self): + """Test getting CPU model (mocked for Linux).""" + st.get_os_type = lambda: st.LINUX + st.read_file = mocked_read_file + st.os.path.exists = lambda fp: mocked_os_path_exists(PROC_CPUINFO_FP, fp) + global PROC_CPUINFO_TXT + + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_X86 + self.assertEqual(get_cpu_model(), "Intel(R) Xeon(R) CPU E5-2670 0 @ 2.60GHz") + + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_POWER + self.assertEqual(get_cpu_model(), "IBM,8205-E6C") + + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_ARM + self.assertEqual(get_cpu_model(), "ARMv7 Processor rev 5 (v7l)") + + def test_cpu_model_darwin(self): + """Test getting CPU model (mocked for Darwin).""" + st.get_os_type = lambda: st.DARWIN + st.run_cmd = mocked_run_cmd + self.assertEqual(get_cpu_model(), "Intel(R) Core(TM) i5-4258U CPU @ 2.40GHz") + + def test_cpu_speed_native(self): """Test getting CPU speed.""" cpu_speed = get_cpu_speed() - self.assertTrue(isinstance(cpu_speed, float)) - self.assertTrue(cpu_speed > 0.0) + self.assertTrue(isinstance(cpu_speed, float) or cpu_speed is None) + self.assertTrue(cpu_speed > 0.0 or cpu_speed is None) + + def test_cpu_speed_linux(self): + """Test getting CPU speed (mocked for Linux).""" + # test for particular type of system by mocking used functions + st.get_os_type = lambda: st.LINUX + st.read_file = mocked_read_file + st.os.path.exists = lambda fp: mocked_os_path_exists(PROC_CPUINFO_FP, fp) + + # tweak global constant used by mocked_read_file + global PROC_CPUINFO_TXT + + # /proc/cpuinfo on Linux x86 (no cpufreq) + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_X86 + self.assertEqual(get_cpu_speed(), 2600.075) + + # /proc/cpuinfo on Linux POWER + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_POWER + self.assertEqual(get_cpu_speed(), 3550.0) + + # Linux (x86) with cpufreq + st.os.path.exists = lambda fp: mocked_os_path_exists(MAX_FREQ_FP, fp) + self.assertEqual(get_cpu_speed(), 2850.0) + + def test_cpu_speed_darwin(self): + """Test getting CPU speed (mocked for Darwin).""" + st.get_os_type = lambda: st.DARWIN + st.run_cmd = mocked_run_cmd + self.assertEqual(get_cpu_speed(), 2400.0) def test_cpu_vendor(self): """Test getting CPU vendor.""" cpu_vendor = get_cpu_vendor() - self.assertTrue(cpu_vendor in [AMD, ARM, INTEL, UNKNOWN]) + self.assertTrue(cpu_vendor in VENDORS.values() + [UNKNOWN]) + + def test_cpu_vendor_linux(self): + """Test getting CPU vendor (mocked for Linux).""" + st.get_os_type = lambda: st.LINUX + st.read_file = mocked_read_file + st.os.path.exists = lambda fp: mocked_os_path_exists(PROC_CPUINFO_FP, fp) + + global PROC_CPUINFO_TXT + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_X86 + self.assertEqual(get_cpu_vendor(), INTEL) + + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_POWER + self.assertEqual(get_cpu_vendor(), IBM) + + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_ARM + self.assertEqual(get_cpu_vendor(), ARM) + + def test_cpu_vendor_darwin(self): + """Test getting CPU vendor (mocked for Darwin).""" + st.get_os_type = lambda: st.DARWIN + st.run_cmd = mocked_run_cmd + self.assertEqual(get_cpu_vendor(), INTEL) + + def test_cpu_family_native(self): + """Test get_cpu_family function.""" + cpu_family = get_cpu_family() + self.assertTrue(cpu_family in CPU_FAMILIES or cpu_family == UNKNOWN) + + def test_cpu_family_linux(self): + """Test get_cpu_family function (mocked for Linux).""" + st.get_os_type = lambda: st.LINUX + st.read_file = mocked_read_file + st.os.path.exists = lambda fp: mocked_os_path_exists(PROC_CPUINFO_FP, fp) + global PROC_CPUINFO_TXT + + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_X86 + self.assertEqual(get_cpu_family(), INTEL) + + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_ARM + self.assertEqual(get_cpu_family(), ARM) + + PROC_CPUINFO_TXT = PROC_CPUINFO_TXT_POWER + self.assertEqual(get_cpu_family(), POWER) + + def test_cpu_family_darwin(self): + """Test get_cpu_family function (mocked for Darwin).""" + st.get_os_type = lambda: st.DARWIN + st.run_cmd = mocked_run_cmd + self.assertEqual(get_cpu_family(), INTEL) def test_os_type(self): """Test getting OS type.""" os_type = get_os_type() self.assertTrue(os_type in [DARWIN, LINUX]) - def test_shared_lib_ext(self): + def test_shared_lib_ext_native(self): """Test getting extension for shared libraries.""" ext = get_shared_lib_ext() self.assertTrue(ext in ['dylib', 'so']) - def test_platform_name(self): + def test_shared_lib_ext_native(self): + """Test getting extension for shared libraries (mocked for Linux).""" + st.get_os_type = lambda: st.LINUX + self.assertEqual(get_shared_lib_ext(), 'so') + + def test_shared_lib_ext_native(self): + """Test getting extension for shared libraries (mocked for Darwin).""" + st.get_os_type = lambda: st.DARWIN + self.assertEqual(get_shared_lib_ext(), 'dylib') + + def test_platform_name_native(self): """Test getting platform name.""" platform_name_nover = get_platform_name() self.assertTrue(isinstance(platform_name_nover, basestring)) @@ -85,6 +358,18 @@ def test_platform_name(self): self.assertTrue(platform_name_ver.startswith(platform_name_ver)) self.assertTrue(len_ver >= len_nover) + def test_platform_name_linux(self): + """Test getting platform name (mocked for Linux).""" + st.get_os_type = lambda: st.LINUX + self.assertTrue(re.match('.*-unknown-linux$', get_platform_name())) + self.assertTrue(re.match('.*-unknown-linux-gnu$', get_platform_name(withversion=True))) + + def test_platform_name_darwin(self): + """Test getting platform name (mocked for Darwin).""" + st.get_os_type = lambda: st.DARWIN + self.assertTrue(re.match('.*-apple-darwin$', get_platform_name())) + self.assertTrue(re.match('.*-apple-darwin.*$', get_platform_name(withversion=True))) + def test_os_name(self): """Test getting OS name.""" os_name = get_os_name() @@ -95,16 +380,60 @@ def test_os_version(self): os_version = get_os_version() self.assertTrue(isinstance(os_version, basestring) or os_version == UNKNOWN) - def test_glibc_version(self): + def test_glibc_version_native(self): """Test getting glibc version.""" glibc_version = get_glibc_version() self.assertTrue(isinstance(glibc_version, basestring) or glibc_version == UNKNOWN) + def test_glibc_version_linux(self): + """Test getting glibc version (mocked for Linux).""" + st.get_os_type = lambda: st.LINUX + st.run_cmd = mocked_run_cmd + self.assertEqual(get_glibc_version(), '2.12') + + def test_glibc_version_darwin(self): + """Test getting glibc version (mocked for Darwin).""" + st.get_os_type = lambda: st.DARWIN + self.assertEqual(get_glibc_version(), UNKNOWN) + def test_system_info(self): """Test getting system info.""" system_info = get_system_info() self.assertTrue(isinstance(system_info, dict)) + def test_det_parallelism_native(self): + """Test det_parallelism function (native calls).""" + self.assertTrue(det_parallelism() > 0) + # specified parallellism + self.assertEqual(det_parallelism(par=5), 5) + # max parallellism caps + self.assertEqual(det_parallelism(maxpar=1), 1) + self.assertEqual(det_parallelism(16, 1), 1) + self.assertEqual(det_parallelism(par=5, maxpar=2), 2) + self.assertEqual(det_parallelism(par=5, maxpar=10), 5) + + def test_det_parallelism_mocked(self): + """Test det_parallelism function (with mocked ulimit/get_avail_core_count).""" + orig_get_avail_core_count = st.get_avail_core_count + + # mock number of available cores to 8 + st.get_avail_core_count = lambda: 8 + self.assertTrue(det_parallelism(), 8) + # make 'ulimit -u' return '40', which should result in default (max) parallelism of 4 ((40-15)/6) + st.run_cmd = mocked_run_cmd + self.assertTrue(det_parallelism(), 4) + self.assertTrue(det_parallelism(par=6), 4) + self.assertTrue(det_parallelism(maxpar=2), 2) + + st.get_avail_core_count = orig_get_avail_core_count + + def test_det_terminal_size(self): + """Test det_terminal_size function.""" + (height, width) = st.det_terminal_size() + self.assertTrue(isinstance(height, int) and height > 0) + self.assertTrue(isinstance(width, int) and width > 0) + + def suite(): """ returns all the testcases in this module """ return TestLoader().loadTestsFromTestCase(SystemToolsTest) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 45eefd7ad8..d952c57b66 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,23 +37,19 @@ import easybuild.tools.modules as modules from easybuild.framework.easyconfig.easyconfig import EasyConfig, ActiveMNS +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import write_file +from easybuild.tools.modules import modules_tool from easybuild.tools.toolchain.utilities import search_toolchain from test.framework.utilities import find_full_path +from easybuild.tools import systemtools as st +import easybuild.tools.toolchain.compiler +easybuild.tools.toolchain.compiler.systemtools.get_compiler_family = lambda: st.POWER + class ToolchainTest(EnhancedTestCase): """ Baseclass for toolchain testcases """ - def setUp(self): - """Set up everything for a unit test.""" - super(ToolchainTest, self).setUp() - - # start with a clean slate - modules.modules_tool().purge() - - # make sure path with modules for testing is added to MODULEPATH - self.orig_modpath = os.environ.get('MODULEPATH', '') - os.environ['MODULEPATH'] = find_full_path(os.path.join('test', 'framework', 'modules')) - def get_toolchain(self, name, version=None): """Get a toolchain object instance to test with.""" tc_class, _ = search_toolchain(name) @@ -83,31 +79,23 @@ def test_get_variable_compilers(self): tc = self.get_toolchain("goalf", version="1.1.0-no-OFED") tc.prepare() - cc = tc.get_variable('CC') - self.assertEqual(cc, "gcc") - cxx = tc.get_variable('CXX') - self.assertEqual(cxx, "g++") - f77 = tc.get_variable('F77') - self.assertEqual(f77, "gfortran") - f90 = tc.get_variable('F90') - self.assertEqual(f90, "gfortran") - mpicc = tc.get_variable('MPICC') - self.assertEqual(mpicc, "mpicc") - mpicxx = tc.get_variable('MPICXX') - self.assertEqual(mpicxx, "mpicxx") - mpif77 = tc.get_variable('MPIF77') - self.assertEqual(mpif77, "mpif77") - mpif90 = tc.get_variable('MPIF90') - self.assertEqual(mpif90, "mpif90") - - ompi_cc = tc.get_variable('OMPI_CC') - self.assertEqual(ompi_cc, "gcc") - ompi_cxx = tc.get_variable('OMPI_CXX') - self.assertEqual(ompi_cxx, "g++") - ompi_f77 = tc.get_variable('OMPI_F77') - self.assertEqual(ompi_f77, "gfortran") - ompi_fc = tc.get_variable('OMPI_FC') - self.assertEqual(ompi_fc, "gfortran") + self.assertEqual(tc.get_variable('CC'), 'gcc') + self.assertEqual(tc.get_variable('CXX'), 'g++') + self.assertEqual(tc.get_variable('F77'), 'gfortran') + self.assertEqual(tc.get_variable('F90'), 'gfortran') + self.assertEqual(tc.get_variable('FC'), 'gfortran') + + self.assertEqual(tc.get_variable('MPICC'), 'mpicc') + self.assertEqual(tc.get_variable('MPICXX'), 'mpicxx') + # OpenMPI 1.4.5, so old MPI compiler wrappers for Fortran + self.assertEqual(tc.get_variable('MPIF77'), 'mpif77') + self.assertEqual(tc.get_variable('MPIF90'), 'mpif90') + self.assertEqual(tc.get_variable('MPIFC'), 'mpif90') + + self.assertEqual(tc.get_variable('OMPI_CC'), 'gcc') + self.assertEqual(tc.get_variable('OMPI_CXX'), 'g++') + self.assertEqual(tc.get_variable('OMPI_F77'), 'gfortran') + self.assertEqual(tc.get_variable('OMPI_FC'), 'gfortran') def test_get_variable_mpi_compilers(self): """Test get_variable function to obtain compiler variables.""" @@ -115,32 +103,23 @@ def test_get_variable_mpi_compilers(self): tc.set_options({'usempi': True}) tc.prepare() - cc = tc.get_variable('CC') - self.assertEqual(cc, "mpicc") - cxx = tc.get_variable('CXX') - self.assertEqual(cxx, "mpicxx") - f77 = tc.get_variable('F77') - self.assertEqual(f77, "mpif77") - f90 = tc.get_variable('F90') - self.assertEqual(f90, "mpif90") - - mpicc = tc.get_variable('MPICC') - self.assertEqual(mpicc, "mpicc") - mpicxx = tc.get_variable('MPICXX') - self.assertEqual(mpicxx, "mpicxx") - mpif77 = tc.get_variable('MPIF77') - self.assertEqual(mpif77, "mpif77") - mpif90 = tc.get_variable('MPIF90') - self.assertEqual(mpif90, "mpif90") - - ompi_cc = tc.get_variable('OMPI_CC') - self.assertEqual(ompi_cc, "gcc") - ompi_cxx = tc.get_variable('OMPI_CXX') - self.assertEqual(ompi_cxx, "g++") - ompi_f77 = tc.get_variable('OMPI_F77') - self.assertEqual(ompi_f77, "gfortran") - ompi_fc = tc.get_variable('OMPI_FC') - self.assertEqual(ompi_fc, "gfortran") + self.assertEqual(tc.get_variable('CC'), 'mpicc') + self.assertEqual(tc.get_variable('CXX'), 'mpicxx') + # OpenMPI 1.4.5, so old MPI compiler wrappers for Fortran + self.assertEqual(tc.get_variable('F77'), 'mpif77') + self.assertEqual(tc.get_variable('F90'), 'mpif90') + self.assertEqual(tc.get_variable('FC'), 'mpif90') + + self.assertEqual(tc.get_variable('MPICC'), 'mpicc') + self.assertEqual(tc.get_variable('MPICXX'), 'mpicxx') + self.assertEqual(tc.get_variable('MPIF77'), 'mpif77') + self.assertEqual(tc.get_variable('MPIF90'), 'mpif90') + self.assertEqual(tc.get_variable('MPIFC'), 'mpif90') + + self.assertEqual(tc.get_variable('OMPI_CC'), 'gcc') + self.assertEqual(tc.get_variable('OMPI_CXX'), 'g++') + self.assertEqual(tc.get_variable('OMPI_F77'), 'gfortran') + self.assertEqual(tc.get_variable('OMPI_FC'), 'gfortran') def test_get_variable_seq_compilers(self): """Test get_variable function to obtain compiler variables.""" @@ -148,14 +127,11 @@ def test_get_variable_seq_compilers(self): tc.set_options({'usempi': True}) tc.prepare() - cc_seq = tc.get_variable('CC_SEQ') - self.assertEqual(cc_seq, "gcc") - cxx_seq = tc.get_variable('CXX_SEQ') - self.assertEqual(cxx_seq, "g++") - f77_seq = tc.get_variable('F77_SEQ') - self.assertEqual(f77_seq, "gfortran") - f90_seq = tc.get_variable('F90_SEQ') - self.assertEqual(f90_seq, "gfortran") + self.assertEqual(tc.get_variable('CC_SEQ'), 'gcc') + self.assertEqual(tc.get_variable('CXX_SEQ'), 'g++') + self.assertEqual(tc.get_variable('F77_SEQ'), 'gfortran') + self.assertEqual(tc.get_variable('F90_SEQ'), 'gfortran') + self.assertEqual(tc.get_variable('FC_SEQ'), 'gfortran') def test_get_variable_libs_list(self): """Test get_variable function to obtain list of libraries.""" @@ -193,7 +169,7 @@ def test_validate_pass_by_value(self): def test_optimization_flags(self): """Test whether optimization flags are being set correctly.""" - flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS'] + flag_vars = ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS'] # check default optimization flag (e.g. -O2) tc = self.get_toolchain("goalf", version="1.1.0-no-OFED") @@ -215,11 +191,12 @@ def test_optimization_flags(self): self.assertTrue(tc.COMPILER_SHARED_OPTION_MAP[opt] in flags) else: self.assertTrue(tc.COMPILER_SHARED_OPTION_MAP[opt] in flags) + modules.modules_tool().purge() def test_optimization_flags_combos(self): """Test whether combining optimization levels works as expected.""" - flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS'] + flag_vars = ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS'] # check combining of optimization flags (doesn't make much sense) # lowest optimization should always be picked @@ -230,6 +207,7 @@ def test_optimization_flags_combos(self): flags = tc.get_variable(var) flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP['lowopt'] self.assertTrue(flag in flags) + modules.modules_tool().purge() tc = self.get_toolchain("goalf", version="1.1.0-no-OFED") tc.set_options({'noopt': True, 'lowopt':True}) @@ -238,6 +216,7 @@ def test_optimization_flags_combos(self): flags = tc.get_variable(var) flag = '-%s' % tc.COMPILER_SHARED_OPTION_MAP['noopt'] self.assertTrue(flag in flags) + modules.modules_tool().purge() tc = self.get_toolchain("goalf", version="1.1.0-no-OFED") tc.set_options({'noopt':True, 'lowopt': True, 'opt':True}) @@ -250,7 +229,7 @@ def test_optimization_flags_combos(self): def test_misc_flags_shared(self): """Test whether shared compiler flags are set correctly.""" - flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS'] + flag_vars = ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS'] # setting option should result in corresponding flag to be set (shared options) for opt in ['pic', 'verbose', 'debug', 'static', 'shared']: @@ -266,11 +245,12 @@ def test_misc_flags_shared(self): self.assertTrue(flag in flags, "%s: True means %s in %s" % (opt, flag, flags)) else: self.assertTrue(flag not in flags, "%s: False means no %s in %s" % (opt, flag, flags)) + modules.modules_tool().purge() def test_misc_flags_unique(self): """Test whether unique compiler flags are set correctly.""" - flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS'] + flag_vars = ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS'] # setting option should result in corresponding flag to be set (unique options) for opt in ['unroll', 'optarch', 'openmp']: @@ -278,17 +258,21 @@ def test_misc_flags_unique(self): tc = self.get_toolchain("goalf", version="1.1.0-no-OFED") tc.set_options({opt: enable}) tc.prepare() - flag = '-%s' % tc.COMPILER_UNIQUE_OPTION_MAP[opt] + if opt == 'optarch': + flag = '-%s' % tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[tc.arch] + else: + flag = '-%s' % tc.COMPILER_UNIQUE_OPTION_MAP[opt] for var in flag_vars: flags = tc.get_variable(var) if enable: self.assertTrue(flag in flags, "%s: True means %s in %s" % (opt, flag, flags)) else: self.assertTrue(flag not in flags, "%s: False means no %s in %s" % (opt, flag, flags)) + modules.modules_tool().purge() def test_override_optarch(self): """Test whether overriding the optarch flag works.""" - flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS'] + flag_vars = ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS'] for optarch_var in ['march=lovelylovelysandybridge', None]: build_options = {'optarch': optarch_var} init_config(build_options=build_options) @@ -300,7 +284,8 @@ def test_override_optarch(self): if optarch_var is not None: flag = '-%s' % optarch_var else: - flag = '-march=native' + # default optarch flag + flag = tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[tc.arch] for var in flag_vars: flags = tc.get_variable(var) @@ -308,11 +293,12 @@ def test_override_optarch(self): self.assertTrue(flag in flags, "optarch: True means %s in %s" % (flag, flags)) else: self.assertFalse(flag in flags, "optarch: False means no %s in %s" % (flag, flags)) + modules.modules_tool().purge() def test_misc_flags_unique_fortran(self): """Test whether unique Fortran compiler flags are set correctly.""" - flag_vars = ['FFLAGS', 'F90FLAGS'] + flag_vars = ['FCFLAGS', 'FFLAGS', 'F90FLAGS'] # setting option should result in corresponding flag to be set (Fortran unique options) for opt in ['i8', 'r8']: @@ -327,11 +313,12 @@ def test_misc_flags_unique_fortran(self): self.assertTrue(flag in flags, "%s: True means %s in %s" % (opt, flag, flags)) else: self.assertTrue(flag not in flags, "%s: False means no %s in %s" % (opt, flag, flags)) + modules.modules_tool().purge() def test_precision_flags(self): """Test whether precision flags are being set correctly.""" - flag_vars = ['CFLAGS', 'CXXFLAGS', 'FFLAGS', 'F90FLAGS'] + flag_vars = ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS'] # check default precision flag tc = self.get_toolchain("goalf", version="1.1.0-no-OFED") @@ -354,6 +341,7 @@ def test_precision_flags(self): self.assertTrue(val in flags) else: self.assertTrue(val not in flags) + modules.modules_tool().purge() def test_cgoolf_toolchain(self): """Test for cgoolf toolchain.""" @@ -364,6 +352,7 @@ def test_cgoolf_toolchain(self): self.assertEqual(tc.get_variable('CXX'), 'clang++') self.assertEqual(tc.get_variable('F77'), 'gfortran') self.assertEqual(tc.get_variable('F90'), 'gfortran') + self.assertEqual(tc.get_variable('FC'), 'gfortran') def test_comp_family(self): """Test determining compiler family.""" @@ -371,6 +360,30 @@ def test_comp_family(self): tc.prepare() self.assertEqual(tc.comp_family(), "GCC") + def test_mpi_family(self): + """Test determining MPI family.""" + # check subtoolchain w/o MPI + tc = self.get_toolchain("GCC", version="4.7.2") + tc.prepare() + self.assertEqual(tc.mpi_family(), None) + modules.modules_tool().purge() + + # check full toolchain including MPI + tc = self.get_toolchain("goalf", version="1.1.0-no-OFED") + tc.prepare() + self.assertEqual(tc.mpi_family(), "OpenMPI") + modules.modules_tool().purge() + + # check another one + tmpdir, imkl_module_path, imkl_module_txt = self.setup_sandbox_for_intel_fftw() + tc = self.get_toolchain("ictce", version="4.1.13") + tc.prepare() + self.assertEqual(tc.mpi_family(), "IntelMPI") + + # cleanup + shutil.rmtree(tmpdir) + write_file(imkl_module_path, imkl_module_txt) + def test_goolfc(self): """Test whether goolfc is handled properly.""" tc = self.get_toolchain("goolfc", version="1.3.12") @@ -379,7 +392,7 @@ def test_goolfc(self): tc.prepare() nvcc_flags = r' '.join([ - r'-Xcompiler="-O2 -march=native"', + r'-Xcompiler="-O2 -%s"' % tc.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[tc.arch], # the use of -lcudart in -Xlinker is a bit silly but hard to avoid r'-Xlinker=".* -lm -lrt -lcudart -lpthread"', r' '.join(["-gencode %s" % x for x in opts['cuda_gencode']]), @@ -399,25 +412,31 @@ def test_goolfc(self): # check CUDA runtime lib self.assertTrue("-lrt -lcudart" in tc.get_variable('LIBS')) - def test_ictce_toolchain(self): - """Test for ictce toolchain.""" + def setup_sandbox_for_intel_fftw(self, imklver='10.3.12.361'): + """Set up sandbox for Intel FFTW""" # hack to make Intel FFTW lib check pass # rewrite $root in imkl module so we can put required lib*.a files in place tmpdir = tempfile.mkdtemp() test_modules_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules')) - imkl_module_path = os.path.join(test_modules_path, 'imkl', '10.3.12.361') + imkl_module_path = os.path.join(test_modules_path, 'imkl', imklver) imkl_module_txt = open(imkl_module_path, 'r').read() regex = re.compile('^(set\s*root).*$', re.M) imkl_module_alt_txt = regex.sub(r'\1\t%s' % tmpdir, imkl_module_txt) open(imkl_module_path, 'w').write(imkl_module_alt_txt) fftw_libs = ['fftw3xc_intel', 'fftw3x_cdft', 'mkl_cdft_core', 'mkl_blacs_intelmpi_lp64'] - fftw_libs += ['mkl_blacs_intelmpi_lp64', 'mkl_intel_lp64', 'mkl_sequential', 'mkl_core'] - for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64']: + fftw_libs += ['mkl_blacs_intelmpi_lp64', 'mkl_intel_lp64', 'mkl_sequential', 'mkl_core', 'mkl_intel_ilp64'] + for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64', 'lib/em64t']: os.makedirs(os.path.join(tmpdir, subdir)) for fftlib in fftw_libs: - open(os.path.join(tmpdir, subdir, 'lib%s.a' % fftlib), 'w').write('foo') + write_file(os.path.join(tmpdir, subdir, 'lib%s.a' % fftlib), 'foo') + + return tmpdir, imkl_module_path, imkl_module_txt + + def test_ictce_toolchain(self): + """Test for ictce toolchain.""" + tmpdir, imkl_module_path, imkl_module_txt = self.setup_sandbox_for_intel_fftw() tc = self.get_toolchain("ictce", version="4.1.13") tc.prepare() @@ -426,6 +445,8 @@ def test_ictce_toolchain(self): self.assertEqual(tc.get_variable('CXX'), 'icpc') self.assertEqual(tc.get_variable('F77'), 'ifort') self.assertEqual(tc.get_variable('F90'), 'ifort') + self.assertEqual(tc.get_variable('FC'), 'ifort') + modules.modules_tool().purge() tc = self.get_toolchain("ictce", version="4.1.13") opts = {'usempi': True} @@ -436,10 +457,13 @@ def test_ictce_toolchain(self): self.assertEqual(tc.get_variable('CXX'), 'mpicxx') self.assertEqual(tc.get_variable('F77'), 'mpif77') self.assertEqual(tc.get_variable('F90'), 'mpif90') + self.assertEqual(tc.get_variable('FC'), 'mpif90') self.assertEqual(tc.get_variable('MPICC'), 'mpicc') self.assertEqual(tc.get_variable('MPICXX'), 'mpicxx') self.assertEqual(tc.get_variable('MPIF77'), 'mpif77') self.assertEqual(tc.get_variable('MPIF90'), 'mpif90') + self.assertEqual(tc.get_variable('MPIFC'), 'mpif90') + modules.modules_tool().purge() tc = self.get_toolchain("ictce", version="4.1.13") opts = {'usempi': True, 'openmp': True} @@ -448,31 +472,208 @@ def test_ictce_toolchain(self): self.assertTrue('-mt_mpi' in tc.get_variable('CFLAGS')) self.assertTrue('-mt_mpi' in tc.get_variable('CXXFLAGS')) + self.assertTrue('-mt_mpi' in tc.get_variable('FCFLAGS')) self.assertTrue('-mt_mpi' in tc.get_variable('FFLAGS')) self.assertTrue('-mt_mpi' in tc.get_variable('F90FLAGS')) self.assertEqual(tc.get_variable('CC'), 'mpicc') self.assertEqual(tc.get_variable('CXX'), 'mpicxx') self.assertEqual(tc.get_variable('F77'), 'mpif77') self.assertEqual(tc.get_variable('F90'), 'mpif90') + self.assertEqual(tc.get_variable('FC'), 'mpif90') self.assertEqual(tc.get_variable('MPICC'), 'mpicc') self.assertEqual(tc.get_variable('MPICXX'), 'mpicxx') self.assertEqual(tc.get_variable('MPIF77'), 'mpif77') self.assertEqual(tc.get_variable('MPIF90'), 'mpif90') + self.assertEqual(tc.get_variable('MPIFC'), 'mpif90') # cleanup shutil.rmtree(tmpdir) - open(imkl_module_path, 'w').write(imkl_module_txt) + write_file(imkl_module_path, imkl_module_txt) + + def test_toolchain_verification(self): + """Test verification of toolchain definition.""" + tc = self.get_toolchain("goalf", version="1.1.0-no-OFED") + tc.prepare() + modules.modules_tool().purge() - def tearDown(self): - """Cleanup.""" - # purge any loaded modules before restoring $MODULEPATH + # toolchain modules missing a toolchain element should fail verification + error_msg = "List of toolchain dependency modules and toolchain definition do not match" + tc = self.get_toolchain("goalf", version="1.1.0-no-OFED-brokenFFTW") + self.assertErrorRegex(EasyBuildError, error_msg, tc.prepare) modules.modules_tool().purge() - super(ToolchainTest, self).tearDown() + tc = self.get_toolchain("goalf", version="1.1.0-no-OFED-brokenBLACS") + self.assertErrorRegex(EasyBuildError, error_msg, tc.prepare) + modules.modules_tool().purge() + + # missing optional toolchain elements are fine + tc = self.get_toolchain('goolfc', version='1.3.12') + opts = {'cuda_gencode': ['arch=compute_35,code=sm_35', 'arch=compute_10,code=compute_10']} + tc.set_options(opts) + tc.prepare() + + def test_nosuchtoolchain(self): + """Test preparing for a toolchain for which no module is available.""" + tc = self.get_toolchain('intel', version='1970.01') + self.assertErrorRegex(EasyBuildError, "No module found for toolchain", tc.prepare) + + def test_mpi_cmd_for(self): + """Test mpi_cmd_for function.""" + tmpdir, imkl_module_path, imkl_module_txt = self.setup_sandbox_for_intel_fftw() + + tc = self.get_toolchain('ictce', version='4.1.13') + tc.prepare() + + mpi_cmd_for_re = re.compile("^mpirun --file=.*/mpdboot -machinefile .*/nodes -np 4 test$") + self.assertTrue(mpi_cmd_for_re.match(tc.mpi_cmd_for('test', 4))) + + # cleanup + shutil.rmtree(tmpdir) + write_file(imkl_module_path, imkl_module_txt) + + def test_prepare_deps(self): + """Test preparing for a toolchain when dependencies are involved.""" + tc = self.get_toolchain('GCC', version='4.6.4') + deps = [ + { + 'name': 'OpenMPI', + 'version': '1.6.4', + 'full_mod_name': 'OpenMPI/1.6.4-GCC-4.6.4', + 'short_mod_name': 'OpenMPI/1.6.4-GCC-4.6.4', + 'external_module': False, + }, + ] + tc.add_dependencies(deps) + tc.prepare() + mods = ['GCC/4.6.4', 'hwloc/1.6.2-GCC-4.6.4', 'OpenMPI/1.6.4-GCC-4.6.4'] + self.assertTrue([m['mod_name'] for m in modules_tool().list()], mods) + + def test_prepare_deps_external(self): + """Test preparing for a toolchain when dependencies and external modules are involved.""" + deps = [ + { + 'name': 'OpenMPI', + 'version': '1.6.4', + 'full_mod_name': 'OpenMPI/1.6.4-GCC-4.6.4', + 'short_mod_name': 'OpenMPI/1.6.4-GCC-4.6.4', + 'external_module': False, + 'external_module_metadata': {}, + }, + # no metadata available + { + 'name': None, + 'version': None, + 'full_mod_name': 'toy/0.0', + 'short_mod_name': 'toy/0.0', + 'external_module': True, + 'external_module_metadata': {}, + } + ] + tc = self.get_toolchain('GCC', version='4.6.4') + tc.add_dependencies(deps) + tc.prepare() + mods = ['GCC/4.6.4', 'hwloc/1.6.2-GCC-4.6.4', 'OpenMPI/1.6.4-GCC-4.6.4', 'toy/0.0'] + self.assertTrue([m['mod_name'] for m in modules_tool().list()], mods) + self.assertTrue(os.environ['EBROOTTOY'].endswith('software/toy/0.0')) + self.assertEqual(os.environ['EBVERSIONTOY'], '0.0') + self.assertFalse('EBROOTFOOBAR' in os.environ) + + # with metadata + deps[1] = { + 'full_mod_name': 'toy/0.0', + 'short_mod_name': 'toy/0.0', + 'external_module': True, + 'external_module_metadata': { + 'name': ['toy', 'foobar'], + 'version': ['1.2.3', '4.5'], + 'prefix': 'FOOBAR_PREFIX', + } + } + tc = self.get_toolchain('GCC', version='4.6.4') + tc.add_dependencies(deps) + os.environ['FOOBAR_PREFIX'] = '/foo/bar' + tc.prepare() + mods = ['GCC/4.6.4', 'hwloc/1.6.2-GCC-4.6.4', 'OpenMPI/1.6.4-GCC-4.6.4', 'toy/0.0'] + self.assertTrue([m['mod_name'] for m in modules_tool().list()], mods) + self.assertEqual(os.environ['EBROOTTOY'], '/foo/bar') + self.assertEqual(os.environ['EBVERSIONTOY'], '1.2.3') + self.assertEqual(os.environ['EBROOTFOOBAR'], '/foo/bar') + self.assertEqual(os.environ['EBVERSIONFOOBAR'], '4.5') + + self.assertEqual(modules.get_software_root('foobar'), '/foo/bar') + self.assertEqual(modules.get_software_version('toy'), '1.2.3') - os.environ['MODULEPATH'] = self.orig_modpath - # reinitialize modules tool after touching $MODULEPATH - modules.modules_tool() + def test_old_new_iccifort(self): + """Test whether preparing for old/new Intel compilers works correctly.""" + tmpdir1, imkl_module_path1, imkl_module_txt1 = self.setup_sandbox_for_intel_fftw(imklver='10.3.12.361') + tmpdir2, imkl_module_path2, imkl_module_txt2 = self.setup_sandbox_for_intel_fftw(imklver='10.2.6.038') + + # incl. -lguide + libblas_mt_ictce3 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" + libblas_mt_ictce3 += " -Wl,--end-group -Wl,-Bdynamic -liomp5 -lguide -lpthread" + + # no -lguide + libblas_mt_ictce4 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" + libblas_mt_ictce4 += " -Wl,--end-group -Wl,-Bdynamic -liomp5 -lpthread" + + # incl. -lmkl_solver* + libscalack_ictce3 = "-lmkl_scalapack_lp64 -lmkl_solver_lp64_sequential -lmkl_blacs_intelmpi_lp64" + libscalack_ictce3 += " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core" + + # no -lmkl_solver* + libscalack_ictce4 = "-lmkl_scalapack_lp64 -lmkl_blacs_intelmpi_lp64 -lmkl_intel_lp64 -lmkl_sequential -lmkl_core" + + libblas_mt_goolfc = "-lopenblas -lgfortran" + libscalack_goolfc = "-lscalapack -lopenblas -lgfortran" + + tc = self.get_toolchain('goolfc', version='1.3.12') + tc.prepare() + self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_goolfc) + self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_goolfc) + modules_tool().purge() + + tc = self.get_toolchain('ictce', version='4.1.13') + tc.prepare() + self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_ictce4) + self.assertTrue(libscalack_ictce4 in os.environ['LIBSCALAPACK']) + modules_tool().purge() + + tc = self.get_toolchain('ictce', version='3.2.2.u3') + tc.prepare() + self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_ictce3) + self.assertTrue(libscalack_ictce3 in os.environ['LIBSCALAPACK']) + modules_tool().purge() + + tc = self.get_toolchain('ictce', version='4.1.13') + tc.prepare() + self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_ictce4) + self.assertTrue(libscalack_ictce4 in os.environ['LIBSCALAPACK']) + modules_tool().purge() + + tc = self.get_toolchain('ictce', version='3.2.2.u3') + tc.prepare() + self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_ictce3) + self.assertTrue(libscalack_ictce3 in os.environ['LIBSCALAPACK']) + modules_tool().purge() + + libscalack_ictce4 = libscalack_ictce4.replace('_lp64', '_ilp64') + tc = self.get_toolchain('ictce', version='4.1.13') + opts = {'i8': True} + tc.set_options(opts) + tc.prepare() + self.assertTrue(libscalack_ictce4 in os.environ['LIBSCALAPACK']) + modules_tool().purge() + + tc = self.get_toolchain('goolfc', version='1.3.12') + tc.prepare() + self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_goolfc) + self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_goolfc) + + # cleanup + shutil.rmtree(tmpdir1) + shutil.rmtree(tmpdir2) + write_file(imkl_module_path1, imkl_module_txt1) + write_file(imkl_module_path2, imkl_module_txt2) def suite(): """ return all the tests""" diff --git a/test/framework/toolchainvariables.py b/test/framework/toolchainvariables.py index 9d7b996b33..5a9f8c369e 100644 --- a/test/framework/toolchainvariables.py +++ b/test/framework/toolchainvariables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 6c0a5d4cfd..2aa8469112 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -27,7 +27,6 @@ @author: Kenneth Hoste (Ghent University) """ -import fileinput import glob import grp import os @@ -37,13 +36,18 @@ import sys import tempfile from test.framework.utilities import EnhancedTestCase +from test.framework.package import mock_fpm from unittest import TestLoader from unittest import main as unittestmain from vsc.utils.fancylogger import setLogLevelDebug, logToScreen import easybuild.tools.module_naming_scheme # required to dynamically load test module naming scheme(s) +from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import mkdir, write_file +from easybuild.tools.config import get_module_syntax +from easybuild.tools.filetools import adjust_permissions, mkdir, read_file, which, write_file +from easybuild.tools.modules import modules_tool +from easybuild.tools.version import VERSION as EASYBUILD_VERSION class ToyBuildTest(EnhancedTestCase): @@ -56,24 +60,9 @@ def setUp(self): fd, self.dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) - # adjust PYTHONPATH such that test easyblocks are found - import easybuild - eb_blocks_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'sandbox')) - if not eb_blocks_path in sys.path: - sys.path.append(eb_blocks_path) - easybuild = reload(easybuild) - - import easybuild.easyblocks - reload(easybuild.easyblocks) - reload(easybuild.tools.module_naming_scheme) - # clear log write_file(self.logfile, '') - self.test_buildpath = tempfile.mkdtemp() - self.test_installpath = tempfile.mkdtemp() - self.test_sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'sources') - def tearDown(self): """Cleanup.""" super(ToyBuildTest, self).tearDown() @@ -93,10 +82,14 @@ def check_toy(self, installpath, outtxt, version='0.0', versionprefix='', versio # if the module exists, it should be fine toy_module = os.path.join(installpath, 'modules', 'all', 'toy', full_version) msg = "module for toy build toy/%s found (path %s)" % (full_version, toy_module) + if get_module_syntax() == 'Lua': + toy_module += '.lua' self.assertTrue(os.path.exists(toy_module), msg) # module file is symlinked according to moduleclass toy_module_symlink = os.path.join(installpath, 'modules', 'tools', 'toy', full_version) + if get_module_syntax() == 'Lua': + toy_module_symlink += '.lua' self.assertTrue(os.path.islink(toy_module_symlink)) self.assertTrue(os.path.exists(toy_module_symlink)) @@ -146,6 +139,8 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True raise_error=raise_error) except Exception, err: myerr = err + if raise_error: + raise myerr if verify: self.check_toy(self.test_installpath, outtxt, versionsuffix=versionsuffix) @@ -153,7 +148,7 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True if test_readme: # make sure postinstallcmds were used toy_install_path = os.path.join(self.test_installpath, 'software', 'toy', full_ver) - self.assertEqual(open(os.path.join(toy_install_path, 'README'), 'r').read(), "TOY\n") + self.assertEqual(read_file(os.path.join(toy_install_path, 'README')), "TOY\n") # make sure full test report was dumped, and contains sensible information if test_report is not None: @@ -171,35 +166,32 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True r"List of loaded modules", r"Environment", ] - test_report_txt = open(test_report).read() + test_report_txt = read_file(test_report) for regex_pattern in regex_patterns: regex = re.compile(regex_pattern, re.M) msg = "Pattern %s found in full test report: %s" % (regex.pattern, test_report_txt) self.assertTrue(regex.search(test_report_txt), msg) - if raise_error and (myerr is not None): - raise myerr + return outtxt def test_toy_broken(self): """Test deliberately broken toy build.""" tmpdir = tempfile.mkdtemp() broken_toy_ec = os.path.join(tmpdir, "toy-broken.eb") toy_ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb') - broken_toy_ec_txt = open(toy_ec_file, 'r').read() + broken_toy_ec_txt = read_file(toy_ec_file) broken_toy_ec_txt += "checksums = ['clearywrongchecksum']" - f = open(broken_toy_ec, 'w') - f.write(broken_toy_ec_txt) - f.close() + write_file(broken_toy_ec, broken_toy_ec_txt) error_regex = "Checksum verification .* failed" self.assertErrorRegex(EasyBuildError, error_regex, self.test_toy_build, ec_file=broken_toy_ec, tmpdir=tmpdir, verify=False, fails=True, verbose=False, raise_error=True) # make sure log file is retained, also for failed build - log_path_pattern = os.path.join(tmpdir, 'easybuild-*', 'easybuild-toy-0.0*.log') + log_path_pattern = os.path.join(tmpdir, 'eb-*', 'easybuild-toy-0.0*.log') self.assertTrue(len(glob.glob(log_path_pattern)) == 1, "Log file found at %s" % log_path_pattern) # make sure individual test report is retained, also for failed build - test_report_fp_pattern = os.path.join(tmpdir, 'easybuild-*', 'easybuild-toy-0.0*test_report.md') + test_report_fp_pattern = os.path.join(tmpdir, 'eb-*', 'easybuild-toy-0.0*test_report.md') self.assertTrue(len(glob.glob(test_report_fp_pattern)) == 1, "Test report %s found" % test_report_fp_pattern) # test dumping full test report (doesn't raise an exception) @@ -219,12 +211,13 @@ def test_toy_tweaked(self): # tweak easyconfig by appending to it ec_extra = '\n'.join([ "versionsuffix = '-tweaked'", - "modextrapaths = {'SOMEPATH': ['foo/bar', 'baz']}", + "modextrapaths = {'SOMEPATH': ['foo/bar', 'baz', '']}", "modextravars = {'FOO': 'bar'}", "modloadmsg = 'THANKS FOR LOADING ME, I AM %(name)s v%(version)s'", - "modtclfooter = 'puts stderr \"oh hai!\"'", + "modtclfooter = 'puts stderr \"oh hai!\"'", # ignored when module syntax is Lua + "modluafooter = 'io.stderr:write(\"oh hai!\")'" # ignored when module syntax is Tcl ]) - open(ec_file, 'a').write(ec_extra) + write_file(ec_file, ec_extra, append=True) args = [ ec_file, @@ -237,13 +230,27 @@ def test_toy_tweaked(self): outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True) self.check_toy(self.test_installpath, outtxt, versionsuffix='-tweaked') toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-tweaked') - toy_module_txt = open(toy_module, 'r').read() - - self.assertTrue(re.search('setenv\s*FOO\s*"bar"', toy_module_txt)) - self.assertTrue(re.search('prepend-path\s*SOMEPATH\s*\$root/foo/bar', toy_module_txt)) - self.assertTrue(re.search('prepend-path\s*SOMEPATH\s*\$root/baz', toy_module_txt)) - self.assertTrue(re.search('module-info mode load.*\n\s*puts stderr\s*.*I AM toy v0.0', toy_module_txt)) - self.assertTrue(re.search('puts stderr "oh hai!"', toy_module_txt)) + if get_module_syntax() == 'Lua': + toy_module += '.lua' + toy_module_txt = read_file(toy_module) + + if get_module_syntax() == 'Tcl': + self.assertTrue(re.search(r'^setenv\s*FOO\s*"bar"$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root/foo/bar$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root/baz$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'module-info mode load.*\n\s*puts stderr\s*.*I AM toy v0.0"$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^puts stderr "oh hai!"$', toy_module_txt, re.M)) + elif get_module_syntax() == 'Lua': + self.assertTrue(re.search(r'^setenv\("FOO", "bar"\)', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", pathJoin\(root, "baz"\)\)$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", root\)$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^if mode\(\) == "load" then\n\s*io.stderr:write\(".*I AM toy v0.0"\)$', + toy_module_txt, re.M)) + self.assertTrue(re.search(r'^io.stderr:write\("oh hai!"\)$', toy_module_txt, re.M)) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) def test_toy_buggy_easyblock(self): """Test build using a buggy/broken easyblock, make sure a traceback is reported.""" @@ -255,7 +262,7 @@ def test_toy_buggy_easyblock(self): 'verify': False, 'verbose': False, } - err_regex = r"crashed with an error.*Traceback[\S\s]*toy_buggy.py.*build_step[\S\s]*global name 'run_cmd'" + err_regex = r"Traceback[\S\s]*toy_buggy.py.*build_step[\S\s]*global name 'run_cmd'" self.assertErrorRegex(EasyBuildError, err_regex, self.test_toy_build, **kwargs) def test_toy_build_formatv2(self): @@ -357,11 +364,9 @@ def test_toy_download_sources(self): tmpdir = tempfile.mkdtemp() # copy toy easyconfig file, and append source_urls to it shutil.copy2(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb'), tmpdir) - ec_file = os.path.join(tmpdir, 'toy-0.0.eb') - f = open(ec_file, 'a') source_url = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sandbox', 'sources', 'toy') - f.write('\nsource_urls = ["file://%s"]\n' % source_url) - f.close() + ec_file = os.path.join(tmpdir, 'toy-0.0.eb') + write_file(ec_file, '\nsource_urls = ["file://%s"]\n' % source_url, append=True) # unset $EASYBUILD_XPATH env vars, to make sure --prefix is picked up for cfg_opt in ['build', 'install', 'source']: @@ -418,14 +423,15 @@ def test_toy_permissions(self): ('030', None, curr_grp, 0740, 0640, 0740), # no write for group, with specified group ('077', None, None, 0700, 0600, 0700), # no access for other/group ]: + # empty the install directory, to ensure any created directories adher to the permissions + shutil.rmtree(self.test_installpath) + if cfg_group is None and ec_group is None: allargs = [toy_ec_file] elif ec_group is not None: shutil.copy2(toy_ec_file, self.test_buildpath) tmp_ec_file = os.path.join(self.test_buildpath, os.path.basename(toy_ec_file)) - f = open(tmp_ec_file, 'a') - f.write("\ngroup = '%s'" % ec_group) - f.close() + write_file(tmp_ec_file, "\ngroup = '%s'" % ec_group, append=True) allargs = [tmp_ec_file] allargs.extend(args) if umask is not None: @@ -458,8 +464,12 @@ def test_toy_permissions(self): (('modules', ), dir_perms), (('modules', 'all'), dir_perms), (('modules', 'all', 'toy'), dir_perms), - (('modules', 'all', 'toy', '0.0'), fil_perms), ]) + if get_module_syntax() == 'Tcl': + paths_perms.append((('modules', 'all', 'toy', '0.0'), fil_perms)) + elif get_module_syntax() == 'Lua': + paths_perms.append((('modules', 'all', 'toy', '0.0.lua'), fil_perms)) + for path, correct_perms in paths_perms: fullpath = glob.glob(os.path.join(self.test_installpath, *path))[0] perms = os.stat(fullpath).st_mode & 0777 @@ -469,8 +479,30 @@ def test_toy_permissions(self): path_gid = os.stat(fullpath).st_gid self.assertEqual(path_gid, grp.getgrnam(group).gr_gid) - # cleanup for next iteration - shutil.rmtree(self.test_installpath) + # restore original umask + os.umask(orig_umask) + + def test_toy_permissions_installdir(self): + """Test --read-only-installdir and --group-write-installdir.""" + # set umask hard to verify default reliably + orig_umask = os.umask(0022) + + self.test_toy_build() + installdir_perms = os.stat(os.path.join(self.test_installpath, 'software', 'toy', '0.0')).st_mode & 0777 + self.assertEqual(installdir_perms, 0755, "%s has default permissions" % self.test_installpath) + shutil.rmtree(self.test_installpath) + + self.test_toy_build(extra_args=['--read-only-installdir']) + installdir_perms = os.stat(os.path.join(self.test_installpath, 'software', 'toy', '0.0')).st_mode & 0777 + self.assertEqual(installdir_perms, 0555, "%s has read-only permissions" % self.test_installpath) + installdir_perms = os.stat(os.path.join(self.test_installpath, 'software', 'toy')).st_mode & 0777 + self.assertEqual(installdir_perms, 0755, "%s has default permissions" % self.test_installpath) + adjust_permissions(os.path.join(self.test_installpath, 'software', 'toy', '0.0'), stat.S_IWUSR, add=True) + shutil.rmtree(self.test_installpath) + + self.test_toy_build(extra_args=['--group-writable-installdir']) + installdir_perms = os.stat(os.path.join(self.test_installpath, 'software', 'toy', '0.0')).st_mode & 0777 + self.assertEqual(installdir_perms, 0775, "%s has group write permissions" % self.test_installpath) # restore original umask os.umask(orig_umask) @@ -520,64 +552,61 @@ def test_allow_system_deps(self): # copy toy easyconfig file, and append source_urls to it shutil.copy2(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb'), tmpdir) ec_file = os.path.join(tmpdir, 'toy-0.0.eb') - f = open(ec_file, 'a') - f.write("\nallow_system_deps = [('Python', SYS_PYTHON_VERSION)]\n") - f.close() + write_file(ec_file, "\nallow_system_deps = [('Python', SYS_PYTHON_VERSION)]\n", append=True) self.test_toy_build(ec_file=ec_file) + shutil.rmtree(tmpdir) def test_toy_hierarchical(self): """Test toy build under example hierarchical module naming scheme.""" + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + self.setup_hierarchical_modules() mod_prefix = os.path.join(self.test_installpath, 'modules', 'all') - # simply copy module files under 'Core' and 'Compiler' to test install path - # EasyBuild is responsible for making sure that the toolchain can be loaded using the short module name - install_mod_path = os.path.join(self.test_installpath, 'modules', 'all') - mkdir(install_mod_path, parents=True) - for mod_subdir in ['Core', 'Compiler']: - src_mod_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules', mod_subdir) - shutil.copytree(src_mod_path, os.path.join(install_mod_path, mod_subdir)) - - # tweak prepend-path statements in GCC/OpenMPI modules to ensure correct paths - for modfile in [ - os.path.join(install_mod_path, 'Core', 'GCC', '4.7.2'), - os.path.join(install_mod_path, 'Compiler', 'GCC', '4.7.2', 'OpenMPI', '1.6.4'), - ]: - for line in fileinput.input(modfile, inplace=1): - line = re.sub(r"(module\s*use\s*)/tmp/modules/all", - r"\1%s/modules/all" % self.test_installpath, - line) - sys.stdout.write(line) - args = [ - os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb'), + os.path.join(test_easyconfigs, 'toy-0.0.eb'), '--sourcepath=%s' % self.test_sourcepath, '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % self.test_installpath, '--debug', '--unittest-file=%s' % self.logfile, '--force', - '--robot=%s' % os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs'), + '--robot=%s' % test_easyconfigs, '--module-naming-scheme=HierarchicalMNS', ] # test module paths/contents with gompi build extra_args = [ - '--try-toolchain=gompi,1.4.10', + '--try-toolchain=goolf,1.4.10', ] self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True) # make sure module file is installed in correct path toy_module_path = os.path.join(mod_prefix, 'MPI', 'GCC', '4.7.2', 'OpenMPI', '1.6.4', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_module_path += '.lua' self.assertTrue(os.path.exists(toy_module_path)) - # check that toolchain load is expanded to loads for toolchain dependencies - modtxt = open(toy_module_path, 'r').read() - self.assertFalse(re.search("load gompi", modtxt)) - self.assertTrue(re.search("load GCC", modtxt)) - self.assertTrue(re.search("load OpenMPI", modtxt)) + # check that toolchain load is expanded to loads for toolchain dependencies, + # except for the ones that extend $MODULEPATH to make the toy module available + if get_module_syntax() == 'Tcl': + load_regex_template = "load %s" + elif get_module_syntax() == 'Lua': + load_regex_template = r'load\("%s/.*"\)' + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + + modtxt = read_file(toy_module_path) + for dep in ['goolf', 'GCC', 'OpenMPI']: + load_regex = re.compile(load_regex_template % dep) + self.assertFalse(load_regex.search(modtxt), "Pattern '%s' not found in %s" % (load_regex.pattern, modtxt)) + for dep in ['OpenBLAS', 'FFTW', 'ScaLAPACK']: + load_regex = re.compile(load_regex_template % dep) + self.assertTrue(load_regex.search(modtxt), "Pattern '%s' found in %s" % (load_regex.pattern, modtxt)) - # test module path with GCC/4.8.2 build + os.remove(toy_module_path) + + # test module path with GCC/4.7.2 build extra_args = [ '--try-toolchain=GCC,4.7.2', ] @@ -585,13 +614,16 @@ def test_toy_hierarchical(self): # make sure module file is installed in correct path toy_module_path = os.path.join(mod_prefix, 'Compiler', 'GCC', '4.7.2', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_module_path += '.lua' self.assertTrue(os.path.exists(toy_module_path)) # no dependencies or toolchain => no module load statements in module file - modtxt = open(toy_module_path, 'r').read() + modtxt = read_file(toy_module_path) self.assertFalse(re.search("module load", modtxt)) + os.remove(toy_module_path) - # test module path with GCC/4.8.2 build, pretend to be an MPI lib by setting moduleclass + # test module path with GCC/4.7.2 build, pretend to be an MPI lib by setting moduleclass extra_args = [ '--try-toolchain=GCC,4.7.2', '--try-amend=moduleclass=mpi', @@ -600,12 +632,37 @@ def test_toy_hierarchical(self): # make sure module file is installed in correct path toy_module_path = os.path.join(mod_prefix, 'Compiler', 'GCC', '4.7.2', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_module_path += '.lua' self.assertTrue(os.path.exists(toy_module_path)) - # no dependencies or toolchain => no module load statements in module file - modtxt = open(toy_module_path, 'r').read() + # 'module use' statements to extend $MODULEPATH are present + modtxt = read_file(toy_module_path) modpath_extension = os.path.join(mod_prefix, 'MPI', 'GCC', '4.7.2', 'toy', '0.0') - self.assertTrue(re.search("^module\s*use\s*%s" % modpath_extension, modtxt, re.M)) + if get_module_syntax() == 'Tcl': + self.assertTrue(re.search("^module\s*use\s*%s" % modpath_extension, modtxt, re.M)) + elif get_module_syntax() == 'Lua': + fullmodpath_extension = os.path.join(self.test_installpath, modpath_extension) + regex = re.compile(r'^prepend_path\("MODULEPATH", "%s"\)' % fullmodpath_extension, re.M) + self.assertTrue(regex.search(modtxt), "Pattern '%s' found in %s" % (regex.pattern, modtxt)) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + os.remove(toy_module_path) + + # ... unless they shouldn't be + extra_args.append('--try-amend=include_modpath_extensions=') # pass empty string as equivalent to False + self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True) + modtxt = read_file(toy_module_path) + modpath_extension = os.path.join(mod_prefix, 'MPI', 'GCC', '4.7.2', 'toy', '0.0') + if get_module_syntax() == 'Tcl': + self.assertFalse(re.search("^module\s*use\s*%s" % modpath_extension, modtxt, re.M)) + elif get_module_syntax() == 'Lua': + fullmodpath_extension = os.path.join(self.test_installpath, modpath_extension) + regex = re.compile(r'^prepend_path\("MODULEPATH", "%s"\)' % fullmodpath_extension, re.M) + self.assertFalse(regex.search(modtxt), "Pattern '%s' found in %s" % (regex.pattern, modtxt)) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + os.remove(toy_module_path) # test module path with dummy/dummy build extra_args = [ @@ -615,11 +672,14 @@ def test_toy_hierarchical(self): # make sure module file is installed in correct path toy_module_path = os.path.join(mod_prefix, 'Core', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_module_path += '.lua' self.assertTrue(os.path.exists(toy_module_path)) # no dependencies or toolchain => no module load statements in module file - modtxt = open(toy_module_path, 'r').read() + modtxt = read_file(toy_module_path) self.assertFalse(re.search("module load", modtxt)) + os.remove(toy_module_path) # test module path with dummy/dummy build, pretend to be a compiler by setting moduleclass extra_args = [ @@ -630,19 +690,51 @@ def test_toy_hierarchical(self): # make sure module file is installed in correct path toy_module_path = os.path.join(mod_prefix, 'Core', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_module_path += '.lua' self.assertTrue(os.path.exists(toy_module_path)) # no dependencies or toolchain => no module load statements in module file - modtxt = open(toy_module_path, 'r').read() + modtxt = read_file(toy_module_path) modpath_extension = os.path.join(mod_prefix, 'Compiler', 'toy', '0.0') - self.assertTrue(re.search("^module\s*use\s*%s" % modpath_extension, modtxt, re.M)) + if get_module_syntax() == 'Tcl': + self.assertTrue(re.search("^module\s*use\s*%s" % modpath_extension, modtxt, re.M)) + elif get_module_syntax() == 'Lua': + fullmodpath_extension = os.path.join(self.test_installpath, modpath_extension) + regex = re.compile(r'^prepend_path\("MODULEPATH", "%s"\)' % fullmodpath_extension, re.M) + self.assertTrue(regex.search(modtxt), "Pattern '%s' found in %s" % (regex.pattern, modtxt)) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + os.remove(toy_module_path) + + # building a toolchain module should also work + args[0] = os.path.join(test_easyconfigs, 'gompi-1.4.10.eb') + modules_tool().purge() + self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=False) + gompi_module_path = os.path.join(mod_prefix, 'Core', 'gompi', '1.4.10') + if get_module_syntax() == 'Lua': + gompi_module_path += '.lua' + self.assertTrue(os.path.exists(gompi_module_path)) def test_toy_advanced(self): """Test toy build with extensions and non-dummy toolchain.""" test_dir = os.path.abspath(os.path.dirname(__file__)) os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules') - test_ec = os.path.join(test_dir, 'easyconfigs', 'toy-0.0-gompi-1.3.12.eb') - self.test_toy_build(ec_file=test_ec, versionsuffix='-gompi-1.3.12') + test_ec = os.path.join(test_dir, 'easyconfigs', 'toy-0.0-gompi-1.3.12-test.eb') + self.test_toy_build(ec_file=test_ec, versionsuffix='-gompi-1.3.12-test') + + def test_toy_hidden(self): + """Test installing a hidden module.""" + ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'toy-0.0.eb') + self.test_toy_build(ec_file=ec_file, extra_args=['--hidden'], verify=False) + # module file is hidden + toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '.0.0') + if get_module_syntax() == 'Lua': + toy_module += '.lua' + self.assertTrue(os.path.exists(toy_module), 'Found hidden module %s' % toy_module) + # installed software is not hidden + toybin = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy') + self.assertTrue(os.path.exists(toybin)) def test_module_filepath_tweaking(self): """Test using --suffix-modules-path.""" @@ -668,11 +760,275 @@ def test_module_filepath_tweaking(self): ] self.eb_main(args, do_build=True, verbose=True) mod_file_prefix = os.path.join(self.test_installpath, 'modules') - self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, 'foobarbaz', 'toy', '0.0'))) - self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, 'TOOLS', 'toy', '0.0'))) - self.assertTrue(os.path.islink(os.path.join(mod_file_prefix, 'TOOLS', 'toy', '0.0'))) - self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, 't', 'toy', '0.0'))) - self.assertTrue(os.path.islink(os.path.join(mod_file_prefix, 't', 'toy', '0.0'))) + mod_file_suffix = '' + if get_module_syntax() == 'Lua': + mod_file_suffix += '.lua' + + self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, 'foobarbaz', 'toy', '0.0' + mod_file_suffix))) + self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, 'TOOLS', 'toy', '0.0' + mod_file_suffix))) + self.assertTrue(os.path.islink(os.path.join(mod_file_prefix, 'TOOLS', 'toy', '0.0' + mod_file_suffix))) + self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, 't', 'toy', '0.0' + mod_file_suffix))) + self.assertTrue(os.path.islink(os.path.join(mod_file_prefix, 't', 'toy', '0.0' + mod_file_suffix))) + + def test_toy_archived_easyconfig(self): + """Test archived easyconfig for a succesful build.""" + repositorypath = os.path.join(self.test_installpath, 'easyconfigs_archive') + extra_args = [ + '--repository=FileRepository', + '--repositorypath=%s' % repositorypath, + ] + self.test_toy_build(raise_error=True, extra_args=extra_args) + + archived_ec = os.path.join(repositorypath, 'toy', 'toy-0.0.eb') + self.assertTrue(os.path.exists(archived_ec)) + ec = EasyConfig(archived_ec) + self.assertEqual(ec.name, 'toy') + self.assertEqual(ec.version, '0.0') + + def test_toy_module_fulltxt(self): + """Strict text comparison of generated module file.""" + self.test_toy_tweaked() + + toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-tweaked') + if get_module_syntax() == 'Lua': + toy_module += '.lua' + toy_mod_txt = read_file(toy_module) + + if get_module_syntax() == 'Lua': + mod_txt_regex_pattern = '\n'.join([ + r'help\(\[\[Toy C program. - Homepage: http://hpcugent.github.com/easybuild\]\]\)', + r'whatis\(\[\[Name: toy\]\]\)', + r'whatis\(\[\[Version: 0.0\]\]\)', + r'whatis\(\[\[Description: Toy C program. - Homepage: http://hpcugent.github.com/easybuild\]\]\)', + r'whatis\(\[\[Homepage: http://hpcugent.github.com/easybuild\]\]\)', + r'', + r'local root = "%s/software/toy/0.0-tweaked"' % self.test_installpath, + r'', + r'conflict\("toy"\)', + r'', + r'prepend_path\("LD_LIBRARY_PATH", pathJoin\(root, "lib"\)\)', + r'prepend_path\("LIBRARY_PATH", pathJoin\(root, "lib"\)\)', + r'prepend_path\("PATH", pathJoin\(root, "bin"\)\)', + r'setenv\("EBROOTTOY", root\)', + r'setenv\("EBVERSIONTOY", "0.0"\)', + r'setenv\("EBDEVELTOY", pathJoin\(root, "easybuild/toy-0.0-tweaked-easybuild-devel"\)\)', + r'', + r'setenv\("FOO", "bar"\)', + r'prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)', + r'prepend_path\("SOMEPATH", pathJoin\(root, "baz"\)\)', + r'prepend_path\("SOMEPATH", root\)', + r'', + r'if mode\(\) == "load" then', + r' io.stderr:write\("THANKS FOR LOADING ME, I AM toy v0.0"\)', + r'end', + r'io.stderr:write\("oh hai\!"\)', + r'-- Built with EasyBuild version .*$', + ]) + elif get_module_syntax() == 'Tcl': + mod_txt_regex_pattern = '\n'.join([ + r'^#%Module', + r'proc ModulesHelp { } {', + r' puts stderr { Toy C program. - Homepage: http://hpcugent.github.com/easybuild', + r' }', + r'}', + r'', + r'module-whatis {Description: Toy C program. - Homepage: http://hpcugent.github.com/easybuild}', + r'', + r'set root %s/software/toy/0.0-tweaked' % self.test_installpath, + r'', + r'conflict toy', + r'', + r'prepend-path LD_LIBRARY_PATH \$root/lib', + r'prepend-path LIBRARY_PATH \$root/lib', + r'prepend-path PATH \$root/bin', + r'setenv EBROOTTOY "\$root"', + r'setenv EBVERSIONTOY "0.0"', + r'setenv EBDEVELTOY "\$root/easybuild/toy-0.0-tweaked-easybuild-devel"', + r'', + r'setenv FOO "bar"', + r'prepend-path SOMEPATH \$root/foo/bar', + r'prepend-path SOMEPATH \$root/baz', + r'prepend-path SOMEPATH \$root', + r'', + r'if { \[ module-info mode load \] } {', + r' puts stderr "THANKS FOR LOADING ME, I AM toy v0.0"', + r'}', + r'puts stderr "oh hai\!"', + r'# Built with EasyBuild version .*$', + ]) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + + mod_txt_regex = re.compile(mod_txt_regex_pattern) + msg = "Pattern '%s' matches with: %s" % (mod_txt_regex.pattern, toy_mod_txt) + self.assertTrue(mod_txt_regex.match(toy_mod_txt), msg) + + def test_external_dependencies(self): + """Test specifying external (build) dependencies.""" + ectxt = read_file(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'toy-0.0-deps.eb')) + toy_ec = os.path.join(self.test_prefix, 'toy-0.0-external-deps.eb') + + # just specify some of the test modules we ship, doesn't matter where they come from + extraectxt = "\ndependencies += [('foobar/1.2.3', EXTERNAL_MODULE)]" + extraectxt += "\nbuilddependencies = [('somebuilddep/0.1', EXTERNAL_MODULE)]" + extraectxt += "\nversionsuffix = '-external-deps'" + write_file(toy_ec, ectxt + extraectxt) + + # install dummy modules + modulepath = os.path.join(self.test_prefix, 'modules') + for mod in ['ictce/4.1.13', 'GCC/4.7.2', 'foobar/1.2.3', 'somebuilddep/0.1']: + mkdir(os.path.join(modulepath, os.path.dirname(mod)), parents=True) + write_file(os.path.join(modulepath, mod), "#%Module") + + self.reset_modulepath([modulepath]) + self.test_toy_build(ec_file=toy_ec, versionsuffix='-external-deps', verbose=True) + + modules_tool().load(['toy/0.0-external-deps']) + # note build dependency is not loaded + mods = ['ictce/4.1.13', 'GCC/4.7.2', 'foobar/1.2.3', 'toy/0.0-external-deps'] + self.assertEqual([x['mod_name'] for x in modules_tool().list()], mods) + + # check behaviour when a non-existing external (build) dependency is included + err_msg = "Missing modules for one or more dependencies marked as external modules:" + + extraectxt = "\nbuilddependencies = [('nosuchbuilddep/0.0.0', EXTERNAL_MODULE)]" + extraectxt += "\nversionsuffix = '-external-deps-broken1'" + write_file(toy_ec, ectxt + extraectxt) + self.assertErrorRegex(EasyBuildError, err_msg, self.test_toy_build, ec_file=toy_ec, + raise_error=True, verbose=False) + + extraectxt = "\ndependencies += [('nosuchmodule/1.2.3', EXTERNAL_MODULE)]" + extraectxt += "\nversionsuffix = '-external-deps-broken2'" + write_file(toy_ec, ectxt + extraectxt) + self.assertErrorRegex(EasyBuildError, err_msg, self.test_toy_build, ec_file=toy_ec, + raise_error=True, verbose=False) + + # --dry-run still works when external modules are missing; external modules are treated as if they were there + outtxt = self.test_toy_build(ec_file=toy_ec, verbose=True, extra_args=['--dry-run'], verify=False) + self.assertTrue(re.search(r"^ \* \[ \] .* \(module: toy/0.0-external-deps-broken2\)", outtxt, re.M)) + + def test_module_only(self): + """Test use of --module-only.""" + ec_files_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + ec_file = os.path.join(ec_files_path, 'toy-0.0-deps.eb') + toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-deps') + + # only consider provided test modules + self.reset_modulepath([os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules')]) + + # sanity check fails without --force if software is not installed yet + common_args = [ + ec_file, + '--sourcepath=%s' % self.test_sourcepath, + '--buildpath=%s' % self.test_buildpath, + '--installpath=%s' % self.test_installpath, + '--debug', + '--unittest-file=%s' % self.logfile, + '--robot=%s' % ec_files_path, + '--module-syntax=Tcl', + ] + args = common_args + ['--module-only'] + err_msg = "Sanity check failed" + self.assertErrorRegex(EasyBuildError, err_msg, self.eb_main, args, do_build=True, raise_error=True) + self.assertFalse(os.path.exists(toy_mod)) + + self.eb_main(args + ['--force'], do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_mod)) + + # make sure load statements for dependencies are included in additional module file generated with --module-only + modtxt = read_file(toy_mod) + self.assertTrue(re.search('load.*ictce/4.1.13', modtxt), "load statement for ictce/4.1.13 found in module") + self.assertTrue(re.search('load.*GCC/4.7.2', modtxt), "load statement for GCC/4.7.2 found in module") + + os.remove(toy_mod) + + # installing another module under a different naming scheme and using Lua module syntax works fine + + # first actually build and install toy software + module + prefix = os.path.join(self.test_installpath, 'software', 'toy', '0.0-deps') + self.eb_main(common_args + ['--force'], do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_mod)) + self.assertTrue(os.path.exists(os.path.join(self.test_installpath, 'software', 'toy', '0.0-deps', 'bin'))) + modtxt = read_file(toy_mod) + self.assertTrue(re.search("set root %s" % prefix, modtxt)) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 1) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1) + + # install (only) additional module under a hierarchical MNS + args = common_args + [ + '--module-only', + '--module-naming-scheme=MigrateFromEBToHMNS', + ] + toy_core_mod = os.path.join(self.test_installpath, 'modules', 'all', 'Core', 'toy', '0.0-deps') + self.assertFalse(os.path.exists(toy_core_mod)) + self.eb_main(args, do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_core_mod)) + # existing install is reused + modtxt2 = read_file(toy_core_mod) + self.assertTrue(re.search("set root %s" % prefix, modtxt2)) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 2) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1) + + # make sure load statements for dependencies are included + modtxt = read_file(toy_core_mod) + self.assertTrue(re.search('load.*ictce/4.1.13', modtxt), "load statement for ictce/4.1.13 found in module") + + os.remove(toy_mod) + os.remove(toy_core_mod) + + # test installing (only) additional module in Lua syntax (if Lmod is available) + lmod_abspath = which('lmod') + if lmod_abspath is not None: + args = common_args[:-1] + [ + '--module-only', + '--module-syntax=Lua', + '--modules-tool=Lmod', + ] + self.assertFalse(os.path.exists(toy_mod + '.lua')) + self.eb_main(args, do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_mod + '.lua')) + # existing install is reused + modtxt3 = read_file(toy_mod + '.lua') + self.assertTrue(re.search('local root = "%s"' % prefix, modtxt3)) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 2) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1) + + # make sure load statements for dependencies are included + modtxt = read_file(toy_mod + '.lua') + self.assertTrue(re.search('load.*ictce/4.1.13', modtxt), "load statement for ictce/4.1.13 found in module") + + def test_package(self): + """Test use of --package and accompanying package configuration settings.""" + mock_fpm(self.test_prefix) + pkgpath = os.path.join(self.test_prefix, 'pkgs') + + extra_args = [ + '--experimental', + '--package', + '--package-release=321', + '--package-tool=fpm', + '--package-type=foo', + '--packagepath=%s' % pkgpath, + ] + + self.test_toy_build(extra_args=extra_args) + + toypkg = os.path.join(pkgpath, 'toy-0.0-eb-%s.321.foo' % EASYBUILD_VERSION) + self.assertTrue(os.path.exists(toypkg), "%s is there" % toypkg) + + def test_package_skip(self): + """Test use of --package with --skip.""" + mock_fpm(self.test_prefix) + pkgpath = os.path.join(self.test_prefix, 'packages') # default path + + self.test_toy_build(['--packagepath=%s' % pkgpath]) + self.assertFalse(os.path.exists(pkgpath), "%s is not created without use of --package" % pkgpath) + + self.test_toy_build(extra_args=['--experimental', '--package', '--skip'], verify=False) + + toypkg = os.path.join(pkgpath, 'toy-0.0-eb-%s.1.rpm' % EASYBUILD_VERSION) + self.assertTrue(os.path.exists(toypkg), "%s is there" % toypkg) + def suite(): """ return all the tests in this file """ diff --git a/test/framework/tweak.py b/test/framework/tweak.py new file mode 100644 index 0000000000..1cb3611c32 --- /dev/null +++ b/test/framework/tweak.py @@ -0,0 +1,117 @@ +## +# Copyright 2014 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Unit tests for framework/easyconfig/tweak.py + +@author: Kenneth Hoste (Ghent University) +""" +import os +from test.framework.utilities import EnhancedTestCase +from unittest import TestLoader, main + +from easybuild.framework.easyconfig.tweak import find_matching_easyconfigs, obtain_ec_for, pick_version + + +class TweakTest(EnhancedTestCase): + """Tests for tweak functionality.""" + def test_pick_version(self): + """Test pick_version function.""" + # if required version is not available, the most recent version less than or equal should be returned + self.assertEqual(('1.4', '1.0'), pick_version('1.4', ['0.5', '1.0', '1.5'])) + + # if required version is available, that should be what's returned + self.assertEqual(('1.0', '1.0'), pick_version('1.0', ['0.5', '1.0', '1.5'])) + + def test_find_matching_easyconfigs(self): + """Test find_matching_easyconfigs function.""" + test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + for (name, installver) in [('GCC', '4.8.2'), ('gzip', '1.5-goolf-1.4.10')]: + ecs = find_matching_easyconfigs(name, installver, [test_easyconfigs_path]) + self.assertTrue(len(ecs) == 1 and ecs[0].endswith('/%s-%s.eb' % (name, installver))) + + ecs = find_matching_easyconfigs('GCC', '*', [test_easyconfigs_path]) + gccvers = ['4.6.3', '4.6.4', '4.7.2', '4.8.2', '4.8.3', '4.9.2'] + self.assertEqual(len(ecs), len(gccvers)) + ecs_basename = [os.path.basename(ec) for ec in ecs] + for gccver in gccvers: + gcc_ec = 'GCC-%s.eb' % gccver + self.assertTrue(gcc_ec in ecs_basename, "%s is included in %s" % (gcc_ec, ecs_basename)) + + def test_obtain_ec_for(self): + """Test obtain_ec_for function.""" + test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + # find existing easyconfigs + specs = { + 'name': 'GCC', + 'version': '4.6.4', + } + (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path]) + self.assertFalse(generated) + self.assertEqual(os.path.basename(ec_file), 'GCC-4.6.4.eb') + + specs = { + 'name': 'ScaLAPACK', + 'version': '2.0.2', + 'toolchain_name': 'gompi', + 'toolchain_version': '1.4.10', + 'versionsuffix': '-OpenBLAS-0.2.6-LAPACK-3.4.2', + } + (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path]) + self.assertFalse(generated) + self.assertEqual(os.path.basename(ec_file), 'ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb') + + specs = { + 'name': 'ifort', + 'versionsuffix': '-GCC-4.8.3', + } + (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path]) + self.assertFalse(generated) + self.assertEqual(os.path.basename(ec_file), 'ifort-2013.5.192-GCC-4.8.3.eb') + + # latest version if not specified + specs = { + 'name': 'GCC', + } + (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path]) + self.assertFalse(generated) + self.assertEqual(os.path.basename(ec_file), 'GCC-4.9.2.eb') + + # generate non-existing easyconfig + os.chdir(self.test_prefix) + specs = { + 'name': 'GCC', + 'version': '5.4.3', + } + (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path]) + self.assertTrue(generated) + self.assertEqual(os.path.basename(ec_file), 'GCC-5.4.3.eb') + + +def suite(): + """ return all the tests in this file """ + return TestLoader().loadTestsFromTestCase(TweakTest) + +if __name__ == '__main__': + main() diff --git a/test/framework/type_checking.py b/test/framework/type_checking.py new file mode 100644 index 0000000000..d5283b1269 --- /dev/null +++ b/test/framework/type_checking.py @@ -0,0 +1,95 @@ +# # +# Copyright 2015-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Unit tests for easyconfig/types.py + +@author: Kenneth Hoste (Ghent University) +""" +from test.framework.utilities import EnhancedTestCase +from unittest import TestLoader, main + +from easybuild.tools.build_log import EasyBuildError +from easybuild.framework.easyconfig.types import check_type_of_param_value, convert_value_type + + +class TypeCheckingTest(EnhancedTestCase): + """Tests for value type checking of easyconfig parameters.""" + + def test_check_type_of_param_value(self): + """Test check_type_of_param_value function.""" + # check selected values that should be strings + for key in ['name', 'version']: + self.assertEqual(check_type_of_param_value(key, 'foo'), (True, 'foo')) + for not_a_string in [100, 1.5, ('bar',), ['baz'], None]: + self.assertEqual(check_type_of_param_value(key, not_a_string), (False, None)) + # value doesn't matter, only type does + self.assertEqual(check_type_of_param_value(key, ''), (True, '')) + + # parameters with no type specification always pass the check + key = 'nosucheasyconfigparametereverhopefully' + for val in ['foo', 100, 1.5, ('bar',), ['baz'], '', None]: + self.assertEqual(check_type_of_param_value(key, val), (True, val)) + + # check use of auto_convert + self.assertEqual(check_type_of_param_value('version', 1.5), (False, None)) + self.assertEqual(check_type_of_param_value('version', 1.5, auto_convert=True), (True, '1.5')) + + def test_convert_value_type(self): + """Test convert_value_type function.""" + # to string + self.assertEqual(convert_value_type(100, basestring), '100') + self.assertEqual(convert_value_type((100,), str), '(100,)') + self.assertEqual(convert_value_type([100], basestring), '[100]') + self.assertEqual(convert_value_type(None, str), 'None') + + # to int/float + self.assertEqual(convert_value_type('100', int), 100) + self.assertEqual(convert_value_type('0', int), 0) + self.assertEqual(convert_value_type('-123', int), -123) + self.assertEqual(convert_value_type('1.6', float), 1.6) + self.assertEqual(convert_value_type('5', float), 5.0) + self.assertErrorRegex(EasyBuildError, "Converting type of .* failed", convert_value_type, '', int) + # 1.6 can't be parsed as an int (yields "invalid literal for int() with base 10" error) + self.assertErrorRegex(EasyBuildError, "Converting type of .* failed", convert_value_type, '1.6', int) + + # idempotency + self.assertEqual(convert_value_type('foo', basestring), 'foo') + self.assertEqual(convert_value_type('foo', str), 'foo') + self.assertEqual(convert_value_type(100, int), 100) + self.assertEqual(convert_value_type(1.6, float), 1.6) + + # no conversion function available for specific type + class Foo(): + pass + self.assertErrorRegex(EasyBuildError, "No conversion function available", convert_value_type, None, Foo) + + +def suite(): + """ returns all the testcases in this module """ + return TestLoader().loadTestsFromTestCase(TypeCheckingTest) + + +if __name__ == '__main__': + main() diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 5674f4e4fc..1210d08d0b 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,15 +28,17 @@ @author: Kenneth Hoste (Ghent University) """ import copy +import fileinput import os import re import shutil import sys import tempfile -from unittest import TestCase from vsc.utils import fancylogger from vsc.utils.patterns import Singleton +from vsc.utils.testing import EnhancedTestCase as _EnhancedTestCase +import easybuild.tools.build_log as eb_build_log import easybuild.tools.options as eboptions import easybuild.tools.toolchain.utilities as tc_utils import easybuild.tools.module_naming_scheme.toolchain as mns_toolchain @@ -44,41 +46,54 @@ from easybuild.framework.easyblock import EasyBlock from easybuild.main import main from easybuild.tools import config -from easybuild.tools.config import module_classes +from easybuild.tools.config import module_classes, set_tmpdir +from easybuild.tools.configobj import ConfigObj from easybuild.tools.environment import modify_env -from easybuild.tools.filetools import read_file +from easybuild.tools.filetools import mkdir, read_file from easybuild.tools.module_naming_scheme import GENERAL_CLASS from easybuild.tools.modules import modules_tool +from easybuild.tools.options import CONFIG_ENV_VAR_PREFIX, EasyBuildOptions -class EnhancedTestCase(TestCase): - """Enhanced test case, provides extra functionality (e.g. an assertErrorRegex method).""" +# make sure tests are robust against any non-default configuration settings; +# involves ignoring any existing configuration files that are picked up, and cleaning the environment +# this is tackled here rather than in suite.py, to make sure this is also done when test modules are ran separately - def assertErrorRegex(self, error, regex, call, *args, **kwargs): - """Convenience method to match regex with the expected error message""" - try: - call(*args, **kwargs) - str_kwargs = ', '.join(['='.join([k,str(v)]) for (k,v) in kwargs.items()]) - str_args = ', '.join(map(str, args) + [str_kwargs]) - self.assertTrue(False, "Expected errors with %s(%s) call should occur" % (call.__name__, str_args)) - except error, err: - if hasattr(err, 'msg'): - msg = err.msg - elif hasattr(err, 'message'): - msg = err.message - elif hasattr(err, 'args'): # KeyError in Python 2.4 only provides message via 'args' attribute - msg = err.args[0] - else: - msg = err - try: - msg = str(msg) - except UnicodeEncodeError: - msg = msg.encode('utf8', 'replace') - self.assertTrue(re.search(regex, msg), "Pattern '%s' is found in '%s'" % (regex, msg)) - self.assertTrue(re.search(regex, msg), "Pattern '%s' is found in '%s'" % (regex, msg)) +# clean up environment from unwanted $EASYBUILD_X env vars +for key in os.environ.keys(): + if key.startswith('%s_' % CONFIG_ENV_VAR_PREFIX): + del os.environ[key] + +# ignore any existing configuration files +go = EasyBuildOptions(go_useconfigfiles=False) +os.environ['EASYBUILD_IGNORECONFIGFILES'] = ','.join(go.options.configfiles) + +# redefine $TEST_EASYBUILD_X env vars as $EASYBUILD_X +test_env_var_prefix = 'TEST_EASYBUILD_' +for key in os.environ.keys(): + if key.startswith(test_env_var_prefix): + val = os.environ[key] + del os.environ[key] + newkey = '%s_%s' % (CONFIG_ENV_VAR_PREFIX, key[len(test_env_var_prefix):]) + os.environ[newkey] = val + +class EnhancedTestCase(_EnhancedTestCase): + """Enhanced test case, provides extra functionality (e.g. an assertErrorRegex method).""" def setUp(self): """Set up testcase.""" + super(EnhancedTestCase, self).setUp() + + # keep track of log handlers + log = fancylogger.getLogger(fname=False) + self.orig_log_handlers = log.handlers[:] + + log.info("setting up test %s" % self.id()) + + self.orig_tmpdir = tempfile.gettempdir() + # use a subdirectory for this test (which we can clean up easily after the test completes) + self.test_prefix = set_tmpdir() + self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) fd, self.logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') os.close(fd) @@ -90,20 +105,31 @@ def setUp(self): # keep track of original environment/Python search path to restore self.orig_sys_path = sys.path[:] - self.orig_paths = {} - for path in ['buildpath', 'installpath', 'sourcepath']: - self.orig_paths[path] = os.environ.get('EASYBUILD_%s' % path.upper(), None) - testdir = os.path.dirname(os.path.abspath(__file__)) self.test_sourcepath = os.path.join(testdir, 'sandbox', 'sources') os.environ['EASYBUILD_SOURCEPATH'] = self.test_sourcepath + os.environ['EASYBUILD_PREFIX'] = self.test_prefix self.test_buildpath = tempfile.mkdtemp() os.environ['EASYBUILD_BUILDPATH'] = self.test_buildpath self.test_installpath = tempfile.mkdtemp() os.environ['EASYBUILD_INSTALLPATH'] = self.test_installpath + + # make sure that the tests only pick up easyconfigs provided with the tests + os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join(testdir, 'easyconfigs') + + # make sure no deprecated behaviour is being triggered (unless intended by the test) + # trip *all* log.deprecated statements by setting deprecation version ridiculously high + self.orig_current_version = eb_build_log.CURRENT_VERSION + os.environ['EASYBUILD_DEPRECATED'] = '10000000' + init_config() + # remove any entries in Python search path that seem to provide easyblocks + for path in sys.path[:]: + if os.path.exists(os.path.join(path, 'easybuild', 'easyblocks', '__init__.py')): + sys.path.remove(path) + # add test easyblocks to Python search path and (re)import and reload easybuild modules import easybuild sys.path.append(os.path.join(testdir, 'sandbox')) @@ -114,63 +140,171 @@ def setUp(self): reload(easybuild.easyblocks.generic) reload(easybuild.tools.module_naming_scheme) # required to run options unit tests stand-alone - # set MODULEPATH to included test modules - os.environ['MODULEPATH'] = os.path.join(testdir, 'modules') - + modtool = modules_tool() # purge out any loaded modules with original $MODULEPATH before running each test - modules_tool().purge() + modtool.purge() + self.reset_modulepath([os.path.join(testdir, 'modules')]) def tearDown(self): """Clean up after running testcase.""" + super(EnhancedTestCase, self).tearDown() + + self.log.info("Cleaning up for test %s", self.id()) + + # go back to where we were before os.chdir(self.cwd) + + # restore original environment modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None # restore original Python search path sys.path = self.orig_sys_path - for path in [self.test_buildpath, self.test_installpath]: - try: - shutil.rmtree(path) - except OSError, err: - pass - - for path in ['buildpath', 'installpath', 'sourcepath']: - if self.orig_paths[path] is not None: - os.environ['EASYBUILD_%s' % path.upper()] = self.orig_paths[path] - else: - if 'EASYBUILD_%s' % path.upper() in os.environ: - del os.environ['EASYBUILD_%s' % path.upper()] - init_config() + # remove any log handlers that were added (so that log files can be effectively removed) + log = fancylogger.getLogger(fname=False) + new_log_handlers = [h for h in log.handlers if h not in self.orig_log_handlers] + for log_handler in new_log_handlers: + log_handler.close() + log.removeHandler(log_handler) - def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbose=False, raise_error=False): + # cleanup test tmp dir + try: + shutil.rmtree(self.test_prefix) + except (OSError, IOError): + pass + + # restore original 'parent' tmpdir + for var in ['TMPDIR', 'TEMP', 'TMP']: + os.environ[var] = self.orig_tmpdir + + # reset to make sure tempfile picks up new temporary directory to use + tempfile.tempdir = None + + def reset_modulepath(self, modpaths): + """Reset $MODULEPATH with specified paths.""" + modtool = modules_tool() + for modpath in os.environ.get('MODULEPATH', '').split(os.pathsep): + modtool.remove_module_path(modpath) + # make very sure $MODULEPATH is totally empty + # some paths may be left behind, e.g. when they contain environment variables + # example: "module unuse Modules/$MODULE_VERSION/modulefiles" may not yield the desired result + os.environ['MODULEPATH'] = '' + for modpath in modpaths: + modtool.add_module_path(modpath) + + def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbose=False, raise_error=False, + reset_env=True, raise_systemexit=False, testing=True): """Helper method to call EasyBuild main function.""" cleanup() myerr = False if logfile is None: logfile = self.logfile + # clear log file + if logfile: + f = open(logfile, 'w') + f.write('') + f.close() + + env_before = copy.deepcopy(os.environ) + try: - main((args, logfile, do_build)) + main(args=args, logfile=logfile, do_build=do_build, testing=testing) except SystemExit: - pass + if raise_systemexit: + raise err except Exception, err: myerr = err if verbose: print "err: %s" % err + if logfile and os.path.exists(logfile): + logtxt = read_file(logfile) + else: + logtxt = None + os.chdir(self.cwd) # make sure config is reinitialized init_config() + # restore environment to what it was before running main, + # changes may have been made by eb_main (e.g. $TMPDIR & co) + if reset_env: + modify_env(os.environ, env_before) + tempfile.tempdir = None + if myerr and raise_error: raise myerr if return_error: - return read_file(self.logfile), myerr + return logtxt, myerr else: - return read_file(self.logfile) + return logtxt + + def setup_hierarchical_modules(self): + """Setup hierarchical modules to run tests on.""" + mod_prefix = os.path.join(self.test_installpath, 'modules', 'all') + + # simply copy module files under 'Core' and 'Compiler' to test install path + # EasyBuild is responsible for making sure that the toolchain can be loaded using the short module name + mkdir(mod_prefix, parents=True) + for mod_subdir in ['Core', 'Compiler', 'MPI']: + src_mod_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules', mod_subdir) + shutil.copytree(src_mod_path, os.path.join(mod_prefix, mod_subdir)) + + # make sure only modules in a hierarchical scheme are available, mixing modules installed with + # a flat scheme like EasyBuildMNS and a hierarhical one like HierarchicalMNS doesn't work + self.reset_modulepath([mod_prefix, os.path.join(mod_prefix, 'Core')]) + + # tweak use statements in modules to ensure correct paths + mpi_pref = os.path.join(mod_prefix, 'MPI', 'GCC', '4.7.2', 'OpenMPI', '1.6.4') + for modfile in [ + os.path.join(mod_prefix, 'Core', 'GCC', '4.7.2'), + os.path.join(mod_prefix, 'Core', 'GCC', '4.8.3'), + os.path.join(mod_prefix, 'Core', 'icc', '2013.5.192-GCC-4.8.3'), + os.path.join(mod_prefix, 'Core', 'ifort', '2013.5.192-GCC-4.8.3'), + os.path.join(mod_prefix, 'Compiler', 'GCC', '4.7.2', 'OpenMPI', '1.6.4'), + os.path.join(mod_prefix, 'Compiler', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049'), + os.path.join(mpi_pref, 'FFTW', '3.3.3'), + os.path.join(mpi_pref, 'OpenBLAS', '0.2.6-LAPACK-3.4.2'), + os.path.join(mpi_pref, 'ScaLAPACK', '2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2'), + ]: + for line in fileinput.input(modfile, inplace=1): + line = re.sub(r"(module\s*use\s*)/tmp/modules/all", + r"\1%s/modules/all" % self.test_installpath, + line) + sys.stdout.write(line) + + def setup_categorized_hmns_modules(self): + """Setup categorized hierarchical modules to run tests on.""" + mod_prefix = os.path.join(self.test_installpath, 'modules', 'all') + + # simply copy module files under 'CategorizedHMNS/{Core,Compiler,MPI}' to test install path + # EasyBuild is responsible for making sure that the toolchain can be loaded using the short module name + mkdir(mod_prefix, parents=True) + for mod_subdir in ['Core', 'Compiler', 'MPI']: + src_mod_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'modules', 'CategorizedHMNS', mod_subdir) + shutil.copytree(src_mod_path, os.path.join(mod_prefix, mod_subdir)) + # create empty module file directory to make C/Tcl modules happy + mpi_pref = os.path.join(mod_prefix, 'MPI', 'GCC', '4.7.2', 'OpenMPI', '1.6.4') + mkdir(os.path.join(mpi_pref, 'base')) + + # make sure only modules in the CategorizedHMNS are available + self.reset_modulepath([os.path.join(mod_prefix, 'Core', 'compiler'), + os.path.join(mod_prefix, 'Core', 'toolchain')]) + + # tweak use statements in modules to ensure correct paths + for modfile in [ + os.path.join(mod_prefix, 'Core', 'compiler', 'GCC', '4.7.2'), + os.path.join(mod_prefix, 'Compiler', 'GCC', '4.7.2', 'mpi', 'OpenMPI', '1.6.4'), + ]: + for line in fileinput.input(modfile, inplace=1): + line = re.sub(r"(module\s*use\s*)/tmp/modules/all", + r"\1%s/modules/all" % self.test_installpath, + line) + sys.stdout.write(line) def cleanup(): @@ -181,8 +315,11 @@ def cleanup(): # empty caches tc_utils._initial_toolchain_instances.clear() easyconfig._easyconfigs_cache.clear() + easyconfig._easyconfig_files_cache.clear() mns_toolchain._toolchain_details_cache.clear() + # reset to make sure tempfile picks up new temporary directory to use + tempfile.tempdir = None def init_config(args=None, build_options=None): """(re)initialize configuration""" @@ -196,15 +333,17 @@ def init_config(args=None, build_options=None): # initialize build options if build_options is None: build_options = { + 'external_modules_metadata': ConfigObj(), 'valid_module_classes': module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } if 'suffix_modules_path' not in build_options: build_options.update({'suffix_modules_path': GENERAL_CLASS}) - config.init_build_options(build_options) + config.init_build_options(build_options=build_options) return eb_go.options + def find_full_path(base_path, trim=(lambda x: x)): """ Determine full path for given base path by looking in sys.path and PYTHONPATH. diff --git a/test/framework/variables.py b/test/framework/variables.py index 8d337173ad..c377b507b6 100644 --- a/test/framework/variables.py +++ b/test/framework/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -78,6 +78,16 @@ class TestVariables(Variables): cmd = CommandFlagList(["gcc", "bar", "baz"]) self.assertEqual(str(cmd), "gcc -bar -baz") + def test_empty_variables(self): + """Test playing around with empty variables.""" + v = Variables() + v.nappend('FOO', []) + self.assertEqual(v['FOO'], []) + v.join('BAR', 'FOO') + self.assertEqual(v['BAR'], []) + v.join('FOOBAR', 'BAR') + self.assertEqual(v['FOOBAR'], []) + def suite(): """ return all the tests""" return TestLoader().loadTestsFromTestCase(VariablesTest) diff --git a/vsc/README.md b/vsc/README.md deleted file mode 100644 index 77be7da954..0000000000 --- a/vsc/README.md +++ /dev/null @@ -1,3 +0,0 @@ -Code from https://github.com/hpcugent/vsc-base - -based on a15bb01eb06b385144325a8d9be20184b1044259 (vsc-base v1.9.2) diff --git a/vsc/__init__.py b/vsc/__init__.py deleted file mode 100644 index d149b4df8e..0000000000 --- a/vsc/__init__.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python -## -# Copyright 2011-2013 Ghent University -# -# This file is part of vsc-base, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# the Hercules foundation (http://www.herculesstichting.be/in_English) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# http://github.com/hpcugent/vsc-base -# -# vsc-base is free software: you can redistribute it and/or modify -# it under the terms of the GNU Library General Public License as -# published by the Free Software Foundation, either version 2 of -# the License, or (at your option) any later version. -# -# vsc-base is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU Library General Public License -# along with vsc-base. If not, see . -## -""" -Initialize vsc package. -The vsc namespace is used in different folders allong the system -so explicitly declare this is also the vsc namespace - -@author: Jens Timmerman (Ghent University) -""" -#from pkgutil import extend_path - -# we're not the only ones in this namespace -# avoid that EasyBuild uses vsc package from somewhere else, e.g. a system vsc-base installation -#__path__ = extend_path(__path__, __name__) #@ReservedAssignment - -# here for backwards compatibility -from vsc.utils import fancylogger diff --git a/vsc/install/shared_setup.py b/vsc/install/shared_setup.py deleted file mode 100644 index 761d80ce26..0000000000 --- a/vsc/install/shared_setup.py +++ /dev/null @@ -1,273 +0,0 @@ -#!/usr/bin/env python -# -*- coding: latin-1 -*- -# # -# Copyright 2009-2013 Ghent University -# -# This file is part of vsc-utils, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# the Hercules foundation (http://www.herculesstichting.be/in_English) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# All rights reserved. -# -# # -""" -Shared module for vsc-base setup - -@author: Stijn De Weirdt (Ghent University) -@author: Andy Georges (Ghent University) -""" -import glob -import os -import shutil -import sys -from distutils import log # also for setuptools -from distutils.dir_util import remove_tree - -# 0 : WARN (default), 1 : INFO, 2 : DEBUG -log.set_verbosity(2) - -has_setuptools = None - - -# We do need all setup files to be included in the source dir if we ever want to install -# the package elsewhere. -EXTRA_SDIST_FILES = ['setup.py'] - - -def find_extra_sdist_files(): - """Looks for files to append to the FileList that is used by the egg_info.""" - print "looking for extra dist files" - filelist = [] - for fn in EXTRA_SDIST_FILES: - if os.path.isfile(fn): - filelist.append(fn) - else: - print "sdist add_defaults Failed to find %s" % fn - print "exiting." - sys.exit(1) - return filelist - - -def remove_extra_bdist_rpm_files(): - """Provides a list of files that should be removed from the source file list when making an RPM. - - This function should be overridden if necessary in the setup.py - - @returns: empty list - """ - return [] - -# The following aims to import from setuptools, but if this is not available, we import the basic functionality from -# distutils instead. Note that setuptools make copies of the scripts, it does _not_ preserve symbolic links. -try: - # raise("no setuptools") # to try distutils, uncomment - from setuptools import setup - from setuptools.command.bdist_rpm import bdist_rpm, _bdist_rpm - from setuptools.command.build_py import build_py - from setuptools.command.install_scripts import install_scripts - from setuptools.command.sdist import sdist - - # egg_info uses sdist directly through manifest_maker - from setuptools.command.egg_info import egg_info - - class vsc_egg_info(egg_info): - """Class to determine the set of files that should be included. - - This amounts to including the default files, as determined by setuptools, extended with the - few extra files we need to add for installation purposes. - """ - - def find_sources(self): - """Default lookup.""" - egg_info.find_sources(self) - self.filelist.extend(find_extra_sdist_files()) - - # TODO: this should be in the setup.py, here we should have a placeholder, so we need not change this for every - # package we deploy - class vsc_bdist_rpm_egg_info(vsc_egg_info): - """Class to determine the source files that should be present in an (S)RPM. - - All __init__.py files that augment namespace packages should be installed by the - dependent package, so we need not install it here. - """ - - def find_sources(self): - """Fins the sources as default and then drop the cruft.""" - vsc_egg_info.find_sources(self) - for f in remove_extra_bdist_rpm_files(): - print "DEBUG: removing %s from source list" % (f) - self.filelist.files.remove(f) - - has_setuptools = True -except: - from distutils.core import setup - from distutils.command.install_scripts import install_scripts - from distutils.command.build_py import build_py - from distutils.command.sdist import sdist - from distutils.command.bdist_rpm import bdist_rpm, _bdist_rpm - - class vsc_egg_info(object): - pass # dummy class for distutils - - class vsc_bdist_rpm_egg_info(vsc_egg_info): - pass # dummy class for distutils - - has_setuptools = False - - -# available authors -ag = ('Andy Georges', 'andy.georges@ugent.be') -jt = ('Jens Timmermans', 'jens.timmermans@ugent.be') -kh = ('Kenneth Hoste', 'kenneth.hoste@ugent.be') -lm = ('Luis Fernando Munoz Meji?as', 'luis.munoz@ugent.be') -sdw = ('Stijn De Weirdt', 'stijn.deweirdt@ugent.be') -wdp = ('Wouter Depypere', 'wouter.depypere@ugent.be') -kw = ('Kenneth Waegeman', 'Kenneth.Waegeman@UGent.be') - -# FIXME: do we need this here? it won;t hurt, but still ... -class vsc_install_scripts(install_scripts): - """Create the (fake) links for mympirun also remove .sh and .py extensions from the scripts.""" - - def __init__(self, *args): - install_scripts.__init__(self, *args) - self.original_outfiles = None - - def run(self): - # old-style class - install_scripts.run(self) - - self.original_outfiles = self.get_outputs()[:] # make a copy - self.outfiles = [] # reset it - for script in self.original_outfiles: - # remove suffixes for .py and .sh - if script.endswith(".py") or script.endswith(".sh"): - shutil.move(script, script[:-3]) - script = script[:-3] - self.outfiles.append(script) - - -class vsc_build_py(build_py): - def find_package_modules (self, package, package_dir): - """Extend build_by (not used for now)""" - result = build_py.find_package_modules(self, package, package_dir) - return result - - -class vsc_bdist_rpm(bdist_rpm): - """ Custom class to build the RPM, since the __inti__.py cannot be included for the packages that have namespace spread across all of the machine.""" - def run(self): - log.error("vsc_bdist_rpm = %s" % (self.__dict__)) - SHARED_TARGET['cmdclass']['egg_info'] = vsc_bdist_rpm_egg_info # changed to allow removal of files - self.run_command('egg_info') # ensure distro name is up-to-date - _bdist_rpm.run(self) - - -# shared target config -SHARED_TARGET = { - 'url': '', - 'download_url': '', - 'package_dir': {'': 'lib'}, - 'cmdclass': { - "install_scripts": vsc_install_scripts, - "egg_info": vsc_egg_info, - "bdist_rpm": vsc_bdist_rpm, - }, -} - - -def cleanup(prefix=''): - """Remove all build cruft.""" - dirs = [prefix + 'build'] + glob.glob(prefix + 'lib/*.egg-info') - for d in dirs: - if os.path.isdir(d): - log.warn("cleanup %s" % d) - try: - remove_tree(d, verbose=False) - except OSError, _: - log.error("cleanup failed for %s" % d) - - for fn in ('setup.cfg',): - ffn = prefix + fn - if os.path.isfile(ffn): - os.remove(ffn) - -def sanitize(v): - """Transforms v into a sensible string for use in setup.cfg.""" - if isinstance(v, str): - return v - - if isinstance(v, list): - return ",".join(v) - - -def parse_target(target): - """Add some fields""" - new_target = {} - new_target.update(SHARED_TARGET) - for k, v in target.items(): - if k in ('author', 'maintainer'): - if not isinstance(v, list): - log.error("%s of config %s needs to be a list (not tuple or string)" % (k, target['name'])) - sys.exit(1) - new_target[k] = ";".join([x[0] for x in v]) - new_target["%s_email" % k] = ";".join([x[1] for x in v]) - else: - if isinstance(v, dict): - # eg command_class - if not k in new_target: - new_target[k] = type(v)() - new_target[k].update(v) - else: - new_target[k] = type(v)() - new_target[k] += v - - log.debug("New target = %s" % (new_target)) - return new_target - - -def build_setup_cfg_for_bdist_rpm(target): - """Generates a setup.cfg on a per-target basis. - - Stores the 'install-requires' in the [bdist_rpm] section - - @type target: dict - - @param target: specifies the options to be passed to setup() - """ - - try: - setup_cfg = open('setup.cfg', 'w') # and truncate - except (IOError, OSError), err: - print "Cannot create setup.cfg for target %s: %s" % (target['name'], err) - sys.exit(1) - - s = ["[bdist_rpm]"] - if 'install_requires' in target: - s += ["requires = %s" % (sanitize(target['install_requires']))] - - if 'provides' in target: - s += ["provides = %s" % (sanitize((target['provides'])))] - target.pop('provides') - - setup_cfg.write("\n".join(s) + "\n") - setup_cfg.close() - - -def action_target(target, setupfn=setup, extra_sdist=[]): - # EXTRA_SDIST_FILES.extend(extra_sdist) - - cleanup() - - build_setup_cfg_for_bdist_rpm(target) - x = parse_target(target) - - setupfn(**x) - -if __name__ == '__main__': - # print all supported packages - all_setups = [x[len('setup_'):-len('.py')] for x in glob.glob('setup_*.py')] - all_packages = ['-'.join(['vsc'] + x.split('_')) for x in all_setups] - print " ".join(all_packages) diff --git a/vsc/utils/__init__.py b/vsc/utils/__init__.py deleted file mode 100644 index 4b023b72ef..0000000000 --- a/vsc/utils/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -## -# Copyright 2011-2013 Ghent University -# -# This file is part of vsc-base, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# the Hercules foundation (http://www.herculesstichting.be/in_English) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# http://github.com/hpcugent/vsc-base -# -# vsc-base is free software: you can redistribute it and/or modify -# it under the terms of the GNU Library General Public License as -# published by the Free Software Foundation, either version 2 of -# the License, or (at your option) any later version. -# -# vsc-base is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU Library General Public License -# along with vsc-base. If not, see . -## -""" -This package contains some utilitie modules use alltrought the vsc packages. - -@author: Jens Timmerman (Ghent University) -""" diff --git a/vsc/utils/affinity.py b/vsc/utils/affinity.py deleted file mode 100644 index a774bb7fc3..0000000000 --- a/vsc/utils/affinity.py +++ /dev/null @@ -1,334 +0,0 @@ -## -# Copyright 2012-2013 Ghent University -# -# This file is part of vsc-base, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# the Hercules foundation (http://www.herculesstichting.be/in_English) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# http://github.com/hpcugent/vsc-base -# -# vsc-base is free software: you can redistribute it and/or modify -# it under the terms of the GNU Library General Public License as -# published by the Free Software Foundation, either version 2 of -# the License, or (at your option) any later version. -# -# vsc-base is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU Library General Public License -# along with vsc-base. If not, see . -## -""" -Linux cpu affinity. - - Based on C{sched.h} and C{bits/sched.h}, - - see man pages for C{sched_getaffinity} and C{sched_setaffinity} - - also provides a C{cpuset} class to convert between human readable cpusets and the bit version -Linux priority - - Based on sys/resources.h and bits/resources.h see man pages for - C{getpriority} and C{setpriority} - -@author: Stijn De Weirdt (Ghent University) -""" - -import ctypes -import os -from ctypes.util import find_library -from vsc.utils.fancylogger import getLogger, setLogLevelDebug - -_logger = getLogger("affinity") - -_libc_lib = find_library('c') -_libc = ctypes.cdll.LoadLibrary(_libc_lib) - -#/* Type for array elements in 'cpu_set_t'. */ -#typedef unsigned long int __cpu_mask; -cpu_mask_t = ctypes.c_ulong - -##define __CPU_SETSIZE 1024 -##define __NCPUBITS (8 * sizeof(__cpu_mask)) -CPU_SETSIZE = 1024 -NCPUBITS = 8 * ctypes.sizeof(cpu_mask_t) -NMASKBITS = CPU_SETSIZE / NCPUBITS - -#/* Priority limits. */ -##define PRIO_MIN -20 /* Minimum priority a process can have. */ -##define PRIO_MAX 20 /* Maximum priority a process can have. */ -PRIO_MIN = -20 -PRIO_MAX = 20 - -#/* The type of the WHICH argument to `getpriority' and `setpriority', -# indicating what flavor of entity the WHO argument specifies. * / -#enum __priority_which -##{ -# PRIO_PROCESS = 0, /* WHO is a process ID. * / -##define PRIO_PROCESS PRIO_PROCESS -# PRIO_PGRP = 1, /* WHO is a process group ID. * / -##define PRIO_PGRP PRIO_PGRP -# PRIO_USER = 2 /* WHO is a user ID. * / -##define PRIO_USER PRIO_USER -##}; -PRIO_PROCESS = 0 -PRIO_PGRP = 1 -PRIO_USER = 2 - -#/* using pid_t for __pid_t */ -#typedef unsigned pid_t; -pid_t = ctypes.c_uint - -##if defined __USE_GNU && !defined __cplusplus -#typedef enum __rlimit_resource __rlimit_resource_t; -#typedef enum __rusage_who __rusage_who_t; -#typedef enum __priority_which __priority_which_t; -##else -#typedef int __rlimit_resource_t; -#typedef int __rusage_who_t; -#typedef int __priority_which_t; -##endif -priority_which_t = ctypes.c_int - -## typedef __u_int __id_t; -id_t = ctypes.c_uint - - -#/* Data structure to describe CPU mask. */ -#typedef struct -#{ -# __cpu_mask __bits[__NMASKBITS]; -#} cpu_set_t; -class cpu_set_t(ctypes.Structure): - """Class that implements the cpu_set_t struct - also provides some methods to convert between bit representation and soem human readable format - """ - _fields_ = [('__bits', cpu_mask_t * NMASKBITS)] - - def __init__(self, *args, **kwargs): - super(cpu_set_t, self).__init__(*args, **kwargs) - self.log = getLogger(self.__class__.__name__) - self.cpus = None - - def __str__(self): - return self.convert_bits_hr() - - def convert_hr_bits(self, txt): - """Convert human readable text into bits""" - self.cpus = [0] * CPU_SETSIZE - for rng in txt.split(','): - indices = [int(x) for x in rng.split('-')] * 2 # always at least 2 elements: twice the same or start,end,start,end - - ## sanity check - if indices[1] < indices[0]: - self.log.raiseException("convert_hr_bits: end is lower then start in '%s'" % rng) - elif indices[0] < 0: - self.log.raiseException("convert_hr_bits: negative start in '%s'" % rng) - elif indices[1] > CPU_SETSIZE + 1 : # also covers start, since end > start - self.log.raiseException("convert_hr_bits: end larger then max %s in '%s'" % (CPU_SETSIZE, rng)) - - self.cpus[indices[0]:indices[1] + 1] = [1] * (indices[1] + 1 - indices[0]) - self.log.debug("convert_hr_bits: converted %s into cpus %s" % (txt, self.cpus)) - - def convert_bits_hr(self): - """Convert __bits into human readable text""" - if self.cpus is None: - self.get_cpus() - cpus_index = [idx for idx, cpu in enumerate(self.cpus) if cpu == 1] - prev = -2 # not adjacent to 0 ! - parsed_idx = [] - for idx in cpus_index: - if prev + 1 < idx: - parsed_idx.append("%s" % idx) - else: - first_idx = parsed_idx[-1].split("-")[0] - parsed_idx[-1] = "%s-%s" % (first_idx, idx) - prev = idx - return ",".join(parsed_idx) - - def get_cpus(self): - """Convert bits in list len == CPU_SETSIZE - Use 1 / 0 per cpu - """ - self.cpus = [] - for bitmask in getattr(self, '__bits'): - for idx in xrange(NCPUBITS): - self.cpus.append(bitmask & 1) - bitmask >>= 1 - return self.cpus - - def set_cpus(self, cpus_list): - """Given list, set it as cpus""" - nr_cpus = len(cpus_list) - if nr_cpus > CPU_SETSIZE: - self.log.warning("set_cpus: length cpu list %s is larger then cpusetsize %s. Truncating to cpusetsize" % - (nr_cpus , CPU_SETSIZE)) - cpus_list = cpus_list[:CPU_SETSIZE] - elif nr_cpus < CPU_SETSIZE: - cpus_list.extend([0] * (CPU_SETSIZE - nr_cpus)) - - self.cpus = cpus_list - - def set_bits(self, cpus=None): - """Given self.cpus, set the bits""" - if cpus is not None: - self.set_cpus(cpus) - __bits = getattr(self, '__bits') - prev_cpus = map(long, self.cpus) - for idx in xrange(NMASKBITS): - cpus = [2 ** cpuidx for cpuidx, val in enumerate(self.cpus[idx * NCPUBITS:(idx + 1) * NCPUBITS]) if val == 1] - __bits[idx] = cpu_mask_t(sum(cpus)) - ## sanity check - if not prev_cpus == self.get_cpus(): - ## get_cpus() rescans - self.log.raiseException("set_bits: something went wrong: previous cpus %s; current ones %s" % (prev_cpus[:20], self.cpus[:20])) - else: - self.log.debug("set_bits: new set to %s" % self.convert_bits_hr()) - - def str_cpus(self): - """Return a string representation of the cpus""" - if self.cpus is None: - self.get_cpus() - return "".join(["%d" % x for x in self.cpus]) - -#/* Get the CPU affinity for a task */ -#extern int sched_getaffinity (pid_t __pid, size_t __cpusetsize, -# cpu_set_t *__cpuset); -def sched_getaffinity(cs=None, pid=None): - """Get the affinity""" - if cs is None: - cs = cpu_set_t() - if pid is None: - pid = os.getpid() - - ec = _libc.sched_getaffinity(pid_t(pid), - ctypes.sizeof(cpu_set_t), - ctypes.pointer(cs)) - if ec == 0: - _logger.debug("sched_getaffinity for pid %s returned cpuset %s" % (pid, cs)) - else: - _logger.error("sched_getaffinity failed for pid %s ec %s" % (pid, ec)) - return cs - - -#/* Set the CPU affinity for a task */ -#extern int sched_setaffinity (pid_t __pid, size_t __cpusetsize, -# cpu_set_t *__cpuset); -def sched_setaffinity(cs, pid=None): - """Set the affinity""" - if pid is None: - pid = os.getpid() - - ec = _libc.sched_setaffinity(pid_t(pid), - ctypes.sizeof(cpu_set_t), - ctypes.pointer(cs)) - if ec == 0: - _logger.debug("sched_setaffinity for pid %s and cpuset %s" % (pid, cs)) - else: - _logger.error("sched_setaffinity failed for pid %s cpuset %s ec %s" % (pid, cs, ec)) - -#/* Get index of currently used CPU. */ -#extern int sched_getcpu (void) __THROW; -def sched_getcpu(): - """Get currently used cpu""" - return _libc.sched_getcpu() - -#Utility function -# tobin not used anymore -def tobin(s): - """Convert integer to binary format""" - ## bin() missing in 2.4 - # eg: self.cpus.extend([int(x) for x in tobin(bitmask).zfill(NCPUBITS)[::-1]]) - if s <= 1: - return str(s) - else: - return tobin(s >> 1) + str(s & 1) - - -#/* Return the highest priority of any process specified by WHICH and WHO -# (see above); if WHO is zero, the current process, process group, or user -# (as specified by WHO) is used. A lower priority number means higher -# priority. Priorities range from PRIO_MIN to PRIO_MAX (above). */ -#extern int getpriority (__priority_which_t __which, id_t __who) __THROW; -# -#/* Set the priority of all processes specified by WHICH and WHO (see above) -# to PRIO. Returns 0 on success, -1 on errors. */ -#extern int setpriority (__priority_which_t __which, id_t __who, int __prio) -# __THROW; -def getpriority(which=None, who=None): - """Get the priority""" - if which is None: - which = PRIO_PROCESS - elif not which in (PRIO_PROCESS, PRIO_PGRP, PRIO_USER,): - _logger.raiseException("getpriority: which %s not in correct range" % which) - if who is None: - who = 0 # current which-ever - prio = _libc.getpriority(priority_which_t(which), - id_t(who), - ) - _logger.debug("getpriority prio %s for which %s who %s" % (prio, which, who)) - - return prio - -def setpriority(prio, which=None, who=None): - """Set the priority (aka nice)""" - if which is None: - which = PRIO_PROCESS - elif not which in (PRIO_PROCESS, PRIO_PGRP, PRIO_USER,): - _logger.raiseException("setpriority: which %s not in correct range" % which) - if who is None: - who = 0 # current which-ever - try: - prio = int(prio) - except: - _logger.raiseException("setpriority: failed to convert priority %s into int" % prio) - - if prio < PRIO_MIN or prio > PRIO_MAX: - _logger.raiseException("setpriority: prio not in allowed range MIN %s MAX %s" % (PRIO_MIN, PRIO_MAX)) - - ec = _libc.setpriority(priority_which_t(which), - id_t(who), - ctypes.c_int(prio) - ) - if ec == 0: - _logger.debug("setpriority for which %s who %s prio %s" % (which, who, prio)) - else: - _logger.error("setpriority failed for which %s who %s prio %s" % (which, who, prio)) - - -if __name__ == '__main__': - ## some examples of usage - setLogLevelDebug() - - cs = cpu_set_t() - print "__bits", cs.__bits - print "sizeof cpu_set_t", ctypes.sizeof(cs) - x = sched_getaffinity() - print "x", x - hr_mask = "1-5,7,9,10-15" - print hr_mask, x.convert_hr_bits(hr_mask) - print x - x.set_bits() - print x - - sched_setaffinity(x) - print sched_getaffinity() - - x.convert_hr_bits("1") - x.set_bits() - sched_setaffinity(x) - y = sched_getaffinity() - print x, y - - print sched_getcpu() - - ## resources - ## nice -n 5 python affinity.py prints 5 here - currentprio = getpriority() - print "getpriority", currentprio - newprio = 10 - setpriority(newprio) - newcurrentprio = getpriority() - print "getpriority", newcurrentprio - assert newcurrentprio == newprio diff --git a/vsc/utils/asyncprocess.py b/vsc/utils/asyncprocess.py deleted file mode 100644 index 8b3ce78012..0000000000 --- a/vsc/utils/asyncprocess.py +++ /dev/null @@ -1,191 +0,0 @@ -# # -# Copyright 2005 Josiah Carlson -# The Asynchronous Python Subprocess recipe was originally created by Josiah Carlson. -# and released under the GNU Library General Public License v2 or any later version -# on Jan 23, 2013. -# -# http://code.activestate.com/recipes/440554/ -# -# Copyright 2009-2013 Ghent University -# -# This file is part of vsc-base, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# the Hercules foundation (http://www.herculesstichting.be/in_English) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# http://github.com/hpcugent/vsc-base -# -# vsc-base is free software: you can redistribute it and/or modify -# it under the terms of the GNU Library General Public License as -# published by the Free Software Foundation, either version 2 of -# the License, or (at your option) any later version. -# -# vsc-base is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU Library General Public License -# along with vsc-base. If not, see . -# # - -""" -Module to allow Asynchronous subprocess use on Windows and Posix platforms - -The 'subprocess' module in Python 2.4 has made creating and accessing subprocess -streams in Python relatively convenient for all supported platforms, -but what if you want to interact with the started subprocess? -That is, what if you want to send a command, read the response, -and send a new command based on that response? - -Now there is a solution. -The included subprocess.Popen subclass adds three new commonly used methods: - - C{recv(maxsize=None)} - - C{recv_err(maxsize=None)} - - and C{send(input)} - -along with a utility method: - - {send_recv(input='', maxsize=None)}. - -C{recv()} and C{recv_err()} both read at most C{maxsize} bytes from the started subprocess. -C{send()} sends strings to the started subprocess. C{send_recv()} will send the provided input, -and read up to C{maxsize} bytes from both C{stdout} and C{stderr}. - -If any of the pipes are closed, the attributes for those pipes will be set to None, -and the methods will return None. - - - downloaded 05/08/2010 - - modified - - added STDOUT handle - - added maxread to recv_some (2012-08-30) - -@author: Josiah Carlson -@author: Stijn De Weirdt (Ghent University) -""" - -import errno -import fcntl # @UnresolvedImport -import os -import select # @UnresolvedImport -import subprocess -import time - - -PIPE = subprocess.PIPE -STDOUT = subprocess.STDOUT -MESSAGE = "Other end disconnected!" - - -class Popen(subprocess.Popen): - def recv(self, maxsize=None): - return self._recv('stdout', maxsize) - - def recv_err(self, maxsize=None): - return self._recv('stderr', maxsize) - - def send_recv(self, inp='', maxsize=None): - return self.send(inp), self.recv(maxsize), self.recv_err(maxsize) - - def get_conn_maxsize(self, which, maxsize): - if maxsize is None: - maxsize = 1024 - elif maxsize == 0: # do not use < 1: -1 means all - maxsize = 1 - return getattr(self, which), maxsize - - def _close(self, which): - getattr(self, which).close() - setattr(self, which, None) - - def send(self, inp): - if not self.stdin: - return None - - if not select.select([], [self.stdin], [], 0)[1]: - return 0 - - try: - written = os.write(self.stdin.fileno(), inp) - except OSError, why: - if why[0] == errno.EPIPE: # broken pipe - return self._close('stdin') - raise - - return written - - def _recv(self, which, maxsize): - conn, maxsize = self.get_conn_maxsize(which, maxsize) - if conn is None: - return None - - flags = fcntl.fcntl(conn, fcntl.F_GETFL) - if not conn.closed: - fcntl.fcntl(conn, fcntl.F_SETFL, flags | os.O_NONBLOCK) - - try: - if not select.select([conn], [], [], 0)[0]: - return '' - - r = conn.read(maxsize) - if not r: - return self._close(which) # close when nothing left to read - - if self.universal_newlines: - r = self._translate_newlines(r) - return r - finally: - if not conn.closed: - fcntl.fcntl(conn, fcntl.F_SETFL, flags) - - -def recv_some(p, t=.1, e=False, tr=5, stderr=False, maxread=None): - """ - @param p: process - @param t: max time to wait without any output before returning - @param e: boolean, raise exception is process stopped - @param tr: time resolution used for intermediate sleep - @param stderr: boolean, read from stderr - @param maxread: stop when max read bytes have been read (before timeout t kicks in) (-1: read all) - - Changes made wrt original: - - add maxread here - - set e to False by default - """ - if maxread is None: - maxread = -1 - - if tr < 1: - tr = 1 - x = time.time() + t - y = [] - len_y = 0 - r = '' - pr = p.recv - if stderr: - pr = p.recv_err - while (maxread < 0 or len_y <= maxread) and (time.time() < x or r): - r = pr(maxread) - if r is None: - if e: - raise Exception(MESSAGE) - else: - break - elif r: - y.append(r) - len_y += len(r) - else: - time.sleep(max((x - time.time()) / tr, 0)) - return ''.join(y) - - -def send_all(p, data): - """ - Send data to process p - """ - while len(data): - sent = p.send(data) - if sent is None: - raise Exception(MESSAGE) - data = buffer(data, sent) diff --git a/vsc/utils/daemon.py b/vsc/utils/daemon.py deleted file mode 100644 index 2568f7cc30..0000000000 --- a/vsc/utils/daemon.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env python - -## -# -# Copyright 2007 Sander Marechal (http://www.jejik.com) -# Released as Public Domain -# Retrieved from: -# http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/ -# -## -""" -Module to make python scripts run in background. - -@author: Sander Marechal -""" - -import sys, os, time, atexit -from signal import SIGTERM - -class Daemon: - """ - A generic daemon class. - - Usage: subclass the Daemon class and override the run() method - """ - def __init__(self, pidfile, stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'): - self.stdin = stdin - self.stdout = stdout - self.stderr = stderr - self.pidfile = pidfile - - def daemonize(self): - """ - do the UNIX double-fork magic, see Stevens' "Advanced - Programming in the UNIX Environment" for details (ISBN 0201563177) - http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16 - """ - try: - pid = os.fork() - if pid > 0: - # exit first parent - sys.exit(0) - except OSError, e: - sys.stderr.write("fork #1 failed: %d (%s)\n" % (e.errno, e.strerror)) - sys.exit(1) - - # decouple from parent environment - os.chdir("/") - os.setsid() - os.umask(0) - - # do second fork - try: - pid = os.fork() - if pid > 0: - # exit from second parent - sys.exit(0) - except OSError, e: - sys.stderr.write("fork #2 failed: %d (%s)\n" % (e.errno, e.strerror)) - sys.exit(1) - - # redirect standard file descriptors - sys.stdout.flush() - sys.stderr.flush() - si = file(self.stdin, 'r') - so = file(self.stdout, 'a+') - se = file(self.stderr, 'a+', 0) - os.dup2(si.fileno(), sys.stdin.fileno()) - os.dup2(so.fileno(), sys.stdout.fileno()) - os.dup2(se.fileno(), sys.stderr.fileno()) - - # write pidfile - atexit.register(self.delpid) - pid = str(os.getpid()) - file(self.pidfile, 'w+').write("%s\n" % pid) - - def delpid(self): - os.remove(self.pidfile) - - def start(self): - """ - Start the daemon - """ - # Check for a pidfile to see if the daemon already runs - try: - pf = file(self.pidfile, 'r') - pid = int(pf.read().strip()) - pf.close() - except IOError: - pid = None - - if pid: - message = "pidfile %s already exist. Daemon already running?\n" - sys.stderr.write(message % self.pidfile) - sys.exit(1) - - # Start the daemon - self.daemonize() - self.run() - - def stop(self): - """ - Stop the daemon - """ - # Get the pid from the pidfile - try: - pf = file(self.pidfile, 'r') - pid = int(pf.read().strip()) - pf.close() - except IOError: - pid = None - - if not pid: - message = "pidfile %s does not exist. Daemon not running?\n" - sys.stderr.write(message % self.pidfile) - return # not an error in a restart - - # Try killing the daemon process - try: - while 1: - os.kill(pid, SIGTERM) - time.sleep(0.1) - except OSError, err: - err = str(err) - if err.find("No such process") > 0: - if os.path.exists(self.pidfile): - os.remove(self.pidfile) - else: - print str(err) - sys.exit(1) - - def restart(self): - """ - Restart the daemon - """ - self.stop() - self.start() - - def run(self): - """ - You should override this method when you subclass Daemon. It will be called after the process has been - daemonized by start() or restart(). - """ - pass diff --git a/vsc/utils/dateandtime.py b/vsc/utils/dateandtime.py deleted file mode 100644 index 9f599502e8..0000000000 --- a/vsc/utils/dateandtime.py +++ /dev/null @@ -1,354 +0,0 @@ -## -# -# Copyright 2012-2013 Ghent University -# -# This file is part of vsc-base, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# the Hercules foundation (http://www.herculesstichting.be/in_English) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# http://github.com/hpcugent/vsc-base -# -# vsc-base is free software: you can redistribute it and/or modify -# it under the terms of the GNU Library General Public License as -# published by the Free Software Foundation, either version 2 of -# the License, or (at your option) any later version. -# -# vsc-base is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU Library General Public License -# along with vsc-base. If not, see . -## -""" -Module with various convenience functions and classes to deal with date, time and timezone - -@author: Stijn De Weirdt (Ghent University) -""" - -import calendar -import re -import time as _time -from datetime import tzinfo, timedelta, datetime, date - -try: - any([0, 1]) -except: - from vsc.utils.missing import any - - -class FancyMonth: - """Convenience class for month math""" - def __init__(self, tmpdate=None, year=None, month=None, day=None): - """Initialise the month based on first day of month of tmpdate""" - - if tmpdate is None: - tmpdate = date.today() - - if day is None: - day = tmpdate.day - if month is None: - month = tmpdate.month - if year is None: - year = tmpdate.year - - self.date = date(year, month, day) - - self.first = None - self.last = None - self.nrdays = None - - # when calculating deltas, include non-full months - # eg when True, nr of months between last day of month - # and first day of following month is 2 - self.include = True - - self.set_details() - - def set_details(self): - """Get first/last day of the month of date""" - class MyCalendar(object): - """Backport minimal calendar.Calendar code from 2.7 to support itermonthdays in 2.4""" - def __init__(self, firstweekday=0): - self.firstweekday = firstweekday # 0 = Monday, 6 = Sunday - - def itermonthdates(self, year, month): - """ - Return an iterator for one month. The iterator will yield datetime.date - values and will always iterate through complete weeks, so it will yield - dates outside the specified month. - """ - _date = date(year, month, 1) - # Go back to the beginning of the week - days = (_date.weekday() - self.firstweekday) % 7 - _date -= timedelta(days=days) - oneday = timedelta(days=1) - while True: - yield _date - _date += oneday - if _date.month != month and _date.weekday() == self.firstweekday: - break - - def itermonthdays(self, year, month): - """ - Like itermonthdates(), but will yield day numbers. For days outside - the specified month the day number is 0. - """ - for _date in self.itermonthdates(year, month): - if _date.month != month: - yield 0 - else: - yield _date.day - - if 'Calendar' in dir(calendar): # py2.5+ - c = calendar.Calendar() - else: - c = MyCalendar() - self.nrdays = len([x for x in c.itermonthdays(self.date.year, self.date.month) if x > 0]) - - self.first = date(self.date.year, self.date.month, 1) - - self.last = date(self.date.year, self.date.month, self.nrdays) - - def get_start_end(self, otherdate): - """Return tuple date and otherdate ordered oldest first""" - if self.date > otherdate: - start = otherdate - end = self.date - else: - start = self.date - end = otherdate - - return start, end - - def number(self, otherdate): - """Calculate the number of months between this month (date actually) and otherdate - """ - if self.include is False: - msg = "number: include=False not implemented" - raise(Exception(msg)) - else: - startdate, enddate = self.get_start_end(otherdate) - - if startdate == enddate: - nr = 0 - else: - nr = (enddate.year - startdate.year) * 12 + enddate.month - startdate.month + 1 - - return nr - - def get_other(self, shift=-1): - """Return month that is shifted shift months: negative integer is in past, positive is in future""" - new = self.date.year * 12 + self.date.month - 1 + shift - return self.__class__(date(new // 12, new % 12 + 1, 01)) - - def interval(self, otherdate): - """Return time ordered list of months between date and otherdate""" - if self.include is False: - msg = "interval: include=False not implemented" - raise(Exception(msg)) - else: - nr = self.number(otherdate) - startdate, enddate = self.get_start_end(otherdate) - - start = self.__class__(startdate) - all_dates = [start.get_other(m) for m in range(nr)] - - return all_dates - - def parser(self, txt): - """Based on strings, return date: eg BEGINTHIS returns first day of the current month""" - supportedtime = ('BEGIN', 'END',) - supportedshift = ['THIS', 'LAST', 'NEXT'] - regtxt = r"^(%s)(%s)?" % ('|'.join(supportedtime), '|'.join(supportedshift)) - - reseervedregexp = re.compile(regtxt) - reg = reseervedregexp.search(txt) - if not reg: - msg = "parse: no match for regexp %s for txt %s" % (regtxt, txt) - raise(Exception(msg)) - - shifttxt = reg.group(2) - if shifttxt is None or shifttxt == 'THIS': - shift = 0 - elif shifttxt == 'LAST': - shift = -1 - elif shifttxt == 'NEXT': - shift = 1 - else: - msg = "parse: unknown shift %s (supported: %s)" % (shifttxt, supportedshift) - raise(Exception(msg)) - - nm = self.get_other(shift) - - timetxt = reg.group(1) - if timetxt == 'BEGIN': - res = nm.first - elif timetxt == 'END': - res = nm.last - else: - msg = "parse: unknown time %s (supported: %s)" % (timetxt, supportedtime) - raise(Exception(msg)) - - return res - - -def date_parser(txt): - """Parse txt - - @type txt: string - - @param txt: date to be parsed. Usually in C{YYYY-MM-DD} format, - but also C{(BEGIN|END)(THIS|LAST|NEXT)MONTH}, or even - C{(BEGIN | END)(JANUARY | FEBRUARY | MARCH | APRIL | MAY | JUNE | JULY | AUGUST | SEPTEMBER | OCTOBER | NOVEMBER | DECEMBER)} - """ - - reserveddate = ('TODAY',) - testsupportedmonths = [txt.endswith(calendar.month_name[x].upper()) for x in range(1, 13)] - - if txt.endswith('MONTH'): - m = FancyMonth() - res = m.parser(txt) - elif any(testsupportedmonths): - # set day=1 or this will fail on day's with an index more then the count of days then the month you want to parse - # e.g. will fail on 31'st when trying to parse april - m = FancyMonth(month=testsupportedmonths.index(True) + 1, day=1) - res = m.parser(txt) - elif txt in reserveddate: - if txt in ('TODAY',): - m = FancyMonth() - res = m.date - else: - msg = 'dateparser: unimplemented reservedword %s' % txt - raise(Exception(msg)) - else: - try: - datetuple = [int(x) for x in txt.split("-")] - res = date(*datetuple) - except: - msg = ("dateparser: failed on '%s' date txt expects a YYYY-MM-DD format or " - "reserved words %s") % (txt, ','.join(reserveddate)) - raise(Exception(msg)) - - return res - - -def datetime_parser(txt): - """Parse txt: tmpdate YYYY-MM-DD HH:MM:SS.mmmmmm in datetime.datetime - - date part is parsed with date_parser - """ - tmpts = txt.split(" ") - tmpdate = date_parser(tmpts[0]) - - datetuple = [tmpdate.year, tmpdate.month, tmpdate.day] - if len(tmpts) > 1: - # add hour and minutes - datetuple.extend([int(x) for x in tmpts[1].split(':')[:2]]) - - try: - sects = tmpts[1].split(':')[2].split('.') - except: - sects = [0] - # add seconds - datetuple.append(int(sects[0])) - if len(sects) > 1: - # add microseconds - datetuple.append(int(float('.%s' % sects[1]) * 10 ** 6)) - - res = datetime(*datetuple) - - return res - - -def timestamp_parser(timestamp): - """Parse timestamp to datetime""" - return datetime.fromtimestamp(float(timestamp)) - -# -# example code from http://docs.python.org/library/datetime.html -# Implements Local, the local timezone -# - -ZERO = timedelta(0) -HOUR = timedelta(hours=1) - - -# A UTC class. -class UTC(tzinfo): - """UTC""" - - def utcoffset(self, dt): - return ZERO - - def tzname(self, dt): - return "UTC" - - def dst(self, dt): - return ZERO - -utc = UTC() - - -class FixedOffset(tzinfo): - """Fixed offset in minutes east from UTC. - - This is a class for building tzinfo objects for fixed-offset time zones. - Note that FixedOffset(0, "UTC") is a different way to build a - UTC tzinfo object. - """ - def __init__(self, offset, name): - self.__offset = timedelta(minutes=offset) - self.__name = name - - def utcoffset(self, dt): - return self.__offset - - def tzname(self, dt): - return self.__name - - def dst(self, dt): - return ZERO - - -STDOFFSET = timedelta(seconds=-_time.timezone) -if _time.daylight: - DSTOFFSET = timedelta(seconds=-_time.altzone) -else: - DSTOFFSET = STDOFFSET - -DSTDIFF = DSTOFFSET - STDOFFSET - - -class LocalTimezone(tzinfo): - """ - A class capturing the platform's idea of local time. - """ - - def utcoffset(self, dt): - if self._isdst(dt): - return DSTOFFSET - else: - return STDOFFSET - - def dst(self, dt): - if self._isdst(dt): - return DSTDIFF - else: - return ZERO - - def tzname(self, dt): - return _time.tzname[self._isdst(dt)] - - def _isdst(self, dt): - tt = (dt.year, dt.month, dt.day, - dt.hour, dt.minute, dt.second, - dt.weekday(), 0, 0) - stamp = _time.mktime(tt) - tt = _time.localtime(stamp) - return tt.tm_isdst > 0 - -Local = LocalTimezone() diff --git a/vsc/utils/fancylogger.py b/vsc/utils/fancylogger.py deleted file mode 100644 index 56a2ded26f..0000000000 --- a/vsc/utils/fancylogger.py +++ /dev/null @@ -1,679 +0,0 @@ -#!/usr/bin/env python -# # -# Copyright 2011-2013 Ghent University -# -# This file is part of vsc-base, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# the Hercules foundation (http://www.herculesstichting.be/in_English) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# http://github.com/hpcugent/vsc-base -# -# vsc-base is free software: you can redistribute it and/or modify -# it under the terms of the GNU Library General Public License as -# published by the Free Software Foundation, either version 2 of -# the License, or (at your option) any later version. -# -# vsc-base is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU Library General Public License -# along with vsc-base. If not, see . -# # -""" -This module implements a fancy logger on top of python logging - -It adds: - - custom specifiers for mpi logging (the mpirank) with autodetection of mpi - - custom specifier for always showing the calling function's name - - rotating file handler - - a default formatter. - - logging to an UDP server (vsc.logging.logdaemon.py f.ex.) - - easily setting loglevel - - easily add extra specifiers in the log record - - internal debugging through environment variables - FANCYLOGGER_GETLOGGER_DEBUG for getLogger - FANCYLOGGER_LOGLEVEL_DEBUG for setLogLevel - -usage: - ->>> from vsc.utils import fancylogger ->>> # will log to screen by default ->>> fancylogger.logToFile('dir/filename') ->>> fancylogger.setLogLevelDebug() # set global loglevel to debug ->>> logger = fancylogger.getLogger(name) # get a logger with a specific name ->>> logger.setLevel(level) # set local debugging level ->>> # If you want the logger to be showing modulename.functionname as the name, use ->>> fancylogger.getLogger(fname=True) ->>> # you can use the handler to set a different formatter by using ->>> handler = fancylogger.logToFile('dir/filename') ->>> formatstring = '%(asctime)-15s %(levelname)-10s %(mpirank)-5s %(funcname)-15s %(threadName)-10s %(message)s' ->>> handler.setFormatter(logging.Formatter(formatstring)) ->>> # setting a global loglevel will impact all logers: ->>> from vsc.utils import fancylogger ->>> logger = fancylogger.getLogger("test") ->>> logger.warning("warning") -2012-01-05 14:03:18,238 WARNING .test. MainThread warning ->>> logger.debug("warning") ->>> fancylogger.setLogLevelDebug() ->>> logger.debug("warning") -2012-01-05 14:03:46,222 DEBUG .test. MainThread warning - -Logging to a udp server: - - set an environment variable FANCYLOG_SERVER and FANCYLOG_SERVER_PORT (optionally) - - this will make fancylogger log to that that server and port instead of the screen. - -@author: Jens Timmerman (Ghent University) -@author: Stijn De Weirdt (Ghent University) -@author: Kenneth Hoste (Ghent University) -""" - -import inspect -import logging -import logging.handlers -import os -import sys -import threading -import traceback -import weakref -from distutils.version import LooseVersion - -# constants -TEST_LOGGING_FORMAT = '%(levelname)-10s %(name)-15s %(threadName)-10s %(message)s' -DEFAULT_LOGGING_FORMAT = '%(asctime)-15s ' + TEST_LOGGING_FORMAT -FANCYLOG_LOGGING_FORMAT = None - -# DEFAULT_LOGGING_FORMAT= '%(asctime)-15s %(levelname)-10s %(module)-15s %(threadName)-10s %(message)s' -MAX_BYTES = 100 * 1024 * 1024 # max bytes in a file with rotating file handler -BACKUPCOUNT = 10 # number of rotating log files to save - -DEFAULT_UDP_PORT = 5005 - -# register new loglevelname -logging.addLevelName(logging.CRITICAL * 2 + 1, 'APOCALYPTIC') -# register EXCEPTION and FATAL alias -logging._levelNames['EXCEPTION'] = logging.ERROR -logging._levelNames['FATAL'] = logging.CRITICAL - - -# mpi rank support -try: - from mpi4py import MPI - _MPIRANK = str(MPI.COMM_WORLD.Get_rank()) - if MPI.COMM_WORLD.Get_size() > 1: - # enable mpi rank when mpi is used - DEFAULT_LOGGING_FORMAT = '%(asctime)-15s %(levelname)-10s %(name)-15s' \ - " mpi: %(mpirank)s %(threadName)-10s %(message)s" -except ImportError: - _MPIRANK = "N/A" - - -class MissingLevelName(KeyError): - pass - - -def getLevelInt(level_name): - """Given a level name, return the int value""" - if not isinstance(level_name, basestring): - raise TypeError('Provided name %s is not a string (type %s)' % (level_name, type(level_name))) - - level = logging.getLevelName(level_name) - if isinstance(level, basestring): - raise MissingLevelName('Unknown loglevel name %s' % level_name) - - return level - - -class FancyStreamHandler(logging.StreamHandler): - """The logging StreamHandler with uniform named arg in __init__ for selecting the stream.""" - def __init__(self, stream=None, stdout=None): - """Initialize the stream (default is sys.stderr) - - stream : a specific stream to use - - stdout: if True and no stream specified, set stream to sys.stdout (False log to stderr) - """ - logging.StreamHandler.__init__(self) - if stream is not None: - pass - elif stdout is False or stdout is None: - stream = sys.stderr - elif stdout is True: - stream = sys.stdout - - self.stream = stream - - -class FancyLogRecord(logging.LogRecord): - """ - This class defines a custom log record. - Adding extra specifiers is as simple as adding attributes to the log record - """ - def __init__(self, *args, **kwargs): - logging.LogRecord.__init__(self, *args, **kwargs) - # modify custom specifiers here - # we won't do this when running with -O, becuase this might be a heavy operation - # the __debug__ operation is actually recognised by the python compiler and it won't even do a single comparison - if __debug__: - self.className = _getCallingClassName(depth=5) - else: - self.className = 'N/A' - self.mpirank = _MPIRANK - - -# Custom logger that uses our log record -class FancyLogger(logging.getLoggerClass()): - """ - This is a custom Logger class that uses the FancyLogRecord - and has extra log methods raiseException and deprecated and - streaming versions for debug,info,warning and error. - """ - # this attribute can be checked to know if the logger is thread aware - _thread_aware = True - - # method definition as it is in logging, can't change this - def makeRecord(self, name, level, pathname, lineno, msg, args, excinfo, func=None, extra=None): - """ - overwrite make record to use a fancy record (with more options) - """ - logrecordcls = logging.LogRecord - if hasattr(self, 'fancyrecord') and self.fancyrecord: - logrecordcls = FancyLogRecord - try: - new_msg = str(msg) - except UnicodeEncodeError: - new_msg = msg.encode('utf8', 'replace') - return logrecordcls(name, level, pathname, lineno, new_msg, args, excinfo) - - def raiseException(self, message, exception=None, catch=False): - """ - logs an exception (as warning, since it can be caught higher up and handled) - and raises it afterwards - catch: boolean, try to catch raised exception and add relevant info to message - (this will also happen if exception is not specified) - """ - fullmessage = message - - if catch or exception is None: - # assumes no control by codemonkey - # lets see if there is something more to report on - exc, detail, tb = sys.exc_info() - if exc is not None: - if exception is None: - exception = exc - # extend the message with the traceback and some more details - # or use self.exception() instead of self.warning()? - tb_text = "\n".join(traceback.format_tb(tb)) - message += " (%s)" % detail - fullmessage += " (%s\n%s)" % (detail, tb_text) - - if exception is None: - exception = Exception - - self.warning(fullmessage) - raise exception(message) - - def deprecated(self, msg, cur_ver, max_ver, depth=2, exception=None, *args, **kwargs): - """ - Log deprecation message, throw error if current version is passed given threshold. - - Checks only major/minor version numbers (MAJ.MIN.x) by default, controlled by 'depth' argument. - """ - loose_cv = LooseVersion(cur_ver) - loose_mv = LooseVersion(max_ver) - - loose_cv.version = loose_cv.version[:depth] - loose_mv.version = loose_mv.version[:depth] - - if loose_cv >= loose_mv: - self.raiseException("DEPRECATED (since v%s) functionality used: %s" % (max_ver, msg), exception=exception) - else: - deprecation_msg = "Deprecated functionality, will no longer work in v%s: %s" % (max_ver, msg) - self.warning(deprecation_msg) - - def _handleFunction(self, function, levelno, **kwargs): - """ - Walk over all handlers like callHandlers and execute function on each handler - """ - c = self - found = 0 - while c: - for hdlr in c.handlers: - found = found + 1 - if levelno >= hdlr.level: - function(hdlr, **kwargs) - if not c.propagate: - c = None # break out - else: - c = c.parent - - def setLevelName(self, level_name): - """Set the level by name.""" - # This is supported in py27 setLevel code, but not in py24 - self.setLevel(getLevelInt(level_name)) - - def streamLog(self, levelno, data): - """ - Add (continuous) data to an existing message stream (eg a stream after a logging.info() - """ - if isinstance(levelno, str): - levelno = getLevelInt(levelno) - - def write_and_flush_stream(hdlr, data=None): - """Write to stream and flush the handler""" - if (not hasattr(hdlr, 'stream')) or hdlr.stream is None: - # no stream or not initialised. - raise("write_and_flush_stream failed. No active stream attribute.") - if data is not None: - hdlr.stream.write(data) - hdlr.flush() - - # only log when appropriate (see logging.Logger.log()) - if self.isEnabledFor(levelno): - self._handleFunction(write_and_flush_stream, levelno, data=data) - - def streamDebug(self, data): - """Get a DEBUG loglevel streamLog""" - self.streamLog('DEBUG', data) - - def streamInfo(self, data): - """Get a INFO loglevel streamLog""" - self.streamLog('INFO', data) - - def streamError(self, data): - """Get a ERROR loglevel streamLog""" - self.streamLog('ERROR', data) - - def _get_parent_info(self, verbose=True): - """Return some logger parent related information""" - def info(x): - res = [x, x.name, logging.getLevelName(x.getEffectiveLevel()), logging.getLevelName(x.level), x.disabled] - if verbose: - res.append([(h, logging.getLevelName(h.level)) for h in x.handlers]) - return res - - parentinfo = [] - logger = self - parentinfo.append(info(logger)) - while logger.parent is not None: - logger = logger.parent - parentinfo.append(info(logger)) - return parentinfo - - def get_parent_info(self, prefix, verbose=True): - """Return pretty text version""" - rev_parent_info = self._get_parent_info(verbose=verbose) - return ["%s %s%s" % (prefix, " " * 4 * idx, info) for idx, info in enumerate(rev_parent_info)] - - def __copy__(self): - """Return shallow copy, in this case reference to current logger""" - return getLogger(self.name, fname=False, clsname=False) - - def __deepcopy__(self, memo): - """This behaviour is undefined, fancylogger will return shallow copy, instead just crashing.""" - return self.__copy__() - - -def thread_name(): - """ - returns the current threads name - """ - return threading.currentThread().getName() - - -def getLogger(name=None, fname=True, clsname=False, fancyrecord=None): - """ - returns a fancylogger - if fname is True, the loggers name will be 'name[.classname].functionname' - if clsname is True the loggers name will be 'name.classname[.functionname]' - This will return a logger with a fancylog record, which includes the className template for the logformat - This can make your code a lot slower, so this can be dissabled by setting fancyrecord to False, and - will also be disabled if a Name is set, and fancyrecord is not set to True - """ - nameparts = [getRootLoggerName()] - - if name: - nameparts.append(name) - elif fancyrecord is None or fancyrecord: # only be fancy if fancyrecord is True or no name is given - fancyrecord = True - fancyrecord = bool(fancyrecord) # make sure fancyrecord is a nice bool, not None - - if clsname: - nameparts.append(_getCallingClassName()) - if fname: - nameparts.append(_getCallingFunctionName()) - fullname = ".".join(nameparts) - - l = logging.getLogger(fullname) - l.fancyrecord = fancyrecord - if os.environ.get('FANCYLOGGER_GETLOGGER_DEBUG', '0').lower() in ('1', 'yes', 'true', 'y'): - print 'FANCYLOGGER_GETLOGGER_DEBUG', - print 'name', name, 'fname', fname, 'fullname', fullname, - print 'parent_info verbose' - print "\n".join(l.get_parent_info("FANCYLOGGER_GETLOGGER_DEBUG")) - sys.stdout.flush() - return l - - -def _getCallingFunctionName(): - """ - returns the name of the function calling the function calling this function - (for internal use only) - """ - try: - return inspect.stack()[2][3] - except Exception: - return "?" - - -def _getCallingClassName(depth=2): - """ - returns the name of the class calling the function calling this function - (for internal use only) - """ - try: - return inspect.stack()[depth][0].f_locals['self'].__class__.__name__ - - except Exception: - return "?" - - -def getRootLoggerName(): - """ - returns the name of the root module - this is the module that is actually running everything and so doing the logging - """ - try: - return inspect.stack()[-1][1].split('/')[-1].split('.')[0] - except Exception: - return "?" - - -def logToScreen(enable=True, handler=None, name=None, stdout=False): - """ - enable (or disable) logging to screen - returns the screenhandler (this can be used to later disable logging to screen) - - if you want to disable logging to screen, pass the earlier obtained screenhandler - - you can also pass the name of the logger for which to log to the screen - otherwise you'll get all logs on the screen - - by default, logToScreen will log to stderr; logging to stdout instead can be done - by setting the 'stdout' parameter to True - """ - handleropts = {'stdout': stdout} - - return _logToSomething(FancyStreamHandler, - handleropts, - loggeroption='logtoscreen_stdout_%s' % str(stdout), - name=name, - enable=enable, - handler=handler, - ) - - -def logToFile(filename, enable=True, filehandler=None, name=None, max_bytes=MAX_BYTES, backup_count=BACKUPCOUNT): - """ - enable (or disable) logging to file - given filename - will log to a file with the given name using a rotatingfilehandler - this will let the file grow to MAX_BYTES and then rotate it - saving the last BACKUPCOUNT files. - - returns the filehandler (this can be used to later disable logging to file) - - if you want to disable logging to file, pass the earlier obtained filehandler - """ - handleropts = {'filename': filename, - 'mode': 'a', - 'maxBytes': max_bytes, - 'backupCount': backup_count, - } - return _logToSomething(logging.handlers.RotatingFileHandler, - handleropts, - loggeroption='logtofile_%s' % filename, - name=name, - enable=enable, - handler=filehandler, - ) - - -def logToUDP(hostname, port=5005, enable=True, datagramhandler=None, name=None): - """ - enable (or disable) logging to udp - given hostname and port. - - returns the filehandler (this can be used to later disable logging to udp) - - if you want to disable logging to udp, pass the earlier obtained filehandler, - and set boolean = False - """ - handleropts = {'hostname': hostname, 'port': port} - return _logToSomething(logging.handlers.DatagramHandler, - handleropts, - loggeroption='logtoudp_%s:%s' % (hostname, str(port)), - name=name, - enable=enable, - handler=datagramhandler, - ) - - -def _logToSomething(handlerclass, handleropts, loggeroption, enable=True, name=None, handler=None): - """ - internal function to enable (or disable) logging to handler named handlername - handleropts is options dictionary passed to create the handler instance - - returns the handler (this can be used to later disable logging to file) - - if you want to disable logging to the handler, pass the earlier obtained handler - """ - logger = getLogger(name, fname=False, clsname=False) - - if not hasattr(logger, loggeroption): - # not set. - setattr(logger, loggeroption, False) # set default to False - - if enable: - if not getattr(logger, loggeroption): - if handler is None: - if FANCYLOG_LOGGING_FORMAT is None: - f_format = DEFAULT_LOGGING_FORMAT - else: - f_format = FANCYLOG_LOGGING_FORMAT - formatter = logging.Formatter(f_format) - handler = handlerclass(**handleropts) - handler.setFormatter(formatter) - logger.addHandler(handler) - setattr(logger, loggeroption, handler) - else: - handler = getattr(logger, loggeroption) - elif not enable: - # stop logging to X - if handler is None: - if len(logger.handlers) == 1: - # removing the last logger doesn't work - # it will be re-added if only one handler is present - # so we will just make it quiet by setting the loglevel extremely high - zerohandler = logger.handlers[0] - # no logging should be done with APOCALYPTIC, so silence happens - zerohandler.setLevel(getLevelInt('APOCALYPTIC')) - else: # remove the handler set with this loggeroption - handler = getattr(logger, loggeroption) - logger.removeHandler(handler) - if hasattr(handler, 'close') and callable(handler.close): - handler.close() - else: - logger.removeHandler(handler) - setattr(logger, loggeroption, False) - return handler - - -def _getSysLogFacility(name=None): - """Look for proper syslog facility - typically the syslog/rsyslog config has an entry - # Log anything (except mail) of level info or higher. - # Don't log private authentication messages! - *.info;mail.none;authpriv.none;cron.none /var/log/messages - - name -> LOG_%s % name.upper() - Default log facility is user /LOG_USER - """ - - if name is None: - name = 'user' - - facility = getattr(logging.handlers.SysLogHandler, - "LOG_%s" % name.upper(), logging.handlers.SysLogHandler.LOG_USER) - - return facility - - -def logToDevLog(enable=True, name=None, handler=None): - """Log to syslog through /dev/log""" - devlog = '/dev/log' - syslogoptions = {'address': devlog, - 'facility': _getSysLogFacility() - } - return _logToSomething(logging.handlers.SysLogHandler, - syslogoptions, 'logtodevlog', enable=enable, name=name, handler=handler) - - -# Change loglevel -def setLogLevel(level): - """ - set a global log level (for this root logger) - """ - if isinstance(level, basestring): - level = getLevelInt(level) - logger = getLogger(fname=False, clsname=False) - logger.setLevel(level) - if os.environ.get('FANCYLOGGER_LOGLEVEL_DEBUG', '0').lower() in ('1', 'yes', 'true', 'y'): - print "FANCYLOGGER_LOGLEVEL_DEBUG", level, logging.getLevelName(level) - print "\n".join(logger.get_parent_info("FANCYLOGGER_LOGLEVEL_DEBUG")) - sys.stdout.flush() - - -def setLogLevelDebug(): - """ - shorthand for setting debug level - """ - setLogLevel('DEBUG') - - -def setLogLevelInfo(): - """ - shorthand for setting loglevel to Info - """ - setLogLevel('INFO') - - -def setLogLevelWarning(): - """ - shorthand for setting loglevel to Warning - """ - setLogLevel('WARNING') - - -def setLogLevelError(): - """ - shorthand for setting loglevel to Error - """ - setLogLevel('ERROR') - - -def getAllExistingLoggers(): - """ - @return: the existing loggers, in a list of C{(name, logger)} tuples - """ - rootlogger = logging.getLogger(name=False) - # undocumented manager (in 2.4 and later) - manager = rootlogger.manager - - loggerdict = getattr(manager, 'loggerDict') - - # return list of (name,logger) tuple - return [x for x in loggerdict.items()] - - -def getAllNonFancyloggers(): - """ - @return: all loggers that are not fancyloggers - """ - return [x for x in getAllExistingLoggers() if not isinstance(x[1], FancyLogger)] - - -def getAllFancyloggers(): - """ - Return all loggers that are not fancyloggers - """ - return [x for x in getAllExistingLoggers() if isinstance(x[1], FancyLogger)] - - -def setLogFormat(f_format): - """Set the log format. (Has to be set before logToSomething is called).""" - global FANCYLOG_LOGGING_FORMAT - FANCYLOG_LOGGING_FORMAT = f_format - - -def setTestLogFormat(): - """Set the log format to the test format (i.e. without timestamp).""" - setLogFormat(TEST_LOGGING_FORMAT) - - -# Register our logger -logging.setLoggerClass(FancyLogger) - -# log to a server if FANCYLOG_SERVER is set. -_default_logTo = None -if 'FANCYLOG_SERVER' in os.environ: - server = os.environ['FANCYLOG_SERVER'] - port = DEFAULT_UDP_PORT - if ':' in server: - server, port = server.split(':') - - # maybe the port was specified in the FANCYLOG_SERVER_PORT env var. this takes precedence - if 'FANCYLOG_SERVER_PORT' in os.environ: - port = int(os.environ['FANCYLOG_SERVER_PORT']) - port = int(port) - - logToUDP(server, port) - _default_logTo = logToUDP -else: - # log to screen by default - logToScreen(enable=True) - _default_logTo = logToScreen - - -_default_handlers = logging._handlerList[:] # There's always one - - -def _enable_disable_default_handlers(enable): - """Interact with the default handlers to enable or disable them""" - if _default_logTo is None: - return - for hndlr in _default_handlers: - # py2.7 are weakrefs, 2.6 not - if isinstance(hndlr, weakref.ref): - handler = hndlr() - else: - handler = hndlr - - try: - _default_logTo(enable=enable, handler=handler) - except: - pass - - -def disableDefaultHandlers(): - """Disable the default handlers on all fancyloggers - - if this is the last logger, it will just set the logLevel very high - """ - _enable_disable_default_handlers(False) - - -def enableDefaultHandlers(): - """(re)Enable the default handlers on all fancyloggers""" - _enable_disable_default_handlers(True) diff --git a/vsc/utils/frozendict.py b/vsc/utils/frozendict.py deleted file mode 100644 index 21604eed7a..0000000000 --- a/vsc/utils/frozendict.py +++ /dev/null @@ -1,56 +0,0 @@ -# taken from https://github.com/slezica/python-frozendict on March 14th 2014 (commit ID b27053e4d1) -# -# Copyright (c) 2012 Santiago Lezica -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -# TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE - -import operator -from UserDict import DictMixin - - -# minor adjustments: -# * renamed to FrozenDict -# * deriving from DictMixin instead of collections.Mapping to make it Python 2.4 compatible -# see also http://docs.python.org/2/library/userdict.html#UserDict.DictMixin -class FrozenDict(object, DictMixin): - - def __init__(self, *args, **kwargs): - self.__dict = dict(*args, **kwargs) - self.__hash = None - - def __getitem__(self, key): - return self.__dict[key] - - def copy(self, **add_or_replace): - return FrozenDict(self, **add_or_replace) - - def __iter__(self): - return iter(self.__dict) - - def __len__(self): - return len(self.__dict) - - def __repr__(self): - return '' % repr(self.__dict) - - def __hash__(self): - if self.__hash is None: - self.__hash = reduce(operator.xor, map(hash, self.iteritems()), 0) - - return self.__hash - - # minor adjustment: define missing keys() method - def keys(self): - return self.__dict.keys() diff --git a/vsc/utils/generaloption.py b/vsc/utils/generaloption.py deleted file mode 100644 index c3b94ee983..0000000000 --- a/vsc/utils/generaloption.py +++ /dev/null @@ -1,1382 +0,0 @@ -# -# -# Copyright 2011-2013 Ghent University -# -# This file is part of vsc-base, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# the Hercules foundation (http://www.herculesstichting.be/in_English) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# http://github.com/hpcugent/vsc-base -# -# vsc-base is free software: you can redistribute it and/or modify -# it under the terms of the GNU Library General Public License as -# published by the Free Software Foundation, either version 2 of -# the License, or (at your option) any later version. -# -# vsc-base is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU Library General Public License -# along with vsc-base. If not, see . -# - -""" -A class that can be used to generated options to python scripts in a general way. - -@author: Stijn De Weirdt (Ghent University) -@author: Jens Timmerman (Ghent University) -""" - -import ConfigParser -import copy -import inspect -import operator -import os -import re -import StringIO -import sys -import textwrap -from optparse import OptionParser, OptionGroup, Option, Values, HelpFormatter -from optparse import BadOptionError, SUPPRESS_USAGE, NO_DEFAULT, OptionValueError -from optparse import SUPPRESS_HELP as nohelp # supported in optparse of python v2.4 -from optparse import _ as _gettext # this is gettext normally -from vsc.utils.dateandtime import date_parser, datetime_parser -from vsc.utils.fancylogger import getLogger, setLogLevel -from vsc.utils.missing import shell_quote, nub -from vsc.utils.optcomplete import autocomplete, CompleterOption - -def set_columns(cols=None): - """Set os.environ COLUMNS variable - - only if it is not set already - """ - if 'COLUMNS' in os.environ: - # do nothing - return - - if cols is None: - stty = '/usr/bin/stty' - if os.path.exists(stty): - try: - cols = int(os.popen('%s size 2>/dev/null' % stty).read().strip().split(' ')[1]) - except: - # do nothing - pass - - if cols is not None: - os.environ['COLUMNS'] = "%s" % cols - - -def check_str_list_tuple(option, opt, value): - """ - check function for strlist and strtuple type - assumes value is comma-separated list - returns list or tuple of strings - """ - split = value.split(',') - if option.type == 'strlist': - return split - elif option.type == 'strtuple': - return tuple(split) - else: - err = _("check_strlist_strtuple: unsupported type %s" % option.type) - raise OptionValueError(err) - - -class ExtOption(CompleterOption): - """Extended options class - - enable/disable support - - Actions: - - shorthelp : hook for shortend help messages - - confighelp : hook for configfile-style help messages - - store_debuglog : turns on fancylogger debugloglevel - - also: 'store_infolog', 'store_warninglog' - - extend : extend default list (or create new one if is None) - - date : convert into datetime.date - - datetime : convert into datetime.datetime - - regex: compile str in regexp - - store_or_None - - set default to None if no option passed, - - set to default if option without value passed, - - set to value if option with value passed - """ - EXTEND_SEPARATOR = ',' - - ENABLE = 'enable' # do nothing - DISABLE = 'disable' # inverse action - - EXTOPTION_EXTRA_OPTIONS = ('extend', 'date', 'datetime', 'regex',) - EXTOPTION_STORE_OR = ('store_or_None',) # callback type - EXTOPTION_LOG = ('store_debuglog', 'store_infolog', 'store_warninglog',) - EXTOPTION_HELP = ('shorthelp', 'confighelp',) - - ACTIONS = Option.ACTIONS + EXTOPTION_EXTRA_OPTIONS + EXTOPTION_STORE_OR + EXTOPTION_LOG + EXTOPTION_HELP - STORE_ACTIONS = Option.STORE_ACTIONS + EXTOPTION_EXTRA_OPTIONS + EXTOPTION_LOG + ('store_or_None',) - TYPED_ACTIONS = Option.TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS + EXTOPTION_STORE_OR - ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS - - TYPE_CHECKER = dict([('strlist', check_str_list_tuple), - ('strtuple', check_str_list_tuple), - ] + Option.TYPE_CHECKER.items()) - TYPES = tuple(['strlist', 'strtuple'] + list(Option.TYPES)) - BOOLEAN_ACTIONS = ('store_true', 'store_false',) + EXTOPTION_LOG - - def _set_attrs(self, attrs): - """overwrite _set_attrs to allow store_or callbacks""" - Option._set_attrs(self, attrs) - if self.action in self.EXTOPTION_STORE_OR: - setattr(self, 'store_or', self.action) - - def store_or(option, opt_str, value, parser, *args, **kwargs): - """Callback for supporting options with optional values.""" - # see http://stackoverflow.com/questions/1229146/parsing-empty-options-in-python - # ugly code, optparse is crap - if parser.rargs and not parser.rargs[0].startswith('-'): - val = parser.rargs[0] - parser.rargs.pop(0) - else: - val = kwargs.get('orig_default', None) - - setattr(parser.values, option.dest, val) - - # without the following, --x=y doesn't work; only --x y - self.nargs = 0 # allow 0 args, will also use 0 args - if self.type is None: - # set to not None, for takes_value to return True - self.type = 'string' - - self.callback = store_or - self.callback_kwargs = {'orig_default': copy.deepcopy(self.default), - } - self.action = 'callback' # act as callback - if self.store_or == 'store_or_None': - self.default = None - else: - raise ValueError("_set_attrs: unknown store_or %s" % self.store_or) - - def take_action(self, action, dest, opt, value, values, parser): - """Extended take_action""" - orig_action = action # keep copy - - if action == 'shorthelp': - parser.print_shorthelp() - parser.exit() - elif action == 'confighelp': - parser.print_confighelp() - parser.exit() - elif action in ('store_true', 'store_false',) + self.EXTOPTION_LOG: - if action in self.EXTOPTION_LOG: - action = 'store_true' - - if opt.startswith("--%s-" % self.ENABLE): - # keep action - pass - elif opt.startswith("--%s-" % self.DISABLE): - # reverse action - if action in ('store_true',) + self.EXTOPTION_LOG: - action = 'store_false' - elif action in ('store_false',): - action = 'store_true' - - if orig_action in self.EXTOPTION_LOG and action == 'store_true': - setLogLevel(orig_action.split('_')[1][:-3].upper()) - - Option.take_action(self, action, dest, opt, value, values, parser) - elif action in self.EXTOPTION_EXTRA_OPTIONS: - if action == "extend": - # comma separated list convert in list - lvalue = value.split(self.EXTEND_SEPARATOR) - values.ensure_value(dest, []).extend(lvalue) - elif action == "date": - lvalue = date_parser(value) - setattr(values, dest, lvalue) - elif action == "datetime": - lvalue = datetime_parser(value) - setattr(values, dest, lvalue) - elif action == "regex": - lvalue = re.compile(r'' + value) - setattr(values, dest, lvalue) - else: - raise(Exception("Unknown extended option action %s (known: %s)" % - (action, self.EXTOPTION_EXTRA_OPTIONS))) - else: - Option.take_action(self, action, dest, opt, value, values, parser) - - # set flag to mark as passed by action (ie not by default) - # - distinguish from setting default value through option - if hasattr(values, '_action_taken'): - values._action_taken[dest] = True - - -class PassThroughOptionParser(OptionParser): - """ - "Pass-through" option parsing -- an OptionParser that ignores - unknown options and lets them pile up in the leftover argument - list. Useful for programs that pass unknown options through - to a sub-program. - from http://www.koders.com/python/fid9DFF5006AF4F52BA6483C4F654E26E6A20DBC73C.aspx?s=add+one#L27 - """ - def __init__(self): - OptionParser.__init__(self, add_help_option=False, usage=SUPPRESS_USAGE) - - def _process_long_opt(self, rargs, values): - """Extend optparse code with catch of unknown long options error""" - try: - OptionParser._process_long_opt(self, rargs, values) - except BadOptionError, err: - self.largs.append(err.opt_str) - - def _process_short_opts(self, rargs, values): - """Process the short options, pass unknown to largs""" - # implementation from recent optparser - arg = rargs.pop(0) - stop = False - i = 1 - for ch in arg[1:]: - opt = "-" + ch - option = self._short_opt.get(opt) - i += 1 # we have consumed a character - - if not option: - # don't fail here, just append to largs - # raise BadOptionError(opt) - self.largs.append(opt) - return - if option.takes_value(): - # Any characters left in arg? Pretend they're the - # next arg, and stop consuming characters of arg. - if i < len(arg): - rargs.insert(0, arg[i:]) - stop = True - - nargs = option.nargs - if len(rargs) < nargs: - if nargs == 1: - self.error(_("%s option requires an argument") % opt) - else: - self.error(_("%s option requires %d arguments") - % (opt, nargs)) - elif nargs == 1: - value = rargs.pop(0) - else: - value = tuple(rargs[0:nargs]) - del rargs[0:nargs] - - else: # option doesn't take a value - value = None - - option.process(opt, value, values, self) - - if stop: - break - - -class ExtOptionGroup(OptionGroup): - """An OptionGroup with support for configfile section names""" - RESERVED_SECTIONS = ['DEFAULT'] - NO_SECTION = ('NO', 'SECTION') - - def __init__(self, *args, **kwargs): - self.log = getLogger(self.__class__.__name__) - section_name = kwargs.pop('section_name', None) - if section_name in self.RESERVED_SECTIONS: - self.log.raiseException('Cannot use reserved name %s for section name.' % section_name) - - OptionGroup.__init__(self, *args, **kwargs) - self.section_name = section_name - self.section_options = [] - - def add_option(self, *args, **kwargs): - """Extract configfile section info""" - option = OptionGroup.add_option(self, *args, **kwargs) - self.section_options.append(option) - - return option - - -class ExtOptionParser(OptionParser): - """ - Make an option parser that limits the C{-h} / C{--shorthelp} to short opts only, - C{-H} / C{--help} for all options. - - Pass options through environment. Like: - - - C{export PROGNAME_SOMEOPTION = value} will generate {--someoption=value} - - C{export PROGNAME_OTHEROPTION = 1} will generate {--otheroption} - - C{export PROGNAME_OTHEROPTION = 0} (or no or false) won't do anything - - distinction is made based on option.action in TYPED_ACTIONS allow - C{--enable-} / C{--disable-} (using eg ExtOption option_class) - """ - shorthelp = ('h', "--shorthelp",) - longhelp = ('H', "--help",) - - VALUES_CLASS = Values - DESCRIPTION_DOCSTRING = False - - def __init__(self, *args, **kwargs): - self.log = getLogger(self.__class__.__name__) - self.help_to_string = kwargs.pop('help_to_string', None) - self.help_to_file = kwargs.pop('help_to_file', None) - self.envvar_prefix = kwargs.pop('envvar_prefix', None) - - # py2.4 epilog compatibilty with py2.7 / optparse 1.5.3 - self.epilog = kwargs.pop('epilog', None) - - if not 'option_class' in kwargs: - kwargs['option_class'] = ExtOption - OptionParser.__init__(self, *args, **kwargs) - - # redefine formatter for py2.4 compat - if not hasattr(self.formatter, 'format_epilog'): - setattr(self.formatter, 'format_epilog', self.formatter.format_description) - - if self.epilog is None: - self.epilog = [] - - if hasattr(self.option_class, 'ENABLE') and hasattr(self.option_class, 'DISABLE'): - epilogtxt = 'Boolean options support %(disable)s prefix to do the inverse of the action,' - epilogtxt += ' e.g. option --someopt also supports --disable-someopt.' - self.epilog.append(epilogtxt % {'disable': self.option_class.DISABLE}) - - def set_description_docstring(self): - """Try to find the main docstring and add it if description is not None""" - stack = inspect.stack()[-1] - try: - docstr = stack[0].f_globals.get('__doc__', None) - except: - self.log.debug("set_description_docstring: no docstring found in latest stack globals") - docstr = None - - if docstr is not None: - indent = " " - # kwargs and ** magic to deal with width - kwargs = { - 'initial_indent': indent * 2, - 'subsequent_indent': indent * 2, - 'replace_whitespace': False, - } - width = os.environ.get('COLUMNS', None) - if width is not None: - # default textwrap width - try: - kwargs['width'] = int(width) - except: - pass - - # deal with newlines in docstring - final_docstr = [''] - for line in str(docstr).strip("\n ").split("\n"): - final_docstr.append(textwrap.fill(line, **kwargs)) - final_docstr.append('') - - return "\n".join(final_docstr) - - def format_description(self, formatter): - """Extend to allow docstring as description""" - description = '' - if self.description == 'NONE_AND_NOT_NONE': - if self.DESCRIPTION_DOCSTRING: - description = self.set_description_docstring() - elif self.description: - description = formatter.format_description(self.get_description()) - - return str(description) - - def set_usage(self, usage): - """Return usage and set try to set autogenerated description.""" - usage = OptionParser.set_usage(self, usage) - - if self.description is None: - self.description = 'NONE_AND_NOT_NONE' - - return usage - - def get_default_values(self): - """Introduce the ExtValues class with class constant - - make it dynamic, otherwise the class constant is shared between multiple instances - - class constant is used to avoid _taken_action as option in the __dict__ - """ - values = OptionParser.get_default_values(self) - - class ExtValues(self.VALUES_CLASS): - _action_taken = {} - - newvalues = ExtValues() - newvalues.__dict__ = values.__dict__.copy() - return newvalues - - def format_help(self, formatter=None): - """For py2.4 compatibility reasons (missing epilog). This is the py2.7 / optparse 1.5.3 code""" - if formatter is None: - formatter = self.formatter - result = [] - if self.usage: - result.append(self.get_usage() + "\n") - if self.description: - result.append(self.format_description(formatter) + "\n") - result.append(self.format_option_help(formatter)) - result.append(self.format_epilog(formatter)) - return "".join(result) - - def format_epilog(self, formatter): - """Allow multiple epilog parts""" - res = [] - if not isinstance(self.epilog, (list, tuple,)): - self.epilog = [self.epilog] - for epi in self.epilog: - res.append(formatter.format_epilog(epi)) - return "".join(res) - - def print_shorthelp(self, fh=None): - """Print a shortened help (no longopts)""" - for opt in self._get_all_options(): - if opt._short_opts is None or len([x for x in opt._short_opts if len(x) > 0]) == 0: - opt.help = nohelp - opt._long_opts = [] # remove all long_opts - - removeoptgrp = [] - for optgrp in self.option_groups: - # remove all option groups that have only nohelp options - if reduce(operator.and_, [opt.help == nohelp for opt in optgrp.option_list]): - removeoptgrp.append(optgrp) - for optgrp in removeoptgrp: - self.option_groups.remove(optgrp) - - self.print_help(fh) - - def print_help(self, fh=None): - """Intercept print to file to print to string and remove the ENABLE/DISABLE options from help""" - if self.help_to_string: - self.help_to_file = StringIO.StringIO() - if fh is None: - fh = self.help_to_file - - if hasattr(self.option_class, 'ENABLE') and hasattr(self.option_class, 'DISABLE'): - def _is_enable_disable(x): - """Does the option start with ENABLE/DISABLE""" - _e = x.startswith("--%s-" % self.option_class.ENABLE) - _d = x.startswith("--%s-" % self.option_class.DISABLE) - return _e or _d - for opt in self._get_all_options(): - # remove all long_opts with ENABLE/DISABLE naming - opt._long_opts = [x for x in opt._long_opts if not _is_enable_disable(x)] - - OptionParser.print_help(self, fh) - - def print_confighelp(self, fh=None): - """Print help as a configfile.""" - - # walk through all optiongroups - # append where necessary, keep track of sections - all_groups = {} - sections = [] - for gr in self.option_groups: - section = gr.section_name - if not (section is None or section == ExtOptionGroup.NO_SECTION): - if not section in sections: - sections.append(section) - ag = all_groups.setdefault(section, []) - ag.extend(gr.section_options) - - # set MAIN section first if exists - main_idx = sections.index('MAIN') - if main_idx > 0: # not needed if it main_idx == 0 - sections.remove('MAIN') - sections.insert(0, 'MAIN') - - option_template = "# %(help)s\n#%(option)s=\n" - txt = '' - for section in sections: - txt += "[%s]\n" % section - for option in all_groups[section]: - data = { - 'help': option.help, - 'option': option.get_opt_string().lstrip('-'), - } - txt += option_template % data - txt += "\n" - - # overwrite the format_help to be able to use the the regular print_help - def format_help(*args, **kwargs): - return txt - self.format_help = format_help - self.print_help(fh) - - def _add_help_option(self): - """Add shorthelp and longhelp""" - self.add_option("-%s" % self.shorthelp[0], - self.shorthelp[1], # *self.shorthelp[1:], syntax error in Python 2.4 - action="shorthelp", - help=_gettext("show short help message and exit")) - self.add_option("-%s" % self.longhelp[0], - self.longhelp[1], # *self.longhelp[1:], syntax error in Python 2.4 - action="help", - help=_gettext("show full help message and exit")) - self.add_option("--confighelp", - action="confighelp", - help=_gettext("show help as annotated configfile")) - - def _get_args(self, args): - """Prepend the options set through the environment""" - regular_args = OptionParser._get_args(self, args) - env_args = self.get_env_options() - return env_args + regular_args # prepend the environment options as longopts - - def get_env_options_prefix(self): - """Return the prefix to use for options passed through the environment""" - # sys.argv[0] or the prog= argument of the optionparser, strip possible extension - if self.envvar_prefix is None: - self.envvar_prefix = self.get_prog_name().rsplit('.', 1)[0].upper() - return self.envvar_prefix - - def get_env_options(self): - """Retrieve options from the environment: prefix_longopt.upper()""" - env_long_opts = [] - if self.envvar_prefix is None: - self.get_env_options_prefix() - - epilogprefixtxt = "All long option names can be passed as environment variables. " - epilogprefixtxt += "Variable name is %(prefix)s_ " - epilogprefixtxt += "eg. --some-opt is same as setting %(prefix)s_SOME_OPT in the environment." - self.epilog.append(epilogprefixtxt % {'prefix': self.envvar_prefix}) - - for opt in self._get_all_options(): - if opt._long_opts is None: - continue - for lo in opt._long_opts: - if len(lo) == 0: - continue - env_opt_name = "%s_%s" % (self.envvar_prefix, lo.lstrip('-').replace('-', '_').upper()) - val = os.environ.get(env_opt_name, None) - if not val is None: - if opt.action in opt.TYPED_ACTIONS: # not all typed actions are mandatory, but let's assume so - env_long_opts.append("%s=%s" % (lo, val)) - else: - # interpretation of values: 0/no/false means: don't set it - if not ("%s" % val).lower() in ("0", "no", "false",): - env_long_opts.append("%s" % lo) - else: - self.log.debug("Environment variable %s is not set" % env_opt_name) - - self.log.debug("Environment variable options with prefix %s: %s" % (self.envvar_prefix, env_long_opts)) - return env_long_opts - - def get_option_by_long_name(self, name): - """Return the option matching the long option name""" - for opt in self._get_all_options(): - if opt._long_opts is None: - continue - for lo in opt._long_opts: - if len(lo) == 0: - continue - dest = lo.lstrip('-') - if name == dest: - return opt - - return None - - -class GeneralOption(object): - """ - 'Used-to-be simple' wrapper class for option parsing - - Options with go_ prefix are for this class, the remainder is passed to the parser - - go_args : use these instead of of sys.argv[1:] - - go_columns : specify column width (in columns) - - go_useconfigfiles : use configfiles or not (default set by CONFIGFILES_USE) - if True, an option --configfiles will be added - - go_configfiles : list of configfiles to parse. Uses ConfigParser.read; last file wins - - go_loggername : name of logger, default classname - - go_mainbeforedefault : set the main options before the default ones - - go_autocompleter : dict with named options to pass to the autocomplete call (eg arg_completer) - if is None: disable autocompletion; default is {} (ie no extra args passed) - - Sections starting with the string 'raw_' in the sectionname will be parsed as raw sections, - meaning there will be no interpolation of the strings. This comes in handy if you want to configure strings - with templates in them. - - Options process order (last one wins) - 0. default defined with option - 1. value in (last) configfile (last configfile wins) - 2. options parsed by option parser - In case the ExtOptionParser is used - 0. value set through environment variable - 1. value set through commandline option - """ - OPTIONNAME_PREFIX_SEPARATOR = '-' - - DEBUG_OPTIONS_BUILD = False # enable debug mode when building the options ? - - USAGE = None - ALLOPTSMANDATORY = True - PARSER = ExtOptionParser - INTERSPERSED = True # mix args with options - - CONFIGFILES_USE = True - CONFIGFILES_RAISE_MISSING = False - CONFIGFILES_INIT = [] # initial list of defaults, overwritten by go_configfiles options - CONFIGFILES_IGNORE = [] - CONFIGFILES_MAIN_SECTION = 'MAIN' # sectionname that contains the non-grouped/non-prefixed options - CONFIGFILE_PARSER = ConfigParser.ConfigParser - - METAVAR_DEFAULT = True # generate a default metavar - METAVAR_MAP = None # metvar, list of longopts map - - OPTIONGROUP_SORTED_OPTIONS = True - - PROCESSED_OPTIONS_PROPERTIES = ['type', 'default', 'action', 'opt_name', 'prefix', 'section_name'] - - VERSION = None # set the version (will add --version) - - DEFAULT_LOGLEVEL = None - DEFAULT_CONFIGFILES = None - DEFAULT_IGNORECONFIGFILES = None - - def __init__(self, **kwargs): - go_args = kwargs.pop('go_args', None) - self.no_system_exit = kwargs.pop('go_nosystemexit', None) # unit test option - self.use_configfiles = kwargs.pop('go_useconfigfiles', self.CONFIGFILES_USE) # use or ignore config files - self.configfiles = kwargs.pop('go_configfiles', self.CONFIGFILES_INIT) # configfiles to parse - prefixloggername = kwargs.pop('go_prefixloggername', False) # name of logger is same as envvar prefix - mainbeforedefault = kwargs.pop('go_mainbeforedefault', False) # Set the main options before the default ones - autocompleter = kwargs.pop('go_autocompleter', {}) # Pass these options to the autocomplete call - - set_columns(kwargs.pop('go_columns', None)) - - kwargs.update({ - 'option_class': ExtOption, - 'usage': kwargs.get('usage', self.USAGE), - 'version': self.VERSION, - }) - self.parser = self.PARSER(**kwargs) - self.parser.allow_interspersed_args = self.INTERSPERSED - - self.configfile_parser = self.CONFIGFILE_PARSER() - self.configfile_remainder = {} - - loggername = self.__class__.__name__ - if prefixloggername: - prefix = self.parser.get_env_options_prefix() - if prefix is not None and len(prefix) > 0: - loggername = prefix.replace('.', '_') # . indicate hierarchy in logging land - - self.log = getLogger(loggername) - self.options = None - self.args = None - - self.autocompleter = autocompleter - - self.auto_prefix = None - self.auto_section_name = None - - self.processed_options = {} - - self.config_prefix_sectionnames_map = {} - - self.set_go_debug() - - if mainbeforedefault: - self.main_options() - self._default_options() - else: - self._default_options() - self.main_options() - - self.parseoptions(options_list=go_args) - - if not self.options is None: - # None for eg usage/help - self.parseconfigfiles() - - self._set_default_loglevel() - - self.postprocess() - - self.validate() - - def set_go_debug(self): - """Check if debug options are on and then set fancylogger to debug. - This is not the default way to set debug, it enables debug logging - in an earlier stage to debug generaloption itself. - """ - if self.options is None: - if self.DEBUG_OPTIONS_BUILD: - setLogLevel('DEBUG') - - def _default_options(self): - """Generate default options: debug/log and configfile""" - self._make_debug_options() - self._make_configfiles_options() - - def _make_debug_options(self): - """Add debug/logging options: debug and info""" - self._logopts = { - 'debug': ("Enable debug log mode", None, "store_debuglog", False, 'd'), - 'info': ("Enable info log mode", None, "store_infolog", False), - 'quiet': ("Enable info quiet/warning mode", None, "store_warninglog", False), - } - - descr = ['Debug and logging options', ''] - self.log.debug("Add debug and logging options descr %s opts %s (no prefix)" % (descr, self._logopts)) - self.add_group_parser(self._logopts, descr, prefix=None) - - def _set_default_loglevel(self): - """Set the default loglevel if no logging options are set""" - loglevel_set = sum([getattr(self.options, name, False) for name in self._logopts.keys()]) - if not loglevel_set and self.DEFAULT_LOGLEVEL is not None: - setLogLevel(self.DEFAULT_LOGLEVEL) - - def _make_configfiles_options(self): - """Add configfiles option""" - opts = { - 'configfiles': ("Parse (additional) configfiles", None, "extend", self.DEFAULT_CONFIGFILES), - 'ignoreconfigfiles': ("Ignore configfiles", None, "extend", self.DEFAULT_IGNORECONFIGFILES), - } - descr = ['Configfile options', ''] - self.log.debug("Add configfiles options descr %s opts %s (no prefix)" % (descr, opts)) - self.add_group_parser(opts, descr, prefix=None, section_name=ExtOptionGroup.NO_SECTION) - - def main_options(self): - """Create the main options automatically""" - # make_init is deprecated - if hasattr(self, 'make_init'): - self.log.debug('main_options: make_init is deprecated. Rename function to main_options.') - getattr(self, 'make_init')() - else: - # function names which end with _options and do not start with main or _ - reg_main_options = re.compile("^(?!_|main).*_options$") - names = [x for x in dir(self) if reg_main_options.search(x)] - if len(names) == 0: - self.log.error("main_options: no options functions implemented") - else: - for name in names: - fn = getattr(self, name) - if callable(fn): # inspect.isfunction fails beacuse this is a boundmethod - self.auto_section_name = '_'.join(name.split('_')[:-1]) - self.log.debug('main_options: adding options from %s (auto_section_name %s)' % - (name, self.auto_section_name)) - fn() - self.auto_section_name = None # reset it - - def make_option_metavar(self, longopt, details): - """Generate the metavar for option longopt - @type longopt: str - @type details: tuple - """ - if self.METAVAR_MAP is not None: - for metavar, longopts in self.METAVAR_MAP.items(): - if longopt in longopts: - return metavar - - if self.METAVAR_DEFAULT: - return longopt.upper() - - def add_group_parser(self, opt_dict, description, prefix=None, otherdefaults=None, section_name=None): - """Make a group parser from a dict - - - @type opt_dict: dict - @type description: a 2 element list (short and long description) - @section_name: str, the name of the section group in the config file. - - @param opt_dict: options, with the form C{"long_opt" : value}. - Value is a C{tuple} containing - C{(help,type,action,default(,optional string=short option; list/tuple=choices; dict=add_option kwargs))} - - help message passed through opt_dict will be extended with type and default - - If section_name is None, prefix will be used. If prefix is None or '', 'DEFAULT' is used. - - """ - if opt_dict is None: - # skip opt_dict None - # if opt_dict is empty dict {}, the eg the descritionis added to the help - self.log.debug("Skipping opt_dict %s with description %s prefix %s" % - (opt_dict, description, prefix)) - return - - if otherdefaults is None: - otherdefaults = {} - - self.log.debug("add_group_parser: passed prefix %s section_name %s" % (prefix, section_name)) - self.log.debug("add_group_parser: auto_prefix %s auto_section_name %s" % - (self.auto_prefix, self.auto_section_name)) - - if prefix is None: - if self.auto_prefix is None: - prefix = '' - else: - prefix = self.auto_prefix - - if section_name is None: - if prefix is not None and len(prefix) > 0 and not (prefix == self.auto_prefix): - section_name = prefix - elif self.auto_section_name is not None and len(self.auto_section_name) > 0: - section_name = self.auto_section_name - else: - section_name = self.CONFIGFILES_MAIN_SECTION - - self.log.debug("add_group_parser: set prefix %s section_name %s" % (prefix, section_name)) - - # add the section name to the help output - if section_name is None or section_name == ExtOptionGroup.NO_SECTION: - section_help = '' - else: - section_help = " (configfile section %s)" % (section_name) - - if description[1]: - short_description = description[0] - long_description = "%s%s" % (description[1], section_help) - else: - short_description = "%s%s" % (description[0], section_help) - long_description = description[1] - - opt_grp = ExtOptionGroup(self.parser, short_description, long_description, section_name=section_name) - keys = opt_dict.keys() - if self.OPTIONGROUP_SORTED_OPTIONS: - keys.sort() # alphabetical - for key in keys: - completer = None - - details = opt_dict[key] - - hlp = details[0] - typ = details[1] - action = details[2] - default = details[3] - # easy override default with otherdefault - if key in otherdefaults: - default = otherdefaults.get(key) - - extra_help = [] - if action in ("extend",) or typ in ('strlist', 'strtuple',): - extra_help.append("type comma-separated list") - elif typ is not None: - extra_help.append("type %s" % typ) - - if default is not None: - if len(str(default)) == 0: - extra_help.append("def ''") # empty string - elif action in ("extend",) or typ in ('strlist', 'strtuple',): - extra_help.append("def %s" % ','.join(default)) - else: - extra_help.append("def %s" % default) - - if len(extra_help) > 0: - hlp += " (%s)" % ("; ".join(extra_help)) - - opt_name, opt_dest = self.make_options_option_name_and_destination(prefix, key) - - args = ["--%s" % opt_name] - - # this has to match PROCESSED_OPTIONS_PROPERTIES - self.processed_options[opt_dest] = [typ, default, action, opt_name, prefix, section_name] # add longopt - if not len(self.processed_options[opt_dest]) == len(self.PROCESSED_OPTIONS_PROPERTIES): - self.log.raiseException("PROCESSED_OPTIONS_PROPERTIES length mismatch") - - nameds = { - 'dest': opt_dest, - 'action': action, - } - metavar = self.make_option_metavar(key, details) - if metavar is not None: - nameds['metavar'] = metavar - - if default is not None: - nameds['default'] = default - - if typ: - nameds['type'] = typ - - passed_kwargs = {} - if len(details) >= 5: - for extra_detail in details[4:]: - if isinstance(extra_detail, (list, tuple,)): - # choices - nameds['choices'] = ["%s" % x for x in extra_detail] # force to strings - hlp += ' (choices: %s)' % ', '.join(nameds['choices']) - elif isinstance(extra_detail, basestring) and len(extra_detail) == 1: - args.insert(0, "-%s" % extra_detail) - elif isinstance(extra_detail, (dict,)): - # extract any optcomplete completer hints - completer = extra_detail.pop('completer', None) - - # add remainder - passed_kwargs.update(extra_detail) - else: - self.log.raiseException("add_group_parser: unknown extra detail %s" % extra_detail) - - # add help - nameds['help'] = hlp - - if hasattr(self.parser.option_class, 'ENABLE') and hasattr(self.parser.option_class, 'DISABLE'): - if action in self.parser.option_class.BOOLEAN_ACTIONS: - args.append("--%s-%s" % (self.parser.option_class.ENABLE, opt_name)) - args.append("--%s-%s" % (self.parser.option_class.DISABLE, opt_name)) - - # force passed_kwargs as final nameds - nameds.update(passed_kwargs) - opt = opt_grp.add_option(*args, **nameds) - - if completer is not None: - opt.completer = completer - - self.parser.add_option_group(opt_grp) - - # map between prefix and sectionnames - prefix_section_names = self.config_prefix_sectionnames_map.setdefault(prefix, []) - if not section_name in prefix_section_names: - prefix_section_names.append(section_name) - self.log.debug("Added prefix %s to list of sectionnames for %s" % (prefix, section_name)) - - def default_parseoptions(self): - """Return default options""" - return sys.argv[1:] - - def autocomplete(self): - """Set the autocompletion magic via optcomplete""" - # very basic for now, no special options - if self.autocompleter is None: - self.log.debug('self.autocompleter is None, disabling autocompleter') - else: - self.log.debug('setting autocomplete with args %s' % self.autocompleter) - autocomplete(self.parser, **self.autocompleter) - - def parseoptions(self, options_list=None): - """Parse the options""" - if options_list is None: - options_list = self.default_parseoptions() - - self.autocomplete() - - try: - (self.options, self.args) = self.parser.parse_args(options_list) - except SystemExit, err: - if self.no_system_exit: - try: - msg = err.message - except: - # py2.4 - msg = '_nomessage_' - self.log.debug("parseoptions: no_system_exit set after parse_args err %s code %s" % - (msg, err.code)) - return - else: - sys.exit(err.code) - - # args should be empty, since everything is optional - if len(self.args) > 1: - self.log.debug("Found remaining args %s" % self.args) - if self.ALLOPTSMANDATORY: - self.parser.error("Invalid arguments args %s" % self.args) - - self.log.debug("Found options %s args %s" % (self.options, self.args)) - - def parseconfigfiles(self): - """Parse configfiles""" - if not self.use_configfiles: - self.log.debug('parseconfigfiles: use_configfiles False, skipping configfiles') - return - - if self.configfiles is None: - self.configfiles = [] - - self.log.debug("parseconfigfiles: configfiles initially set %s" % self.configfiles) - - option_configfiles = self.options.__dict__.get('configfiles', []) # empty list, will win so no defaults - option_ignoreconfigfiles = self.options.__dict__.get('ignoreconfigfiles', self.CONFIGFILES_IGNORE) - - self.log.debug("parseconfigfiles: configfiles set through commandline %s" % option_configfiles) - self.log.debug("parseconfigfiles: ignoreconfigfiles set through commandline %s" % option_ignoreconfigfiles) - if option_configfiles is not None: - self.configfiles.extend(option_configfiles) - - if option_ignoreconfigfiles is None: - option_ignoreconfigfiles = [] - - # Configparser fails on broken config files - # - if config file doesn't exist, it's no issue - configfiles = [] - for fn in self.configfiles: - if not os.path.isfile(fn): - if self.CONFIGFILES_RAISE_MISSING: - self.log.raiseException("parseconfigfiles: configfile %s not found." % fn) - else: - self.log.debug("parseconfigfiles: configfile %s not found, will be skipped" % fn) - - if fn in option_ignoreconfigfiles: - self.log.debug("parseconfigfiles: configfile %s will be ignored %s" % fn) - else: - configfiles.append(fn) - - try: - parsed_files = self.configfile_parser.read(configfiles) - except: - self.log.raiseException("parseconfigfiles: problem during read") - - self.log.debug("parseconfigfiles: following files were parsed %s" % parsed_files) - self.log.debug("parseconfigfiles: following files were NOT parsed %s" % - [x for x in configfiles if not x in parsed_files]) - self.log.debug("parseconfigfiles: sections (w/o DEFAULT) %s" % self.configfile_parser.sections()) - - # walk through list of section names - # - look for options set though config files - configfile_values = {} - configfile_options_default = {} - configfile_cmdline = [] - configfile_cmdline_dest = [] # expected destinations - - # won't parse - cfg_sections = self.config_prefix_sectionnames_map.values() # without DEFAULT - for section in cfg_sections: - if not section in self.config_prefix_sectionnames_map.values(): - self.log.warning("parseconfigfiles: found section %s, won't be parsed" % section) - continue - - # add any non-option related configfile data to configfile_remainder dict - cfg_sections_flat = [name for section_names in cfg_sections for name in section_names] - for section in self.configfile_parser.sections(): - if section not in cfg_sections_flat: - self.log.debug("parseconfigfiles: found section %s, adding to remainder" % section) - remainder = self.configfile_remainder.setdefault(section, {}) - # parse te remaining options, sections starting with 'raw_' as their name will be considered raw sections - - for opt, val in self.configfile_parser.items(section, raw=(section.startswith('raw_'))): - remainder[opt] = val - - # options are passed to the commandline option parser - for prefix, section_names in self.config_prefix_sectionnames_map.items(): - for section in section_names: - # default section is treated separate in ConfigParser - if not self.configfile_parser.has_section(section): - self.log.debug('parseconfigfiles: no section %s' % str(section)) - continue - elif section == ExtOptionGroup.NO_SECTION: - self.log.debug('parseconfigfiles: ignoring NO_SECTION %s' % str(section)) - continue - elif section.lower() == 'default': - self.log.debug('parseconfigfiles: ignoring default section %s' % section) - continue - - for opt, val in self.configfile_parser.items(section): - self.log.debug('parseconfigfiles: section %s option %s val %s' % (section, opt, val)) - - opt_name, opt_dest = self.make_options_option_name_and_destination(prefix, opt) - actual_option = self.parser.get_option_by_long_name(opt_name) - if actual_option is None: - self.log.raiseException('parseconfigfiles: no option corresponding with dest %s' % - opt_dest) - - configfile_options_default[opt_dest] = actual_option.default - - if actual_option.action in ExtOption.BOOLEAN_ACTIONS: - try: - newval = self.configfile_parser.getboolean(section, opt) - self.log.debug(('parseconfigfiles: getboolean for option %s value %s ' - 'in section %s returned %s') % (opt, val, section, newval)) - except: - self.log.raiseException(('parseconfigfiles: failed to getboolean for option %s value %s ' - 'in section %s') % (opt, val, section)) - if hasattr(self.parser.option_class, 'ENABLE') and hasattr(self.parser.option_class, 'DISABLE'): - if newval: - cmd_template = "--enable-%s" - else: - cmd_template = "--disable-%s" - configfile_cmdline_dest.append(opt_dest) - configfile_cmdline.append(cmd_template % opt_name) - else: - self.log.debug(("parseconfigfiles: no enable/disable, not trying to set boolean-valued " - "option %s via cmdline, just setting value to %s" % (opt_name, newval))) - configfile_values[opt_dest] = newval - else: - configfile_cmdline_dest.append(opt_dest) - configfile_cmdline.append("--%s" % opt_name) - configfile_cmdline.append(val) - - # reparse - self.log.debug('parseconfigfiles: going to parse options through cmdline %s' % configfile_cmdline) - try: - (parsed_configfile_options, parsed_configfile_args) = self.parser.parse_args(configfile_cmdline) - except: - self.log.raiseException('parseconfigfiles: failed to parse options through cmdline %s' % - configfile_cmdline) - - if len(parsed_configfile_args) > 0: - self.log.raiseException('parseconfigfiles: not all options were parsed: %s' % parsed_configfile_args) - - for opt_dest in configfile_cmdline_dest: - try: - configfile_values[opt_dest] = getattr(parsed_configfile_options, opt_dest) - except: - self.log.raiseException('parseconfigfiles: failed to retrieve dest %s from parsed_configfile_options' % - opt_dest) - - self.log.debug('parseconfigfiles: parsed values from configfiles: %s' % configfile_values) - - for opt_dest, val in configfile_values.items(): - set_opt = False - if not hasattr(self.options, opt_dest): - self.log.debug('parseconfigfiles: adding new option %s with value %s' % (opt_dest, val)) - set_opt = True - else: - if hasattr(self.options, '_action_taken') and self.options._action_taken.get(opt_dest, None): - # value set through take_action. do not modify by configfile - self.log.debug('parseconfigfiles: option %s already found in _action_taken' % (opt_dest)) - else: - self.log.debug('parseconfigfiles: option %s not found in _action_taken, setting to %s' % - (opt_dest, val)) - set_opt = True - if set_opt: - setattr(self.options, opt_dest, val) - if hasattr(self.options, '_action_taken'): - self.options._action_taken[opt_dest] = True - - def make_options_option_name_and_destination(self, prefix, key): - """Make the options option name""" - if prefix == '': - name = key - else: - name = "".join([prefix, self.OPTIONNAME_PREFIX_SEPARATOR, key]) - - # dest : replace '-' with '_' - dest = name.replace('-', '_') - - return name, dest - - def _get_options_by_property(self, prop_type, prop_value): - """Return all options with property type equal to value""" - if not prop_type in self.PROCESSED_OPTIONS_PROPERTIES: - self.log.raiseException('Invalid prop_type %s for PROCESSED_OPTIONS_PROPERTIES %s' % - (prop_type, self.PROCESSED_OPTIONS_PROPERTIES)) - prop_idx = self.PROCESSED_OPTIONS_PROPERTIES.index(prop_type) - # get all options with prop_type - options = {} - for key in [dest for dest, props in self.processed_options.items() if props[prop_idx] == prop_value]: - options[key] = getattr(self.options, key, None) # None? isn't there always a default - - return options - - def get_options_by_prefix(self, prefix): - """Get all options that set with prefix. Return a dict. The keys are stripped of the prefix.""" - offset = 0 - if prefix: - offset = len(prefix) + len(self.OPTIONNAME_PREFIX_SEPARATOR) - - prefix_dict = {} - for dest, value in self._get_options_by_property('prefix', prefix).items(): - new_dest = dest[offset:] - prefix_dict[new_dest] = value - return prefix_dict - - def get_options_by_section(self, section): - """Get all options from section. Return a dict.""" - return self._get_options_by_property('section_name', section) - - def postprocess(self): - """Some additional processing""" - pass - - def validate(self): - """Final step, allows for validating the options and/or args""" - pass - - def dict_by_prefix(self, merge_empty_prefix=False): - """Break the options dict by prefix; return nested dict. - @param merge_empty_prefix : boolean (default False) also (try to) merge the empty - prefix in the root of the dict. If there is a non-prefixed optionname - that matches a prefix, it will be rejected and error will be logged. - """ - subdict = {} - - prefix_idx = self.PROCESSED_OPTIONS_PROPERTIES.index('prefix') - for prefix in nub([props[prefix_idx] for props in self.processed_options.values()]): - subdict[prefix] = self.get_options_by_prefix(prefix) - - if merge_empty_prefix and '' in subdict: - self.log.debug("dict_by_prefix: merge_empty_prefix set") - for opt, val in subdict[''].items(): - if opt in subdict: - self.log.error("dict_by_prefix: non-prefixed option %s conflicts with prefix of same name." % opt) - else: - subdict[opt] = val - - self.log.debug("dict_by_prefix: subdict %s" % subdict) - return subdict - - def generate_cmd_line(self, ignore=None, add_default=None): - """Create the commandline options that would create the current self.options - opt_name is destination - - @param ignore : regex on destination - @param add_default : print value that are equal to default - """ - if ignore is not None: - self.log.debug("generate_cmd_line ignore %s" % ignore) - ignore = re.compile(ignore) - else: - self.log.debug("generate_cmd_line no ignore") - - args = [] - opt_dests = self.options.__dict__.keys() - opt_dests.sort() - - for opt_dest in opt_dests: - opt_value = self.options.__dict__[opt_dest] - # this is the action as parsed by the class, not the actual action set in option - # (eg action store_or_None is shown here as store_or_None, not as callback) - typ = self.processed_options[opt_dest][self.PROCESSED_OPTIONS_PROPERTIES.index('type')] - default = self.processed_options[opt_dest][self.PROCESSED_OPTIONS_PROPERTIES.index('default')] - action = self.processed_options[opt_dest][self.PROCESSED_OPTIONS_PROPERTIES.index('action')] - opt_name = self.processed_options[opt_dest][self.PROCESSED_OPTIONS_PROPERTIES.index('opt_name')] - - if ignore is not None and ignore.search(opt_dest): - self.log.debug("generate_cmd_line adding %s value %s matches ignore. Not adding to args." % - (opt_name, opt_value)) - continue - - if opt_value == default: - # do nothing - # except for store_or_None and friends - msg = '' - if not (add_default or action in ('store_or_None',)): - msg = ' Not adding to args.' - self.log.debug("generate_cmd_line adding %s value %s default found.%s" % - (opt_name, opt_value, msg)) - if not (add_default or action in ('store_or_None',)): - continue - - if opt_value is None: - # do nothing - self.log.debug("generate_cmd_line adding %s value %s. None found. not adding to args." % - (opt_name, opt_value)) - continue - - if action in ('store_or_None',): - if opt_value == default: - self.log.debug("generate_cmd_line %s adding %s (value is default value %s)" % - (action, opt_name, opt_value)) - args.append("--%s" % (opt_name)) - else: - self.log.debug("generate_cmd_line %s adding %s non-default value %s" % - (action, opt_name, opt_value)) - args.append("--%s=%s" % (opt_name, shell_quote(opt_value))) - elif action in ("store_true", "store_false",) + ExtOption.EXTOPTION_LOG: - # not default! - self.log.debug("generate_cmd_line adding %s value %s. store action found" % - (opt_name, opt_value)) - if (action in ('store_true',) + ExtOption.EXTOPTION_LOG and default is True and opt_value is False) or \ - (action in ('store_false',) and default is False and opt_value is True): - if hasattr(self.parser.option_class, 'ENABLE') and hasattr(self.parser.option_class, 'DISABLE'): - args.append("--%s-%s" % (self.parser.option_class.DISABLE, opt_name)) - else: - self.log.error(("generate_cmd_line: %s : can't set inverse of default %s with action %s " - "with missing ENABLE/DISABLE in option_class") % - (opt_name, default, action)) - else: - if opt_value == default and ((action in ('store_true',) + ExtOption.EXTOPTION_LOG and default is False) - or (action in ('store_false',) and default is True)): - if hasattr(self.parser.option_class, 'ENABLE') and \ - hasattr(self.parser.option_class, 'DISABLE'): - args.append("--%s-%s" % (self.parser.option_class.DISABLE, opt_name)) - else: - self.log.debug(("generate_cmd_line: %s : action %s can only set to inverse of default %s " - "and current value is default. Not adding to args.") % - (opt_name, action, default)) - else: - args.append("--%s" % opt_name) - elif action in ("extend",): - # comma separated - self.log.debug("generate_cmd_line adding %s value %s. extend action, return as comma-separated list" % - (opt_name, opt_value)) - - if default is not None: - # remove these. if default is set, extend extends the default! - for def_el in default: - opt_value.remove(def_el) - - if len(opt_value) == 0: - self.log.debug('generate_cmd_line skipping.') - continue - - args.append("--%s=%s" % (opt_name, shell_quote(",".join(opt_value)))) - elif typ in ('strlist', 'strtuple',): - args.append("--%s=%s" % (opt_name, shell_quote(",".join(opt_value)))) - elif action in ("append",): - # add multiple times - self.log.debug("generate_cmd_line adding %s value %s. append action, return as multiple args" % - (opt_name, opt_value)) - args.extend(["--%s=%s" % (opt_name, shell_quote(v)) for v in opt_value]) - elif action in ("regex",): - self.log.debug("generate_cmd_line adding %s regex pattern %s" % (opt_name, opt_value.pattern)) - args.append("--%s=%s" % (opt_name, shell_quote(opt_value.pattern))) - else: - self.log.debug("generate_cmd_line adding %s value %s" % (opt_name, opt_value)) - args.append("--%s=%s" % (opt_name, shell_quote(opt_value))) - - self.log.debug("commandline args %s" % args) - return args - - -class SimpleOptionParser(ExtOptionParser): - DESCRIPTION_DOCSTRING = True - - -class SimpleOption(GeneralOption): - PARSER = SimpleOptionParser - - def __init__(self, go_dict=None, descr=None, short_groupdescr=None, long_groupdescr=None, config_files=None): - """Initialisation - @param go_dict : General Option option dict - @param short_descr : short description of main options - @param long_descr : longer description of main options - @param config_files : list of configfiles to read options from - - a general options dict has as key the long option name, and is followed by a list/tuple - mandatory are 4 elements : option help, type, action, default - a 5th element is optional and is the short help name (if any) - - the generated help will include the docstring - """ - self.go_dict = go_dict - if short_groupdescr is None: - short_groupdescr = 'Main options' - if long_groupdescr is None: - long_groupdescr = '' - self.descr = [short_groupdescr, long_groupdescr] - - kwargs = { - 'go_prefixloggername': True, - 'go_mainbeforedefault': True, - } - if config_files is not None: - kwargs['go_configfiles'] = config_files - - super(SimpleOption, self).__init__(**kwargs) - - def main_options(self): - if self.go_dict is not None: - prefix = None - self.add_group_parser(self.go_dict, self.descr, prefix=prefix) - - -def simple_option(go_dict=None, descr=None, short_groupdescr=None, long_groupdescr=None, config_files=None): - """A function that returns a single level GeneralOption option parser - - @param go_dict : General Option option dict - @param short_descr : short description of main options - @param long_descr : longer description of main options - @param config_files : list of configfiles to read options from - - a general options dict has as key the long option name, and is followed by a list/tuple - mandatory are 4 elements : option help, type, action, default - a 5th element is optional and is the short help name (if any) - - the generated help will include the docstring - """ - return SimpleOption(go_dict, descr, short_groupdescr, long_groupdescr, config_files) diff --git a/vsc/utils/mail.py b/vsc/utils/mail.py deleted file mode 100644 index 318262a1f2..0000000000 --- a/vsc/utils/mail.py +++ /dev/null @@ -1,254 +0,0 @@ -#!/usr/bin/env python -## -# Copyright 2012-2013 Ghent University -# -# This file is part of vsc-base, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# the Hercules foundation (http://www.herculesstichting.be/in_English) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# http://github.com/hpcugent/vsc-base -# -# vsc-base is free software: you can redistribute it and/or modify -# it under the terms of the GNU Library General Public License as -# published by the Free Software Foundation, either version 2 of -# the License, or (at your option) any later version. -# -# vsc-base is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU Library General Public License -# along with vsc-base. If not, see . -## -""" -Wrapper around the standard Python mail library. - - - Send a plain text message - - Send an HTML message, with a plain text alternative - -@author: Andy Georges (Ghent University) -""" - -import re -import smtplib -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText -from email.mime.image import MIMEImage - -from vsc.utils import fancylogger - - -class VscMailError(Exception): - """Raised if the sending of an email fails for some reason.""" - - def __init__(self, mail_host=None, mail_to=None, mail_from=None, mail_subject=None, err=None): - """Initialisation. - - @type mail_host: string - @type mail_to: string - @type mail_from: string - @type mail_subject: string - @type err: Exception subclass - - @param mail_host: the SMTP host for actually sending the mail. - @param mail_to: a well-formed email address of the recipient. - @param mail_from: a well-formed email address of the sender. - @param mail_subject: the subject of the mail. - @param err: the original exception, if any. - """ - self.mail_host = mail_host - self.mail_to = mail_to - self.mail_from = mail_from - self.mail_subject = mail_subject - self.err = err - - -class VscMail(object): - """Class providing functionality to send out mail.""" - - def __init__(self, mail_host=None): - self.mail_host = mail_host - self.log = fancylogger.getLogger(self.__class__.__name__) - - def _send(self, - mail_from, - mail_to, - mail_subject, - msg): - """Actually send the mail. - - @type mail_from: string representing the sender. - @type mail_to: string representing the recipient. - @type mail_subject: string representing the subject. - @type msg: MIME message. - """ - - try: - if self.mail_host: - self.log.debug("Using %s as the mail host" % (self.mail_host,)) - s = smtplib.SMTP(self.mail_host) - else: - self.log.debug("Using the default mail host") - s = smtplib.SMTP() - s.connect() - try: - s.sendmail(mail_from, mail_to, msg.as_string()) - except smtplib.SMTPHeloError, err: - self.log.error("Cannot get a proper response from the SMTP host" + - (self.mail_host and " %s" % (self.mail_host) or "")) - raise - except smtplib.SMTPRecipientsRefused, err: - self.log.error("All recipients were refused by SMTP host" + - (self.mail_host and " %s" % (self.mail_host) or "") + - " [%s]" % (mail_to)) - raise - except smtplib.SMTPSenderRefused, err: - self.log.error("Sender was refused by SMTP host" + - (self.mail_host and " %s" % (self.mail_host) or "") + - "%s" % (mail_from)) - raise - except smtplib.SMTPDataError, err: - raise - except smtplib.SMTPConnectError, err: - self.log.exception("Cannot connect to the SMTP host" + (self.mail_host and " %s" % (self.mail_host) or "")) - raise VscMailError(mail_host=self.mail_host, - mail_to=mail_to, - mail_from=mail_from, - mail_subject=mail_subject, - err=err) - except Exception, err: - self.log.exception("Some unknown exception occurred in VscMail.sendTextMail. Raising a VscMailError.") - raise VscMailError(mail_host=self.mail_host, - mail_to=mail_to, - mail_from=mail_from, - mail_subject=mail_subject, - err=err) - - def sendTextMail(self, - mail_to, - mail_from, - reply_to, - mail_subject, - message): - """Send out the given message by mail to the given recipient(s). - - @type mail_to: string or list of strings - @type mail_from: string - @type reply_to: string - @type mail_subject: string - @type message: string - - @param mail_to: a valid recipient email address - @param mail_from: a valid sender email address. - @param reply_to: a valid email address for the (potential) replies. - @param mail_subject: the subject of the email. - @param message: the body of the mail. - """ - self.log.info("Sending mail [%s] to %s." % (mail_subject, mail_to)) - - if reply_to is None: - reply_to = mail_from - msg = MIMEText(message) - msg['Subject'] = mail_subject - msg['From'] = mail_from - msg['To'] = mail_to - msg['Reply-to'] = reply_to - - self._send(mail_from, mail_to, mail_subject, msg) - - def _replace_images_cid(self, html, images): - """Replaces all occurences of the src="IMAGE" with src="cid:IMAGE" in the provided html argument. - - @type html: string - @type images: list of strings - - @param html: HTML data, containing image tags for each of the provided images - @param images: references to the images occuring in the HTML payload - - @return: the altered HTML string. - """ - - for im in images: - re_src = re.compile("src=\"%s\"" % im) - (html, count) = re_src.subn("src=\"cid:%s\"" % im, html) - if count == 0: - self.log.raiseException("Could not find image %s in provided HTML." % im, VscMailError) - - return html - - def sendHTMLMail(self, - mail_to, - mail_from, - reply_to, - mail_subject, - html_message, - text_alternative, - images=None, - css=None): - """ - Send an HTML email message, encoded in a MIME/multipart message. - - The images and css are included in the message, and should be provided separately. - - @type mail_to: string or list of strings - @type mail_from: string - @type reply_to: string - @type mail_subject: string - @type html_message: string - @type text_alternative: string - @type images: list of strings - @type css: string - - @param mail_to: a valid recipient email addresses. - @param mail_from: a valid sender email address. - @param reply_to: a valid email address for the (potential) replies. - @param html_message: the actual payload, body of the mail - @param text_alternative: plain-text version of the mail body - @param images: the images that are referenced in the HTML body. These should be available as files on the - filesystem in the directory where the script runs. Caveat: assume jpeg image type. - @param css: CSS definitions - """ - - # Create message container - the correct MIME type is multipart/alternative. - msg_root = MIMEMultipart('alternative') - msg_root['Subject'] = mail_subject - msg_root['From'] = mail_from - msg_root['To'] = mail_to - - msg_root.preamble = 'This is a multi-part message in MIME format. If your email client does not support this (correctly), the first part is the plain text version.' - - # Create the body of the message (a plain-text and an HTML version). - if images is not None: - html_message = self.replace_images_cid(html_message, images) - - # Record the MIME types of both parts - text/plain and text/html_message. - msg_plain = MIMEText(text_alternative, 'plain') - msg_html = MIMEText(html_message, 'html_message') - - # Attach parts into message container. - # According to RFC 2046, the last part of a multipart message, in this case - # the HTML message, is best and preferred. - msg_root.attach(msg_plain) - msg_alt = MIMEMultipart('related') - msg_alt.attach(msg_html) - - if css is not None: - msg_html_css = MIMEText(css, 'css') - msg_html_css.add_header('Content-ID', '') - msg_alt.attach(msg_html_css) - - if images is not None: - for im in images: - image_fp = open(im, 'r') - msg_image = MIMEImage(image_fp.read(), 'jpeg') # FIXME: for now, we assume jpegs - image_fp.close() - msg_image.add_header('Content-ID', "<%s>" % im) - msg_alt.attach(msg_image) - - msg_root.attach(msg_alt) - - self._send(mail_from, mail_to, mail_subject, msg_root) diff --git a/vsc/utils/missing.py b/vsc/utils/missing.py deleted file mode 100644 index 9d4768fec0..0000000000 --- a/vsc/utils/missing.py +++ /dev/null @@ -1,349 +0,0 @@ -#!/usr/bin/env python -# # -# Copyright 2012-2013 Ghent University -# -# This file is part of vsc-base, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# the Hercules foundation (http://www.herculesstichting.be/in_English) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# http://github.com/hpcugent/vsc-base -# -# vsc-base is free software: you can redistribute it and/or modify -# it under the terms of the GNU Library General Public License as -# published by the Free Software Foundation, either version 2 of -# the License, or (at your option) any later version. -# -# vsc-base is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU Library General Public License -# along with vsc-base. If not, see . -# # -""" -Various functions that are missing from the default Python library. - - - nub(list): keep the unique elements in the list - - nub_by(list, predicate): keep the unique elements (first come, first served) that do not satisfy a given predicate - - find_sublist_index(list, sublist): find the index of the first - occurence of the sublist in the list - - Monoid: implementation of the monoid concept - - MonoidDict: dictionary that combines values upon insertiong - according to the given monoid - - RUDict: dictionary that allows recursively updating its values (if they are dicts too) with a new RUDict - - shell_quote / shell_unquote : convenience functions to quote / unquote strings in shell context - -@author: Andy Georges (Ghent University) -@author: Stijn De Weirdt (Ghent University) -""" -import shlex -import subprocess -import time - -from vsc.utils import fancylogger -from vsc.utils.frozendict import FrozenDict - - -def partial(func, *args, **keywords): - """ - Return a new partial object which when called will behave like func called with the positional arguments args - and keyword arguments keywords. If more arguments are supplied to the call, they are appended to args. If additional - keyword arguments are supplied, they extend and override keywords. - new in python 2.5, from https://docs.python.org/2/library/functools.html#functools.partial - """ - def newfunc(*fargs, **fkeywords): - newkeywords = keywords.copy() - newkeywords.update(fkeywords) - return func(*(args + fargs), **newkeywords) - newfunc.func = func - newfunc.args = args - newfunc.keywords = keywords - return newfunc - - -def any(ls): - """Reimplementation of 'any' function, which is not available in Python 2.4 yet.""" - - return sum([bool(x) for x in ls]) != 0 - - -def all(ls): - """Reimplementation of 'all' function, which is not available in Python 2.4 yet.""" - - return sum([bool(x) for x in ls]) == len(ls) - - -def nub(list_): - """Returns the unique items of a list of hashables, while preserving order of - the original list, i.e. the first unique element encoutered is - retained. - - Code is taken from - http://stackoverflow.com/questions/480214/how-do-you-remove-duplicates-from-a-list-in-python-whilst-preserving-order - - Supposedly, this is one of the fastest ways to determine the - unique elements of a list. - - @type list_: a list :-) - - @returns: a new list with each element from `list` appearing only once (cfr. Michelle Dubois). - """ - seen = set() - seen_add = seen.add - return [x for x in list_ if x not in seen and not seen_add(x)] - - -def nub_by(list_, predicate): - """Returns the elements of a list that fullfil the predicate. - - For any pair of elements in the resulting list, the predicate does not hold. For example, the nub above - can be expressed as nub_by(list, lambda x, y: x == y). - - @type list_: a list of items of some type t - @type predicate: a function that takes two elements of type t and returns a bool - - @returns: the nubbed list - """ - seen = set() - seen_add = seen.add - return [x for x in list_ if not any([predicate(x, y) for y in seen]) and not seen_add(x)] - - -def find_sublist_index(ls, sub_ls): - """Find the index at which the sublist sub_ls can be found in ls. - - @type ls: list - @type sub_ls: list - - @return: index of the matching location or None if no match can be made. - """ - sub_length = len(sub_ls) - for i in xrange(len(ls)): - if ls[i:(i + sub_length)] == sub_ls: - return i - - return None - - -class Monoid(object): - """A monoid is a mathematical object with a default element (mempty or null) and a default operation to combine - two elements of a given data type. - - Taken from http://fmota.eu/2011/10/09/monoids-in-python.html under the do whatever you want license. - """ - - def __init__(self, null, mappend): - """Initialise. - - @type null: default element of some data type, e.g., [] for list or 0 for int (identity element in an Abelian group) - @type op: mappend operation to combine two elements of the target datatype - """ - self.null = null - self.mappend = mappend - - def fold(self, xs): - """fold over the elements of the list, combining them into a single element of the target datatype.""" - if hasattr(xs, "__fold__"): - return xs.__fold__(self) - else: - return reduce( - self.mappend, - xs, - self.null - ) - - def __call__(self, *args): - """When the monoid is called, the values are folded over and the resulting value is returned.""" - return self.fold(args) - - def star(self): - """Return a new similar monoid.""" - return Monoid(self.null, self.mappend) - - -class MonoidDict(dict): - """A dictionary with a monoid operation, that allows combining values in the dictionary according to the mappend - operation in the monoid. - """ - - def __init__(self, monoid, *args, **kwargs): - """Initialise. - - @type monoid: Monoid instance - """ - super(MonoidDict, self).__init__(*args, **kwargs) - self.monoid = monoid - - def __setitem__(self, key, value): - """Combine the value the dict has for the key with the new value using the mappend operation.""" - if super(MonoidDict, self).__contains__(key): - current = super(MonoidDict, self).__getitem__(key) - super(MonoidDict, self).__setitem__(key, self.monoid(current, value)) - else: - super(MonoidDict, self).__setitem__(key, value) - - def __getitem__(self, key): - """ Obtain the dictionary value for the given key. If no value is present, - we return the monoid's mempty (null). - """ - if not super(MonoidDict, self).__contains__(key): - return self.monoid.null - else: - return super(MonoidDict, self).__getitem__(key) - - -class RUDict(dict): - """Recursively updatable dictionary. - - When merging with another dictionary (of the same structure), it will keep - updating the values as well if they are dicts or lists. - - Code taken from http://stackoverflow.com/questions/6256183/combine-two-dictionaries-of-dictionaries-python. - """ - - def update(self, E=None, **F): - if E is not None: - if 'keys' in dir(E) and callable(getattr(E, 'keys')): - for k in E: - if k in self: # existing ...must recurse into both sides - self.r_update(k, E) - else: # doesn't currently exist, just update - self[k] = E[k] - else: - for (k, v) in E: - self.r_update(k, {k: v}) - - for k in F: - self.r_update(k, {k: F[k]}) - - def r_update(self, key, other_dict): - """Recursive update.""" - if isinstance(self[key], dict) and isinstance(other_dict[key], dict): - od = RUDict(self[key]) - nd = other_dict[key] - od.update(nd) - self[key] = od - elif isinstance(self[key], list): - if isinstance(other_dict[key], list): - self[key].extend(other_dict[key]) - else: - self[key] = self[key].append(other_dict[key]) - else: - self[key] = other_dict[key] - - -class FrozenDictKnownKeys(FrozenDict): - """A frozen dictionary only allowing known keys.""" - - # list of known keys - KNOWN_KEYS = [] - - def __init__(self, *args, **kwargs): - """Constructor, only way to define the contents.""" - self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) - - # support ignoring of unknown keys - ignore_unknown_keys = kwargs.pop('ignore_unknown_keys', False) - - # handle unknown keys: either ignore them or raise an exception - tmpdict = dict(*args, **kwargs) - unknown_keys = [key for key in tmpdict.keys() if not key in self.KNOWN_KEYS] - if unknown_keys: - if ignore_unknown_keys: - for key in unknown_keys: - self.log.debug("Ignoring unknown key '%s' (value '%s')" % (key, args[0][key])) - # filter key out of dictionary before creating instance - del tmpdict[key] - else: - msg = "Encountered unknown keys %s (known keys: %s)" % (unknown_keys, self.KNOWN_KEYS) - self.log.raiseException(msg, exception=KeyError) - - super(FrozenDictKnownKeys, self).__init__(tmpdict) - - def __getitem__(self, key, *args, **kwargs): - """Redefine __getitem__ to provide a better KeyError message.""" - try: - return super(FrozenDictKnownKeys, self).__getitem__(key, *args, **kwargs) - except KeyError, err: - if key in self.KNOWN_KEYS: - raise KeyError(err) - else: - tup = (key, self.__class__.__name__, self.KNOWN_KEYS) - raise KeyError("Unknown key '%s' for %s instance (known keys: %s)" % tup) - - -def shell_quote(x): - """Add quotes so it can be apssed to shell""" - # use undocumented subprocess API call to quote whitespace (executed with Popen(shell=True)) - # (see http://stackoverflow.com/questions/4748344/whats-the-reverse-of-shlex-split for alternatives if needed) - return subprocess.list2cmdline([str(x)]) - - -def shell_unquote(x): - """Take a literal string, remove the quotes as if it were passed by shell""" - # it expects a string - return shlex.split(str(x))[0] - - -def get_subclasses(klass): - """Get all subclasses recursively""" - res = [] - for cl in klass.__subclasses__(): - res.extend(get_subclasses(cl)) - res.append(cl) - return res - - -class TryOrFail(object): - """ - Perform the function n times, catching each exception in the exception tuple except on the last try - where it will be raised again. - """ - def __init__(self, n, exceptions=(Exception,), sleep=0): - self.n = n - self.exceptions = exceptions - self.sleep = sleep - - def __call__(self, function): - def new_function(*args, **kwargs): - log = fancylogger.getLogger(function.__name__) - for i in xrange(0, self.n): - try: - return function(*args, **kwargs) - except self.exceptions, err: - if i == self.n - 1: - raise - log.exception("try_or_fail caught an exception - attempt %d: %s" % (i, err)) - if self.sleep > 0: - log.warning("try_or_fail is sleeping for %d seconds before the next attempt" % (self.sleep,)) - time.sleep(self.sleep) - - return new_function - - -def post_order(graph, root): - """ - Walk the graph from the given root in a post-order manner by providing the corresponding generator - """ - for node in graph[root]: - for child in post_order(graph, node): - yield child - yield root - - -def topological_sort(graph): - """ - Perform topological sorting of the given graph. - - The graph is a dict with the values for a key being the dependencies, i.e., an arrow from key to each value. - """ - visited = set() - for root in graph: - for node in post_order(graph, root): - if not node in visited: - yield node - visited.add(node) diff --git a/vsc/utils/optcomplete.py b/vsc/utils/optcomplete.py deleted file mode 100644 index 8d070786ca..0000000000 --- a/vsc/utils/optcomplete.py +++ /dev/null @@ -1,629 +0,0 @@ -#******************************************************************************\ -# * Copyright (c) 2003-2004, Martin Blais -# * All rights reserved. -# * -# * Redistribution and use in source and binary forms, with or without -# * modification, are permitted provided that the following conditions are -# * met: -# * -# * * Redistributions of source code must retain the above copyright -# * notice, this list of conditions and the following disclaimer. -# * -# * * Redistributions in binary form must reproduce the above copyright -# * notice, this list of conditions and the following disclaimer in the -# * documentation and/or other materials provided with the distribution. -# * -# * * Neither the name of the Martin Blais, Furius, nor the names of its -# * contributors may be used to endorse or promote products derived from -# * this software without specific prior written permission. -# * -# * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#******************************************************************************\ - -"""Automatic completion for optparse module. - -This module provide automatic bash completion support for programs that use the -optparse module. The premise is that the optparse options parser specifies -enough information (and more) for us to be able to generate completion strings -esily. Another advantage of this over traditional completion schemes where the -completion strings are hard-coded in a separate bash source file, is that the -same code that parses the options is used to generate the completions, so the -completions is always up-to-date with the program itself. - -In addition, we allow you specify a list of regular expressions or code that -define what kinds of files should be proposed as completions to this file if -needed. If you want to implement more complex behaviour, you can instead -specify a function, which will be called with the current directory as an -argument. - -You need to activate bash completion using the shell script function that comes -with optcomplete (see http://furius.ca/optcomplete for more details). - -@author: Martin Blais (blais@furius.ca) -@author: Stijn De Weirdt (Ghent University) - -This is a copy of optcomplete.py (changeset 17:e0a9131a94cc) -from source: https://hg.furius.ca/public/optcomplete - -Modification by stdweird: - - cleanup -""" - -# Bash Protocol Description -# ------------------------- -# 'COMP_CWORD' -# An index into `${COMP_WORDS}' of the word containing the current -# cursor position. This variable is available only in shell -# functions invoked by the programmable completion facilities (*note -# Programmable Completion::). -# -# 'COMP_LINE' -# The current command line. This variable is available only in -# shell functions and external commands invoked by the programmable -# completion facilities (*note Programmable Completion::). -# -# 'COMP_POINT' -# The index of the current cursor position relative to the beginning -# of the current command. If the current cursor position is at the -# end of the current command, the value of this variable is equal to -# `${#COMP_LINE}'. This variable is available only in shell -# functions and external commands invoked by the programmable -# completion facilities (*note Programmable Completion::). -# -# 'COMP_WORDS' -# An array variable consisting of the individual words in the -# current command line. This variable is available only in shell -# functions invoked by the programmable completion facilities (*note -# Programmable Completion::). -# -# 'COMPREPLY' -# An array variable from which Bash reads the possible completions -# generated by a shell function invoked by the programmable -# completion facility (*note Programmable Completion::). - - -import copy -import glob -import logging -import os -import re -import sys -import types - -from optparse import OptionParser, Option -from pprint import pformat - -debugfn = None # for debugging only - -OPTCOMPLETE_ENVIRONMENT = 'OPTPARSE_AUTO_COMPLETE' - -BASH = "bash" - -DEFAULT_SHELL = BASH - -SHELL = DEFAULT_SHELL - -OPTION_CLASS = Option -OPTIONPARSER_CLASS = OptionParser - -def set_optionparser(option_class, optionparser_class): - """Set the default Option and OptionParser class""" - global OPTION_CLASS - global OPTIONPARSER_CLASS - OPTION_CLASS = option_class - OPTIONPARSER_CLASS = optionparser_class - -def get_shell(): - """Determine the shell, update class constant SHELL and return the shell - Idea is to call it just once - """ - global SHELL - SHELL = os.path.basename(os.environ.get("SHELL", DEFAULT_SHELL)) - return SHELL - - -# get the shell -get_shell() - - -class CompleterMissingCallArgument(Exception): - """Exception to raise when call arg is missing""" - - -class Completer(object): - """Base class to derive all other completer classes from. - It generates an empty completion list - """ - CALL_ARGS = None # list of named args that must be passed - CALL_ARGS_OPTIONAL = None # list of named args that can be passed - - def __call__(self, **kwargs): - """Check mandatory args, then return _call""" - all_args = [] - if self.CALL_ARGS is not None: - for arg in self.CALL_ARGS: - all_args.append(arg) - if not arg in kwargs: - msg = "%s __call__ missing mandatory arg %s" % (self.__class__.__name__, arg) - raise CompleterMissingCallArgument(msg) - - if self.CALL_ARGS_OPTIONAL is not None: - all_args.extend(self.CALL_ARGS_OPTIONAL) - - for arg in kwargs.keys(): - if not arg in all_args: - # remove it - kwargs.pop(arg) - - return self._call(**kwargs) - - def _call(self, **kwargs): - """Return empty list""" - return [] - - -class NoneCompleter(Completer): - """Generates empty completion list. For compatibility reasons.""" - pass - - -class ListCompleter(Completer): - """Completes by filtering using a fixed list of strings.""" - def __init__(self, stringlist): - self.olist = stringlist - - def _call(self, **kwargs): - """Return the initialised fixed list of strings""" - return map(str, self.olist) - - -class AllCompleter(Completer): - """Completes by listing all possible files in current directory.""" - CALL_ARGS_OPTIONAL = ['pwd'] - - def _call(self, **kwargs): - return os.listdir(kwargs.get('pwd', '.')) - - -class FileCompleter(Completer): - """Completes by listing all possible files in current directory. - If endings are specified, then limit the files to those.""" - CALL_ARGS_OPTIONAL = ['prefix'] - - def __init__(self, endings=None): - if isinstance(endings, basestring): - endings = [endings] - elif endings is None: - endings = [] - self.endings = tuple(map(str, endings)) - - def _call(self, **kwargs): - # TODO : what does prefix do in bash? - prefix = kwargs.get('prefix', '') - - if SHELL == BASH: - res = ['_filedir'] - if self.endings: - res.append("'@(%s)'" % '|'.join(self.endings)) - return " ".join(res) - else: - res = [] - for path in glob.glob(prefix + '*'): - res.append(path) - if os.path.isdir(path): - # add trailing slashes to directories - res[-1] += os.path.sep - - if self.endings: - res = [path for path in res if os.path.isdir(path) or path.endswith(self.endings)] - - if len(res) == 1 and os.path.isdir(res[0]): - # return two options so that it completes the / but doesn't add a space - return [res[0] + 'a', res[0] + 'b'] - else: - return res - - -class DirCompleter(Completer): - """Completes by listing subdirectories only.""" - CALL_ARGS_OPTIONAL = ['prefix'] - - def _call(self, **kwargs): - # TODO : what does prefix do in bash? - prefix = kwargs.get('prefix', '') - - if SHELL == BASH: - return "_filedir -d" - else: - res = [path + "/" for path in glob.glob(prefix + '*') if os.path.isdir(path)] - - if len(res) == 1: - # return two options so that it completes the / but doesn't add a space - return [res[0] + 'a', res[0] + 'b'] - else: - return res - - -class KnownHostsCompleter(Completer): - """Completes a list of known hostnames""" - def _call(self, **kwargs): - if SHELL == BASH: - return "_known_hosts" - else: - # TODO needs implementation, no autocompletion for now - return [] - - -class RegexCompleter(Completer): - """Completes by filtering all possible files with the given list of regexps.""" - CALL_ARGS_OPTIONAL = ['prefix', 'pwd'] - - def __init__(self, regexlist, always_dirs=True): - self.always_dirs = always_dirs - - if isinstance(regexlist, basestring): - regexlist = [regexlist] - self.regexlist = [] - for regex in regexlist: - if isinstance(regex, basestring): - regex = re.compile(regex) - self.regexlist.append(regex) - - def _call(self, **kwargs): - dn = os.path.dirname(kwargs.get('prefix', '')) - if dn: - pwd = dn - else: - pwd = kwargs.get('pwd', '.') - - ofiles = [] - for fn in os.listdir(pwd): - for r in self.regexlist: - if r.match(fn): - if dn: - fn = os.path.join(dn, fn) - ofiles.append(fn) - break - - if self.always_dirs and os.path.isdir(fn): - ofiles.append(fn + os.path.sep) - - return ofiles - - -class CompleterOption(OPTION_CLASS): - """optparse Option class with completer attribute""" - def __init__(self, *args, **kwargs): - completer = kwargs.pop('completer', None) - OPTION_CLASS.__init__(self, *args, **kwargs) - if completer is not None: - self.completer = completer - - -def extract_word(line, point): - """Return a prefix and suffix of the enclosing word. The character under - the cursor is the first character of the suffix.""" - - if SHELL == BASH and 'IFS' in os.environ: - ifs = [r.group(0) for r in re.finditer(r'.', os.environ['IFS'])] - wsre = re.compile('|'.join(ifs)) - else: - wsre = re.compile(r'\s') - - if point < 0 or point > len(line): - return '', '' - - preii = point - 1 - while preii >= 0: - if wsre.match(line[preii]): - break - preii -= 1 - preii += 1 - - sufii = point - while sufii < len(line): - if wsre.match(line[sufii]): - break - sufii += 1 - - return line[preii : point], line[point : sufii] - - -def error_override(self, msg): - """Hack to keep OptionParser from writing to sys.stderr when - calling self.exit from self.error""" - self.exit(2, msg=None) - - -def guess_first_nonoption(gparser, subcmds_map): - """Given a global options parser, try to guess the first non-option without - generating an exception. This is used for scripts that implement a - subcommand syntax, so that we can generate the appropriate completions for - the subcommand.""" - - - gparser = copy.deepcopy(gparser) - def print_usage_nousage (self, *args, **kwargs): - pass - gparser.print_usage = print_usage_nousage - - prev_interspersed = gparser.allow_interspersed_args # save state to restore - gparser.disable_interspersed_args() - - cwords = os.environ.get('COMP_WORDS', '').split() - - # save original error_func so we can put it back after the hack - error_func = gparser.error - try: - try: - instancemethod = type(OPTIONPARSER_CLASS.error) - # hack to keep OptionParser from writing to sys.stderr - gparser.error = instancemethod(error_override, gparser, OPTIONPARSER_CLASS) - _, args = gparser.parse_args(cwords[1:]) - except SystemExit: - return None - finally: - # undo the hack and restore original OptionParser error function - gparser.error = instancemethod(error_func, gparser, OPTIONPARSER_CLASS) - - value = None - if args: - subcmdname = args[0] - try: - value = subcmds_map[subcmdname] - except KeyError: - pass - - gparser.allow_interspersed_args = prev_interspersed # restore state - - return value # can be None, indicates no command chosen. - - -def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_completer=None, subcommands=None): - """Automatically detect if we are requested completing and if so generate - completion automatically from given parser. - - 'parser' is the options parser to use. - - 'arg_completer' is a callable object that gets invoked to produce a list of - completions for arguments completion (oftentimes files). - - 'opt_completer' is the default completer to the options that require a - value. - - 'subcmd_completer' is the default completer for the subcommand - arguments. - - If 'subcommands' is specified, the script expects it to be a map of - command-name to an object of any kind. We are assuming that this object is - a map from command name to a pair of (options parser, completer) for the - command. If the value is not such a tuple, the method - 'autocomplete(completer)' is invoked on the resulting object. - - This will attempt to match the first non-option argument into a subcommand - name and if so will use the local parser in the corresponding map entry's - value. This is used to implement completion for subcommand syntax and will - not be needed in most cases. - """ - - # If we are not requested for complete, simply return silently, let the code - # caller complete. This is the normal path of execution. - if not os.environ.has_key(OPTCOMPLETE_ENVIRONMENT): - return - # After this point we should never return, only sys.exit(1) - - # Set default completers. - if arg_completer is None: - arg_completer = NoneCompleter() - if opt_completer is None: - opt_completer = FileCompleter() - if subcmd_completer is None: - # subcmd_completer = arg_completer - subcmd_completer = FileCompleter() - - # By default, completion will be arguments completion, unless we find out - # later we're trying to complete for an option. - completer = arg_completer - - # - # Completing... - # - - # Fetching inputs... not sure if we're going to use these. - - # zsh's bashcompinit does not pass COMP_WORDS, replace with - # COMP_LINE for now... - if not os.environ.has_key('COMP_WORDS'): - os.environ['COMP_WORDS'] = os.environ['COMP_LINE'] - - cwords = os.environ.get('COMP_WORDS', '').split() - cline = os.environ.get('COMP_LINE', '') - cpoint = int(os.environ.get('COMP_POINT', 0)) - cword = int(os.environ.get('COMP_CWORD', 0)) - - # Extract word enclosed word. - prefix, suffix = extract_word(cline, cpoint) - - # If requested, try subcommand syntax to find an options parser for that - # subcommand. - if subcommands: - assert isinstance(subcommands, dict) - value = guess_first_nonoption(parser, subcommands) - if value: - if isinstance(value, (list, tuple)): - parser = value[0] - if len(value) > 1 and value[1]: - # override completer for command if it is present. - completer = value[1] - else: - completer = subcmd_completer - autocomplete(parser, completer) - elif hasattr(value, 'autocomplete'): - # Call completion method on object. This should call - # autocomplete() recursively with appropriate arguments. - value.autocomplete(subcmd_completer) - else: - # no completions for that command object - pass - sys.exit(1) - else: # suggest subcommands - completer = ListCompleter(subcommands.keys()) - - # Look at previous word, if it is an option and it requires an argument, - # check for a local completer. If there is no completer, what follows - # directly cannot be another option, so mark to not add those to - # completions. - optarg = False - try: - # Look for previous word, which will be containing word if the option - # has an equals sign in it. - prev = None - if cword < len(cwords): - mo = re.search('(--.*?)=(.*)', cwords[cword]) - if mo: - prev, prefix = mo.groups() - if not prev: - prev = cwords[cword - 1] - - if prev and prev.startswith('-'): - option = parser.get_option(prev) - if option: - if option.nargs > 0: - optarg = True - if hasattr(option, 'completer'): - completer = option.completer - elif option.choices: - completer = ListCompleter(option.choices) - elif option.type in ('string',): - completer = opt_completer - else: - completer = NoneCompleter() - # Warn user at least, it could help him figure out the problem. - elif hasattr(option, 'completer'): - msg = "Error: optparse option with a completer does not take arguments: %s" % (option) - raise SystemExit(msg) - except KeyError: - pass - - completions = [] - - # Options completion. - if not optarg and (not prefix or prefix.startswith('-')): - completions += parser._short_opt.keys() - completions += parser._long_opt.keys() - # Note: this will get filtered properly below. - - completer_kwargs = { - 'pwd': os.getcwd(), - 'cline': cline, - 'cpoint': cpoint, - 'prefix': prefix, - 'suffix': suffix, - } - # File completion. - if completer and (not prefix or not prefix.startswith('-')): - # Call appropriate completer depending on type. - if isinstance(completer, (basestring, list, tuple)): - completer = FileCompleter(completer) - elif not isinstance(completer, (types.FunctionType, types.LambdaType, types.ClassType, types.ObjectType)): - # TODO: what to do here? - pass - - completions = completer(**completer_kwargs) - - if isinstance(completions, basestring): - # is a bash command, just run it - if SHELL in (BASH,): # TODO: zsh - print completions - else: - raise Exception("Commands are unsupported by this shell %s" % SHELL) - else: - # Filter using prefix. - if prefix: - completions = sorted(filter(lambda x: x.startswith(prefix), completions)) - completions = ' '.join(map(str, completions)) - - # Save results - if SHELL == "bash": - print 'COMPREPLY=(' + completions + ')' - else: - print completions - - # Print debug output (if needed). You can keep a shell with 'tail -f' to - # the log file to monitor what is happening. - if debugfn: - txt = "\n".join([ - '---------------------------------------------------------', - 'CWORDS %s' % cwords, - 'CLINE %s' % cline, - 'CPOINT %s' % cpoint, - 'CWORD %s' % cword, - '', - 'Short options', - pformat(parser._short_opt), - '', - 'Long options', - pformat(parser._long_opt), - 'Prefix %s' % prefix, - 'Suffix %s', suffix, - 'completions %s' % completions, - ]) - if isinstance(debugfn, logging.Logger): - debugfn.debug(txt) - else: - f = open(debugfn, 'a') - f.write(txt) - f.close() - - # Exit with error code (we do not let the caller continue on purpose, this - # is a run for completions only.) - sys.exit(1) - - -class CmdComplete(object): - - """Simple default base class implementation for a subcommand that supports - command completion. This class is assuming that there might be a method - addopts(self, parser) to declare options for this subcommand, and an - optional completer data member to contain command-specific completion. Of - course, you don't really have to use this, but if you do it is convenient to - have it here.""" - - def autocomplete(self, completer=None): - parser = OPTIONPARSER_CLASS(self.__doc__.strip()) - if hasattr(self, 'addopts'): - fnc = getattr(self, 'addopts') - fnc(parser) - - if hasattr(self, 'completer'): - completer = getattr(self, 'completer') - - return autocomplete(parser, completer) - - -def gen_cmdline(cmd_list, partial): - """Create the commandline to generate simulated tabcompletion output - @param cmd_list: command to execute as list of strings - @param partial: the string to autocomplete (typically, partial is an element of the cmd_list) - """ - cmdline = " ".join(cmd_list) - - env = [] - env.append("%s=1" % OPTCOMPLETE_ENVIRONMENT) - env.append('COMP_LINE="%s"' % cmdline) - env.append('COMP_WORDS=(%s)' % cmdline) - env.append('COMP_POINT=%s' % len(cmdline)) - env.append('COMP_CWORD=%s' % cmd_list.index(partial)) - - return "%s %s" % (" ".join(env), cmd_list[0]) - diff --git a/vsc/utils/patterns.py b/vsc/utils/patterns.py deleted file mode 100644 index efb614551a..0000000000 --- a/vsc/utils/patterns.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python -## -# Copyright 2012-2013 Ghent University -# -# This file is part of vsc-base, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# the Hercules foundation (http://www.herculesstichting.be/in_English) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# http://github.com/hpcugent/vsc-base -# -# vsc-base is free software: you can redistribute it and/or modify -# it under the terms of the GNU Library General Public License as -# published by the Free Software Foundation, either version 2 of -# the License, or (at your option) any later version. -# -# vsc-base is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU Library General Public License -# along with vsc-base. If not, see . -## -""" -Module offering the Singleton class. - - -This class can be used as the C{__metaclass__} class field to ensure only a -single instance of the class gets used in the run of an application or -script. - ->>> class A(object): -... __metaclass__ = Singleton - -@author: Andy Georges (Ghent University) -""" - - -class Singleton(type): - """Serves as metaclass for classes that should implement the Singleton pattern. - - See http://stackoverflow.com/questions/6760685/creating-a-singleton-in-python - """ - _instances = {} - - def __call__(cls, *args, **kwargs): - if cls not in cls._instances: - cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) - return cls._instances[cls] diff --git a/vsc/utils/rest.py b/vsc/utils/rest.py deleted file mode 100644 index a7e87ec001..0000000000 --- a/vsc/utils/rest.py +++ /dev/null @@ -1,268 +0,0 @@ -## -# This file is part of agithub -# Originally created by Jonathan Paugh -# -# https://github.com/jpaugh/agithub -# -# Copyright 2012 Jonathan Paugh -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -## -""" -This module contains Rest api utilities, -Mainly the RestClient, which you can use to easily pythonify a rest api. - -based on https://github.com/jpaugh/agithub/commit/1e2575825b165c1cb7cbd85c22e2561fc4d434d3 - -@author: Jonathan Paugh -@author: Jens Timmerman -""" -import base64 -import urllib -import urllib2 -try: - import json -except ImportError: - import simplejson as json - -from vsc.utils import fancylogger -from vsc.utils.missing import partial - - -class Client(object): - """An implementation of a REST client""" - DELETE = 'DELETE' - GET = 'GET' - HEAD = 'HEAD' - PATCH = 'PATCH' - POST = 'POST' - PUT = 'PUT' - - HTTP_METHODS = ( - DELETE, - GET, - HEAD, - PATCH, - POST, - PUT, - ) - - USER_AGENT = 'vsc-rest-client' - - def __init__(self, url, username=None, password=None, token=None, token_type='Token', user_agent=None): - """ - Create a Client object, - this client can consume a REST api hosted at host/endpoint - - If a username is given a password or a token is required. - You can not use a password and a token. - token_type is the typoe fo th the authorization token text in the http authentication header, defaults to Token - This should be set to 'Bearer' for certain OAuth implementations. - """ - self.auth_header = None - self.username = username - self.url = url - - if not user_agent: - self.user_agent = self.USER_AGENT - else: - self.user_agent = user_agent - - handler = urllib2.HTTPSHandler() - self.opener = urllib2.build_opener(handler) - - if username is not None: - if password is None and token is None: - raise TypeError("You need a password or an OAuth token to authenticate as " + username) - if password is not None and token is not None: - raise TypeError("You cannot use both password and OAuth token authenication") - - if password is not None: - self.auth_header = self.hash_pass(password, username) - elif token is not None: - self.auth_header = '%s %s' % (token_type, token) - - def get(self, url, headers={}, **params): - """ - Do a http get request on the given url with given headers and parameters - Parameters is a dictionary that will will be urlencoded - """ - url += self.urlencode(params) - return self.request(self.GET, url, None, headers) - - def head(self, url, headers={}, **params): - """ - Do a http head request on the given url with given headers and parameters - Parameters is a dictionary that will will be urlencoded - """ - url += self.urlencode(params) - return self.request(self.HEAD, url, None, headers) - - def delete(self, url, headers={}, **params): - """ - Do a http delete request on the given url with given headers and parameters - Parameters is a dictionary that will will be urlencoded - """ - url += self.urlencode(params) - return self.request(self.DELETE, url, None, headers) - - def post(self, url, body=None, headers={}, **params): - """ - Do a http post request on the given url with given body, headers and parameters - Parameters is a dictionary that will will be urlencoded - """ - url += self.urlencode(params) - return self.request(self.POST, url, json.dumps(body), headers) - - def put(self, url, body=None, headers={}, **params): - """ - Do a http put request on the given url with given body, headers and parameters - Parameters is a dictionary that will will be urlencoded - """ - url += self.urlencode(params) - return self.request(self.PUT, url, json.dumps(body), headers) - - def patch(self, url, body=None, headers={}, **params): - """ - Do a http patch request on the given url with given body, headers and parameters - Parameters is a dictionary that will will be urlencoded - """ - url += self.urlencode(params) - return self.request(self.PATCH, url, json.dumps(body), headers) - - def request(self, method, url, body, headers): - if self.auth_header is not None: - headers['Authorization'] = self.auth_header - headers['User-Agent'] = self.user_agent - fancylogger.getLogger().debug('cli request: %s, %s, %s, %s', method, url, body, headers) - #TODO: in recent python: Context manager - conn = self.get_connection(method, url, body, headers) - status = conn.code - body = conn.read() - try: - pybody = json.loads(body) - except ValueError: - pybody = body - fancylogger.getLogger().debug('reponse len: %s ', len(pybody)) - conn.close() - return status, pybody - - def urlencode(self, params): - if not params: - return '' - return '?' + urllib.urlencode(params) - - def hash_pass(self, password, username=None): - if not username: - username = self.username - return 'Basic ' + base64.b64encode('%s:%s' % (username, password)).strip() - - def get_connection(self, method, url, body, headers): - if not self.url.endswith('/') and not url.startswith('/'): - sep = '/' - else: - sep = '' - request = urllib2.Request(self.url + sep + url, data=body) - for header, value in headers.iteritems(): - request.add_header(header, value) - request.get_method = lambda: method - fancylogger.getLogger().debug('opening request: %s%s%s', self.url, sep, url) - connection = self.opener.open(request) - return connection - - -class RequestBuilder(object): - '''RequestBuilder(client).path.to.resource.method(...) - stands for - RequestBuilder(client).client.method('path/to/resource, ...) - - Also, if you use an invalid path, too bad. Just be ready to catch a - You can use item access instead of attribute access. This is - convenient for using variables' values and required for numbers. - bad status from github.com. (Or maybe an httplib.error...) - - To understand the method(...) calls, check out github.client.Client. - ''' - def __init__(self, client): - """Constructor""" - self.client = client - self.url = '' - - def __getattr__(self, key): - """ - Overwrite __getattr__ to build up the equest url - this enables us to do bla.some.path['something'] - and get the url bla/some/path/something - """ - # make sure key is a string - key = str(key) - # our methods are lowercase, but our HTTP_METHOD constants are upercase, so check if it is in there, but only - # if it was a lowercase key - # this is here so bla.something.get() should work, and not result in bla/something/get being returned - if key.upper() in self.client.HTTP_METHODS and [x for x in key if x.islower()]: - mfun = getattr(self.client, key) - fun = partial(mfun, url=self.url) - return fun - self.url += '/' + key - return self - - __getitem__ = __getattr__ - - def __str__(self): - '''If you ever stringify this, you've (probably) messed up - somewhere. So let's give a semi-helpful message. - ''' - return "I don't know about %s, You probably want to do a get or other http request, use .get()" % self.url - - def __repr__(self): - return '%s: %s' % (self.__class__, self.url) - - -class RestClient(object): - """ - A client with a request builder, so you can easily create rest requests - e.g. to create a github Rest API client just do - >>> g = RestClient('https://api.github.com', username='user', password='pass') - >>> g = RestClient('https://api.github.com', token='oauth token') - >>> status, data = g.issues.get(filter='subscribed') - >>> data - ... [ list_, of, stuff ] - >>> status, data = g.repos.jpaugh64.repla.issues[1].get() - >>> data - ... { 'dict': 'my issue data', } - >>> name, repo = 'jpaugh64', 'repla' - >>> status, data = g.repos[name][repo].issues[1].get() - ... same thing - >>> status, data = g.funny.I.donna.remember.that.one.get() - >>> status - ... 404 - - That's all there is to it. (blah.post() should work, too.) - - NOTE: It is up to you to spell things correctly. Github doesn't even - try to validate the url you feed it. On the other hand, it - automatically supports the full API--so why should you care? - """ - def __init__(self, *args, **kwargs): - """We create a client with the given arguments""" - self.client = Client(*args, **kwargs) - - def __getattr__(self, key): - """Get an attribute, we will build a request with it""" - return RequestBuilder(self.client).__getattr__(key) diff --git a/vsc/utils/run.py b/vsc/utils/run.py deleted file mode 100644 index 8e61c95bf6..0000000000 --- a/vsc/utils/run.py +++ /dev/null @@ -1,843 +0,0 @@ -# -# Copyright 2009-2013 Ghent University -# -# This file is part of vsc-base, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# the Hercules foundation (http://www.herculesstichting.be/in_English) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# http://github.com/hpcugent/vsc-base -# -# vsc-base is free software: you can redistribute it and/or modify -# it under the terms of the GNU Library General Public License as -# published by the Free Software Foundation, either version 2 of -# the License, or (at your option) any later version. -# -# vsc-base is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU Library General Public License -# along with vsc-base. If not, see . -# - -""" -Python module to execute a command - -Historical overview of existing equivalent code - - - EasyBuild filetools module - - C{run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None)} - - C{run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None)} - - - Executes a command cmd - - looks for questions and tries to answer based on qa dictionary - - returns exitcode and stdout+stderr (mixed) - - no input though stdin - - if C{log_ok} or C{log_all} are set -> will C{log.error} if non-zero exit-code - - if C{simple} is C{True} -> instead of returning a tuple (output, ec) it will just return C{True} or C{False} signifying succes - - C{regexp} -> Regex used to check the output for errors. If C{True} will use default (see C{parselogForError}) - - if log_output is True -> all output of command will be logged to a tempfile - - path is the path run_cmd should chdir to before doing anything - - - Q&A: support reading stdout asynchronous and replying to a question through stdin - - - Manage C{managecommands} module C{Command} class - - C{run} method - - - python-package-vsc-utils run module Command class - - C{run} method - - - C{mympirun} (old) - - C{runrun(self, cmd, returnout=False, flush=False, realcmd=False)}: - - C{runrunnormal(self, cmd, returnout=False, flush=False)} - - C{runrunfile(self, cmd, returnout=False, flush=False)} - - - C{hanything} commands/command module - - C{run} method - - fake pty support - -@author: Stijn De Weirdt (Ghent University) -""" - -import errno -import logging -import os -import pty -import re -import signal -import sys -import time - -from vsc.utils.fancylogger import getLogger, getAllExistingLoggers - - -PROCESS_MODULE_ASYNCPROCESS_PATH = 'vsc.utils.asyncprocess' -PROCESS_MODULE_SUBPROCESS_PATH = 'subprocess' - -RUNRUN_TIMEOUT_OUTPUT = '' -RUNRUN_TIMEOUT_EXITCODE = 123 -RUNRUN_QA_MAX_MISS_EXITCODE = 124 - -BASH = '/bin/bash' -SHELL = BASH - - -class DummyFunction(object): - def __getattr__(self, name): - def dummy(*args, **kwargs): - pass - return dummy - - -class Run(object): - """Base class for static run method""" - INIT_INPUT_CLOSE = True - USE_SHELL = True - SHELL = SHELL # set the shell via the module constant - - @classmethod - def run(cls, cmd, **kwargs): - """static method - return (exitcode,output) - """ - r = cls(cmd, **kwargs) - return r._run() - - def __init__(self, cmd=None, **kwargs): - """ - Handle initiliastion - @param cmd: command to run - @param input: set "simple" input - @param startpath: directory to change to before executing command - @param disable_log: use fake logger (won't log anything) - @param use_shell: use the subshell - @param shell: change the shell - """ - self.input = kwargs.pop('input', None) - self.startpath = kwargs.pop('startpath', None) - self.use_shell = kwargs.pop('use_shell', self.USE_SHELL) - self.shell = kwargs.pop('shell', self.SHELL) - - if kwargs.pop('disable_log', None): - self.log = DummyFunction() # No logging - if not hasattr(self, 'log'): - self.log = getLogger(self._get_log_name()) - - self.cmd = cmd # actual command - - self._cwd_before_startpath = None - - self._process_module = None - self._process = None - - self.readsize = 1024 # number of bytes to read blocking - - self._shellcmd = None - self._popen_named_args = None - - self._process_exitcode = None - self._process_output = None - - self._post_exitcode_log_failure = self.log.error - - super(Run, self).__init__(**kwargs) - - def _get_log_name(self): - """Set the log name""" - return self.__class__.__name__ - - def _prep_module(self, modulepath=None, extendfromlist=None): - # these will provide the required Popen, PIPE and STDOUT - if modulepath is None: - modulepath = PROCESS_MODULE_SUBPROCESS_PATH - - fromlist = ['Popen', 'PIPE', 'STDOUT'] - if extendfromlist is not None: - fromlist.extend(extendfromlist) - - self._process_modulepath = modulepath - - self._process_module = __import__(self._process_modulepath, globals(), locals(), fromlist) - - def _run(self): - """actual method - Structure - - - pre - - convert command to shell command - DONE - - chdir before start - DONE - - - start C{Popen} - DONE - - support async and subprocess - DONE - - support for - - filehandle - - PIPE - DONE - - pty - DONE - - - main - - should capture exitcode and output - - features - - separate stdout and stderr ? - - simple single run - - no timeout/waiting - DONE - - flush to - - stdout - - logger - DONE - - both stdout and logger - - process intermediate output - - qa - - input - - qa - - from file ? - - text - DONE - - - post - - parse with regexp - - raise/log error on match - - return - - return output - - log output - - write to file - - return in string - DONE - - on C{ec > 0} - - error - DONE - - raiseException - - simple - - just return True/False - """ - self._run_pre() - self._wait_for_process() - return self._run_post() - - def _run_pre(self): - """Non-blocking start""" - if self._process_module is None: - self._prep_module() - - if self.startpath is not None: - self._start_in_path() - - if self._shellcmd is None: - self._make_shell_command() - - if self._popen_named_args is None: - self._make_popen_named_args() - - self._init_process() - - self._init_input() - - def _run_post(self): - self._cleanup_process() - - self._post_exitcode() - - self._post_output() - - if self.startpath is not None: - self._return_to_previous_start_in_path() - - return self._run_return() - - def _start_in_path(self): - """Change path before the run""" - if self.startpath is None: - self.log.debug("_start_in_path: no startpath set") - return - - if os.path.exists(self.startpath): - if os.path.isdir(self.startpath): - try: - self._cwd_before_startpath = os.getcwd() # store it some one can return to it - os.chdir(self.startpath) - except: - self.raiseException("_start_in_path: failed to change path from %s to startpath %s" % - (self._cwd_before_startpath, self.startpath)) - else: - self.log.raiseExcpetion("_start_in_path: provided startpath %s exists but is no directory" % - self.startpath) - else: - self.raiseException("_start_in_path: startpath %s does not exist" % self.startpath) - - def _return_to_previous_start_in_path(self): - """Change to original path before the change to startpath""" - if self._cwd_before_startpath is None: - self.log.warning("_return_to_previous_start_in_path: previous cwd is empty. Not trying anything") - return - - if os.path.exists(self._cwd_before_startpath): - if os.path.isdir(self._cwd_before_startpath): - try: - currentpath = os.getcwd() - if not currentpath == self.startpath: - self.log.warning(("_return_to_previous_start_in_path: current diretory %s does not match " - "startpath %s") % (currentpath, self.startpath)) - os.chdir(self._cwd_before_startpath) - except: - self.raiseException(("_return_to_previous_start_in_path: failed to change path from current %s " - "to previous path %s") % (currentpath, self._cwd_before_startpath)) - else: - self.log.raiseExcpetion(("_return_to_previous_start_in_path: provided previous cwd path %s exists " - "but is no directory") % self._cwd_before_startpath) - else: - self.raiseException("_return_to_previous_start_in_path: previous cwd path %s does not exist" % - self._cwd_before_startpath) - - def _make_popen_named_args(self, others=None): - """Create the named args for Popen""" - self._popen_named_args = { - 'stdout': self._process_module.PIPE, - 'stderr': self._process_module.STDOUT, - 'stdin': self._process_module.PIPE, - 'close_fds': True, - 'shell': self.use_shell, - 'executable': self.shell, - } - if others is not None: - self._popen_named_args.update(others) - - self.log.debug("_popen_named_args %s" % self._popen_named_args) - - def _make_shell_command(self): - """Convert cmd into shell command""" - if self.cmd is None: - self.log.raiseExcpetion("_make_shell_command: no cmd set.") - - if isinstance(self.cmd, basestring): - self._shellcmd = self.cmd - elif isinstance(self.cmd, (list, tuple,)): - self._shellcmd = " ".join(self.cmd) - else: - self.log.raiseException("Failed to convert cmd %s (type %s) into shell command" % (self.cmd, type(self.cmd))) - - def _init_process(self): - """Initialise the self._process""" - try: - self._process = self._process_module.Popen(self._shellcmd, **self._popen_named_args) - except OSError: - self.log.raiseException("_init_process: init Popen shellcmd %s failed: %s" % (self._shellcmd)) - - def _init_input(self): - """Handle input, if any in a simple way""" - if self.input is not None: # allow empty string (whatever it may mean) - try: - self._process.stdin.write(self.input) - except: - self.log.raiseException("_init_input: Failed write input %s to process" % self.input) - - if self.INIT_INPUT_CLOSE: - self._process.stdin.close() - self.log.debug("_init_input: process stdin closed") - else: - self.log.debug("_init_input: process stdin NOT closed") - - def _wait_for_process(self): - """The main loop - This one has most simple loop - """ - try: - self._process_exitcode = self._process.wait() - self._process_output = self._read_process(-1) # -1 is read all - except: - self.log.raiseException("_wait_for_process: problem during wait exitcode %s output %s" % - (self._process_exitcode, self._process_output)) - - def _cleanup_process(self): - """Cleanup any leftovers from the process""" - - def _read_process(self, readsize=None): - """Read from process, return out""" - if readsize is None: - readsize = self.readsize - if readsize is None: - readsize = -1 # read all - self.log.debug("_read_process: going to read with readsize %s" % readsize) - out = self._process.stdout.read(readsize) - return out - - def _post_exitcode(self): - """Postprocess the exitcode in self._process_exitcode""" - if not self._process_exitcode == 0: - self._post_exitcode_log_failure("_post_exitcode: problem occured with cmd %s: output %s" % - (self.cmd, self._process_output)) - else: - self.log.debug("_post_exitcode: success cmd %s: output %s" % (self.cmd, self._process_output)) - - def _post_output(self): - """Postprocess the output in self._process_output""" - pass - - def _run_return(self): - """What to return""" - return self._process_exitcode, self._process_output - - def _killtasks(self, tasks=None, sig=signal.SIGKILL, kill_pgid=False): - """ - Kill all tasks - @param: tasks list of processids - @param: sig, signal to use to kill - @apram: kill_pgid, send kill to group - """ - if tasks is None: - self.log.error("killtasks no tasks passed") - elif isinstance(tasks, basestring): - try: - tasks = [int(tasks)] - except: - self.log.error("killtasks failed to convert tasks string %s to int" % tasks) - - for pid in tasks: - pgid = os.getpgid(pid) - try: - os.kill(int(pid), sig) - if kill_pgid: - os.killpg(pgid, sig) - self.log.debug("Killed %s with signal %s" % (pid, sig)) - except OSError, err: - # ERSCH is no such process, so no issue - if not err.errno == errno.ESRCH: - self.log.error("Failed to kill %s: %s" % (pid, err)) - except Exception, err: - self.log.error("Failed to kill %s: %s" % (pid, err)) - - def stop_tasks(self): - """Cleanup current run""" - self._killtasks(tasks=[self._process.pid]) - try: - os.waitpid(-1, os.WNOHANG) - except: - pass - - -class RunNoWorries(Run): - """When the exitcode is >0, log.debug instead of log.error""" - def __init__(self, cmd, **kwargs): - super(RunNoWorries, self).__init__(cmd, **kwargs) - self._post_exitcode_log_failure = self.log.debug - - -class RunLoopException(Exception): - def __init__(self, code, output): - self.code = code - self.output = output - - def __str__(self): - return "%s code %s output %s" % (self.__class__.__name__, self.code, self.output) - - -class RunLoop(Run): - """Main process is a while loop which reads the output in blocks - need to read from time to time. - otherwise the stdout/stderr buffer gets filled and it all stops working - """ - LOOP_TIMEOUT_INIT = 0.1 - LOOP_TIMEOUT_MAIN = 1 - - def __init__(self, cmd, **kwargs): - super(RunLoop, self).__init__(cmd, **kwargs) - self._loop_count = None - self._loop_continue = None # intial state, change this to break out the loop - - def _wait_for_process(self): - """Loop through the process in timesteps - collected output is run through _loop_process_output - """ - # these are initialised outside the function (cannot be forgotten, but can be overwritten) - self._loop_count = 0 # internal counter - self._loop_continue = True - self._process_output = '' - - # further initialisation - self._loop_initialise() - - time.sleep(self.LOOP_TIMEOUT_INIT) - ec = self._process.poll() - try: - while self._loop_continue and ec < 0: - output = self._read_process() - self._process_output += output - # process after updating the self._process_ vars - self._loop_process_output(output) - - if len(output) == 0: - time.sleep(self.LOOP_TIMEOUT_MAIN) - ec = self._process.poll() - - self._loop_count += 1 - - self.log.debug("_wait_for_process: loop stopped after %s iterations (ec %s loop_continue %s)" % - (self._loop_count, ec, self._loop_continue)) - - # read remaining data (all of it) - output = self._read_process(-1) - - self._process_output += output - self._process_exitcode = ec - - # process after updating the self._process_ vars - self._loop_process_output_final(output) - except RunLoopException, err: - self.log.debug('RunLoopException %s' % err) - self._process_output = err.output - self._process_exitcode = err.code - - def _loop_initialise(self): - """Initialisation before the loop starts""" - pass - - def _loop_process_output(self, output): - """Process the output that is read in blocks - simplest form: do nothing - """ - pass - - def _loop_process_output_final(self, output): - """Process the remaining output that is read - simplest form: do the same as _loop_process_output - """ - self._loop_process_output(output) - - -class RunLoopLog(RunLoop): - LOOP_LOG_LEVEL = logging.INFO - - def _wait_for_process(self): - # initialise the info logger - self.log.info("Going to run cmd %s" % self._shellcmd) - super(RunLoopLog, self)._wait_for_process() - - def _loop_process_output(self, output): - """Process the output that is read in blocks - send it to the logger. The logger need to be stream-like - """ - self.log.streamLog(self.LOOP_LOG_LEVEL, output) - super(RunLoopLog, self)._loop_process_output(output) - - -class RunLoopStdout(RunLoop): - - def _loop_process_output(self, output): - """Process the output that is read in blocks - send it to the stdout - """ - sys.stdout.write(output) - sys.stdout.flush() - super(RunLoopStdout, self)._loop_process_output(output) - - -class RunAsync(Run): - """Async process class""" - - def _prep_module(self, modulepath=None, extendfromlist=None): - # these will provide the required Popen, PIPE and STDOUT - if modulepath is None: - modulepath = PROCESS_MODULE_ASYNCPROCESS_PATH - if extendfromlist is None: - extendfromlist = ['send_all', 'recv_some'] - super(RunAsync, self)._prep_module(modulepath=modulepath, extendfromlist=extendfromlist) - - def _read_process(self, readsize=None): - """Read from async process, return out""" - if readsize is None: - readsize = self.readsize - - if self._process.stdout is None: - # Nothing yet/anymore - return '' - - try: - if readsize is not None and readsize < 0: - # read all blocking (it's not why we should use async - out = self._process.stdout.read() - else: - # non-blocking read (readsize is a maximum to return ! - out = self._process_module.recv_some(self._process, maxread=readsize) - return out - except (IOError, Exception): - # recv_some may throw Exception - self.log.exception("_read_process: read failed") - return '' - - -class RunFile(Run): - """Popen to filehandle""" - def __init__(self, cmd, **kwargs): - self.filename = kwargs.pop('filename', None) - self.filehandle = None - super(RunFile, self).__init__(cmd, **kwargs) - - def _make_popen_named_args(self, others=None): - if others is None: - if os.path.exists(self.filename): - if os.path.isfile(self.filename): - self.log.warning("_make_popen_named_args: going to overwrite existing file %s" % self.filename) - elif os.path.isdir(self.filename): - self.raiseException(("_make_popen_named_args: writing to filename %s impossible. Path exists and " - "is a directory.") % self.filename) - else: - self.raiseException("_make_popen_named_args: path exists and is not a file or directory %s" % - self.filename) - else: - dirname = os.path.dirname(self.filename) - if dirname and not os.path.isdir(dirname): - try: - os.makedirs(dirname) - except: - self.log.raiseException(("_make_popen_named_args: dirname %s for file %s does not exists. " - "Creating it failed.") % (dirname, self.filename)) - - try: - self.filehandle = open(self.filename, 'w') - except: - self.log.raiseException("_make_popen_named_args: failed to open filehandle for file %s" % self.filename) - - others = { - 'stdout': self.filehandle, - } - - super(RunFile, self)._make_popen_named_args(others=others) - - def _cleanup_process(self): - """Close the filehandle""" - try: - self.filehandle.close() - except: - self.log.raiseException("_cleanup_process: failed to close filehandle for filename %s" % self.filename) - - def _read_process(self, readsize=None): - """Meaningless for filehandle""" - return '' - - -class RunPty(Run): - """Pty support (eg for screen sessions)""" - def _read_process(self, readsize=None): - """This does not work for pty""" - return '' - - def _make_popen_named_args(self, others=None): - if others is None: - (master, slave) = pty.openpty() - others = { - 'stdin': slave, - 'stdout': slave, - 'stderr': slave - } - super(RunPty, self)._make_popen_named_args(others=others) - - -class RunTimeout(RunLoop, RunAsync): - """Question/Answer processing""" - - def __init__(self, cmd, **kwargs): - self.timeout = float(kwargs.pop('timeout', None)) - self.start = time.time() - super(RunTimeout, self).__init__(cmd, **kwargs) - - def _loop_process_output(self, output): - """""" - time_passed = time.time() - self.start - if self.timeout is not None and time_passed > self.timeout: - self.log.debug("Time passed %s > timeout %s." % (time_passed, self.timeout)) - self.stop_tasks() - - # go out of loop - raise RunLoopException(RUNRUN_TIMEOUT_EXITCODE, RUNRUN_TIMEOUT_OUTPUT) - super(RunTimeout, self)._loop_process_output(output) - - -class RunQA(RunLoop, RunAsync): - """Question/Answer processing""" - LOOP_MAX_MISS_COUNT = 20 - INIT_INPUT_CLOSE = False - CYCLE_ANSWERS = True - - def __init__(self, cmd, **kwargs): - """ - Add question and answer style running - @param qa: dict with exact questions and answers - @param qa_reg: dict with (named) regex-questions and answers (answers can contain named string templates) - @param no_qa: list of regex that can block the output, but is not seen as a question. - - Regular expressions are compiled, just pass the (raw) text. - """ - qa = kwargs.pop('qa', {}) - qa_reg = kwargs.pop('qa_reg', {}) - no_qa = kwargs.pop('no_qa', []) - self._loop_miss_count = None # maximum number of misses - self._loop_previous_ouput_length = None # track length of output through loop - - super(RunQA, self).__init__(cmd, **kwargs) - - self.qa, self.qa_reg, self.no_qa = self._parse_qa(qa, qa_reg, no_qa) - - def _parse_qa(self, qa, qa_reg, no_qa): - """ - process the QandA dictionary - - given initial set of Q and A (in dict), return dict of reg. exp. and A - - - make regular expression that matches the string with - - replace whitespace - - replace newline - - qa_reg: question is compiled as is, and whitespace+ending is added - - provided answers can be either strings or lists of strings (which will be used iteratively) - """ - - def escape_special(string): - specials = '.*+?(){}[]|\$^' - return re.sub(r"([%s])" % ''.join(['\%s' % x for x in specials]), r"\\\1", string) - - SPLIT = '[\s\n]+' - REG_SPLIT = re.compile(r"" + SPLIT) - - def process_answers(answers): - """Construct list of newline-terminated answers (as strings).""" - if isinstance(answers, basestring): - answers = [answers] - elif isinstance(answers, list): - # list is manipulated when answering matching question, so take a copy - answers = answers[:] - else: - msg_tmpl = "Invalid type for answer, not a string or list: %s (%s)" - self.log.raiseException(msg_tmpl % (type(answers), answers), exception=TypeError) - # add optional split at the end - for i in [idx for idx, a in enumerate(answers) if not a.endswith('\n')]: - answers[i] += '\n' - return answers - - def process_question(question): - """Convert string question to regex.""" - split_q = [escape_special(x) for x in REG_SPLIT.split(question)] - reg_q_txt = SPLIT.join(split_q) + SPLIT.rstrip('+') + "*$" - reg_q = re.compile(r"" + reg_q_txt) - if reg_q.search(question): - return reg_q - else: - # this is just a sanity check on the created regex, can this actually occur? - msg_tmpl = "_parse_qa process_question: question %s converted in %s does not match itself" - self.log.raiseException(msg_tmpl % (question.pattern, reg_q_txt), exception=ValueError) - - new_qa = {} - self.log.debug("new_qa: ") - for question, answers in qa.items(): - reg_q = process_question(question) - new_qa[reg_q] = process_answers(answers) - self.log.debug("new_qa[%s]: %s" % (reg_q.pattern.__repr__(), answers)) - - new_qa_reg = {} - self.log.debug("new_qa_reg: ") - for question, answers in qa_reg.items(): - reg_q = re.compile(r"" + question + r"[\s\n]*$") - new_qa_reg[reg_q] = process_answers(answers) - self.log.debug("new_qa_reg[%s]: %s" % (reg_q.pattern.__repr__(), answers)) - - # simple statements, can contain wildcards - new_no_qa = [re.compile(r"" + x + r"[\s\n]*$") for x in no_qa] - self.log.debug("new_no_qa: %s" % [x.pattern.__repr__() for x in new_no_qa]) - - return new_qa, new_qa_reg, new_no_qa - - def _loop_initialise(self): - """Initialisation before the loop starts""" - self._loop_miss_count = 0 - self._loop_previous_ouput_length = 0 - - def _loop_process_output(self, output): - """Process the output that is read in blocks - check the output passed to questions available - """ - hit = False - - self.log.debug('output %s all_output %s' % (output, self._process_output)) - - # qa first and then qa_reg - nr_qa = len(self.qa) - for idx, (question, answers) in enumerate(self.qa.items() + self.qa_reg.items()): - res = question.search(self._process_output) - if output and res: - answer = answers[0] % res.groupdict() - if len(answers) > 1: - prev_answer = answers.pop(0) - if self.CYCLE_ANSWERS: - answers.append(prev_answer) - self.log.debug("New answers list for question %s: %s" % (question.pattern, answers)) - self.log.debug("_loop_process_output: answer %s question %s (std: %s) out %s" % - (answer, question.pattern, idx >= nr_qa, self._process_output[-50:])) - self._process_module.send_all(self._process, answer) - hit = True - break - - if not hit: - curoutlen = len(self._process_output) - if curoutlen > self._loop_previous_ouput_length: - # still progress in output, just continue (but don't reset miss counter either) - self._loop_previous_ouput_length = curoutlen - else: - noqa = False - for r in self.no_qa: - if r.search(self._process_output): - self.log.debug("_loop_process_output: no_qa found for out %s" % self._process_output[-50:]) - noqa = True - if not noqa: - self._loop_miss_count += 1 - else: - self._loop_miss_count = 0 # rreset miss counter on hit - - if self._loop_miss_count > self.LOOP_MAX_MISS_COUNT: - self.log.debug("loop_process_output: max misses LOOP_MAX_MISS_COUNT %s reached. End of output: %s" % - (self.LOOP_MAX_MISS_COUNT, self._process_output[-500:])) - self.stop_tasks() - - # go out of loop - raise RunLoopException(RUNRUN_QA_MAX_MISS_EXITCODE, self._process_output) - super(RunQA, self)._loop_process_output(output) - - -class RunAsyncLoop(RunLoop, RunAsync): - """Async read in loop""" - pass - - -class RunAsyncLoopLog(RunLoopLog, RunAsync): - """Async read, log to logger""" - pass - - -class RunQALog(RunLoopLog, RunQA): - """Async loop QA with LoopLog""" - pass - - -class RunQAStdout(RunLoopStdout, RunQA): - """Async loop QA with LoopLogStdout""" - pass - - -class RunAsyncLoopStdout(RunLoopStdout, RunAsync): - """Async read, flush to stdout""" - pass - - -# convenient names -# eg: from vsc.utils.run import trivial - -run_simple = Run.run -run_simple_noworries = RunNoWorries.run - -run_async = RunAsync.run -run_asyncloop = RunAsyncLoop.run -run_timeout = RunTimeout.run - -run_to_file = RunFile.run -run_async_to_stdout = RunAsyncLoopStdout.run - -run_qa = RunQA.run -run_qalog = RunQALog.run -run_qastdout = RunQAStdout.run - -if __name__ == "__main__": - run_simple('echo ok') diff --git a/vsc/utils/wrapper.py b/vsc/utils/wrapper.py deleted file mode 100644 index 537c1f252b..0000000000 --- a/vsc/utils/wrapper.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -This work is licensed under a Creative Commons Attribution-ShareAlike 3.0 Unported License -with attribution required - -Original code by http://stackoverflow.com/users/416467/kindall from answer 4 of -http://stackoverflow.com/questions/9057669/how-can-i-intercept-calls-to-pythons-magic-methods-in-new-style-classes -""" -class Wrapper(object): - """Wrapper class that provides proxy access to an instance of some - internal instance.""" - - __wraps__ = None - __ignore__ = "class mro new init setattr getattr getattribute" - - def __init__(self, obj): - if self.__wraps__ is None: - raise TypeError("base class Wrapper may not be instantiated") - elif isinstance(obj, self.__wraps__): - self._obj = obj - else: - raise ValueError("wrapped object must be of %s" % self.__wraps__) - - # provide proxy access to regular attributes of wrapped object - def __getattr__(self, name): - return getattr(self._obj, name) - - # create proxies for wrapped object's double-underscore attributes - class __metaclass__(type): - def __init__(cls, name, bases, dct): - - def make_proxy(name): - def proxy(self, *args): - return getattr(self._obj, name) - return proxy - - type.__init__(cls, name, bases, dct) - if cls.__wraps__: - ignore = set("__%s__" % n for n in cls.__ignore__.split()) - for name in dir(cls.__wraps__): - if name.startswith("__"): - if name not in ignore and name not in dct: - setattr(cls, name, property(make_proxy(name)))