diff --git a/Makefile b/Makefile index 94cfabda458..6172491fadc 100644 --- a/Makefile +++ b/Makefile @@ -480,6 +480,8 @@ build/check-doc.build-stamp: doc/build/html/index.html doc/build/html/htmlcov/in test -s doc/build/html/autoapi/mlos_viz/index.html test -s doc/build/html/autoapi/mlos_viz/dabl/index.html grep -q -e '--config CONFIG' doc/build/html//mlos_bench.run.usage.html + # Look for malformed rst links: `LinkName `_ + if find doc/build/html -name '*.html' -print0 | xargs -0 grep -m1 '>`_' | grep -m1 .; then echo "Bad links found"; false; fi # Check doc logs for errors (but skip over some known ones) ... @cat doc/build/log.txt \ | egrep -C1 -e WARNING -e CRITICAL -e ERROR \ diff --git a/mlos_bench/mlos_bench/config/__init__.py b/mlos_bench/mlos_bench/config/__init__.py index 508d09c7a40..9743e5e104e 100644 --- a/mlos_bench/mlos_bench/config/__init__.py +++ b/mlos_bench/mlos_bench/config/__init__.py @@ -197,9 +197,40 @@ "current_dir": "$PWD", "some_expanded_var": "$some_var: $experiment_id", "location": "eastus", + + // This can be specified in the CLI config or the globals config + "tunable_params_map": { + // a map of tunable_params variables to their covariant group names + "environment1_tunables": [ + "covariant_group_name", + "another_covariant_group_name" + ], + "environment2_tunables": [ + // empty list means no tunables + // are enabled for this environment + // during this experiment + // (e.g., only use defaults for this environment) + ], } -There are additional details about variable propagation in the +Users can have multiple global config files, each specified with a ``--globals`` +CLI arg or ``"globals"`` CLI config property. + +At runtime, parameters from these files will be combined into a single +dictionary, in the order they appear, and pushed to the root +:py:class:`Environment `. + +Any global or :py:class:`~.Environment` parameter can also be overridden from +the command line, by simply specifying ``--PARAMETER_NAME PARAMETER_VALUE``. + +Another common use of global config files is to store sensitive data (e.g., +passwords, tokens, etc.) that should not be version-controlled. + +This way, users can keep their experiment-specific parameters separately from +the Environment configs making them more reusable. + +There are additional details about `Variable Propagation +<../environments/index.html#variable-propagation>`_ in the :py:mod:`mlos_bench.environments` module. Well Known Variables @@ -216,6 +247,13 @@ - ``$trial_runner_id``: A unique identifier for the ``TrialRunner``. This can be useful when running multiple trials in parallel (e.g., to provision a numbered VM per worker). +- ``$tunable_params_map``: A map of ``tunable_params`` ``$name`` to their list of covariant group names. + This is usually used in a CLI ``--config`` CLI config or ``--globals`` + (e.g., "experiment") config file and is used to control what the + ``"tunable_params": $tunable_group_name`` specified in the the + :py:mod:`mlos_bench.environments` JSONC configs resolves to. + This can be used to control which tunables are enabled for tuning for an + experiment without having to change the underlying Environment config. Tunable Configs ^^^^^^^^^^^^^^^ @@ -353,4 +391,4 @@ `mlos_bench/tests/config/README.md `_ for additional documentation and examples in the source tree. -""" +""" # pylint: disable=line-too-long # noqa: E501 diff --git a/mlos_bench/mlos_bench/config/environments/network/virtual-network-environment.jsonc b/mlos_bench/mlos_bench/config/environments/network/virtual-network-environment.jsonc index 2a9aa7691a2..bab6308459c 100644 --- a/mlos_bench/mlos_bench/config/environments/network/virtual-network-environment.jsonc +++ b/mlos_bench/mlos_bench/config/environments/network/virtual-network-environment.jsonc @@ -18,7 +18,7 @@ "config": { // FIXME: There aren't currently any tunable params for the network environment. // But one is required to workaround a bug (#613) in the config storage layer. - "tunable_params": ["dummy_params"], + "tunable_params": ["dummy_params_group1"], // Typically don't want to deprovision the network environment on teardown // since other experiments in this same RG might be using it. diff --git a/mlos_bench/mlos_bench/config/tunables/dummy-tunables.jsonc b/mlos_bench/mlos_bench/config/tunables/dummy-tunables.jsonc index 6fcfa3e6f8e..9f812e0a15e 100644 --- a/mlos_bench/mlos_bench/config/tunables/dummy-tunables.jsonc +++ b/mlos_bench/mlos_bench/config/tunables/dummy-tunables.jsonc @@ -1,6 +1,6 @@ // FIXME: A workaround to a bug (#613) in the config system that requires non-empty configs. { - "dummy_params": { + "dummy_params_group1": { "cost": 0, "description": "Dummy parameter group to allow a non-empty config for testing creating singleton resources that have no tunable params.", "params": { @@ -11,5 +11,35 @@ "default": "dummy" } } + }, + "dummy_params_group2": { + "cost": 1, + "description": "Dummy parameter group for testing", + "params": { + "dummy_param_int": { + "description": "An integer dummy parameter.", + "type": "int", + "range": [0, 100], + "default": 0 + }, + "dummy_param_float": { + "description": "A float dummy parameter.", + "type": "float", + "range": [0, 1], + "default": 0.5 + } + } + }, + "dummy_params_group3": { + "cost": 1, + "description": "Another dummy parameter group for testing", + "params": { + "dummy_param3": { + "description": "One more dummy parameter.", + "type": "float", + "range": [-1, 1], + "default": 0.0 + } + } } } diff --git a/mlos_bench/mlos_bench/environments/__init__.py b/mlos_bench/mlos_bench/environments/__init__.py index 6c8170bac28..e8519556dab 100644 --- a/mlos_bench/mlos_bench/environments/__init__.py +++ b/mlos_bench/mlos_bench/environments/__init__.py @@ -28,15 +28,53 @@ This lets Environments be very flexible in what they can accomplish. Environments can be stacked together with the :py:class:`.CompositeEnv` class to -represent complex setups (e.g., an appication running on a remote VM with a +represent complex setups (e.g., an application running on a remote VM with a benchmark running from a local machine). See below for the set of Environments currently available in this package. Note that additional ones can also be created by extending the base -:py:class:`.Environment` class and referencing them in the :py:mod:`json configs +:py:class:`~.Environment` class and referencing them in the :py:mod:`json configs ` using the ``class`` key. +Environment Parameterization +++++++++++++++++++++++++++++ + +Each :py:class:`~.Environment` can have a set of parameters that define the +environment's configuration. These parameters can be *constant* (i.e., immutable from one trial +run to the next) or *tunable* (i.e., suggested by the optimizer or provided by the user). The +following clauses in the environment configuration are used to declare these parameters: + +- ``tunable_params``: + A list of :py:mod:`tunable ` parameters' (covariant) *groups*. + At each trial, the Environment will obtain the new values of these parameters + from the outside (e.g., from the :py:mod:`Optimizer `). + + Typically, this is set using variable expansion via the special + ``tunable_params_map`` key in the `globals config + <../config/index.html#globals-and-variable-substitution>`_. + +- ``const_args``: + A dictionary of *constant* parameters along with their values. + +- ``required_args``: + A list of *constant* parameters supplied to the environment externally + (i.e., from a parent environment, global config file, or command line). + +Again, tunable parameters change on every trial, while constant parameters stay fixed for the +entire experiment. + +During the ``setup`` and ``run`` phases, MLOS will combine the constant and +tunable parameters and their values into a single dictionary and pass it to the +corresponding method. + +Values of constant parameters defined in the Environment config can be +overridden with the values from the command line and/or external config files. +That allows MLOS users to have reusable immutable environment configurations and +move all experiment-specific or sensitive data outside of the version-controlled +files. We discuss the `variable propagation `_ mechanism +in the section below. + Environment Tunables ++++++++++++++++++++ @@ -63,14 +101,262 @@ json files when the :py:class:`~mlos_bench.launcher.Launcher` processes the initial set of config files. -The ``tunable_params`` setting in the ``config`` section of the Environment config -can also be used to limit *which* of the ``TunableGroups`` should be used for the -Environment. +The ``tunable_params`` setting in the ``config`` section of the Environment config can then be +used to limit *which* of the ``TunableGroups`` should be used for the Environment. + +Tunable Parameters Map +^^^^^^^^^^^^^^^^^^^^^^ + +Although the full set of tunable parameters (and groups) of each Environment is always known in +advance, in practice we often want to limit it to a smaller subset for a given experiment. This +can be done by adding an extra level of indirection and specifying the ``tunable_params_map`` in +the global config. ``tunable_params_map`` associates a variable name with a list of +:py:class:`~mlos_bench.tunables.tunable_groups.TunableGroups` names, e.g., + + .. code-block:: json + + // experiment-globals.mlos.jsonc + { + "tunable_params_map": { + "tunables_ref1": ["tunable_group1", "tunable_group2"], + "tunables_ref2": [] // Useful to disable all tunables. + } + } + +Later, in the Environment config, we can use these variable names to refer to the +tunable groups we want to use for that Environment: + + .. code-block:: json + + // environment.mlos.jsonc + { + // ... + "config": { + "tunable_params": [ + "$tunables_ref1", // Will be replaced with "tunable_group1", "tunable_group2" + "$tunables_ref2", // A no-op + "tunable_group3" // Can still refer to a group directly. + ], + // ... etc. + +Note: this references the `dummy-tunables.jsonc +`_ +file for simplicity. + +Using such ``"$tunables_ref"`` variables in the Environment config allows us to dynamically +change the set of active ``TunableGroups`` for a given Environment using the global config +without modifying the Environment configuration files for each experiment, thus making them +more modular and composable. + +Variable Propagation +++++++++++++++++++++ + +Parameters declared in the ``const_args`` or ``required_args`` sections of the Environment +config can be overridden with values specified in the external config files or the command +line. In fact, ``const_args`` or ``required_args`` sections can be viewed as placeholders +for the parameters that are being pushed to the environment from the outside. + +The same parameter can be present in both ``const_args`` and ``required_args`` sections. +``required_args`` is just a way to emphasize the importance of the parameter and create a +placeholder for it when no default value can be specified the ``const_args`` section. +If a ``required_args`` parameter is not present in the ``const_args`` section, +and can't be resolved from the ``globals`` this allows MLOS to fail fast and +return an error to the user indicating an incomplete config. + +Note that the parameter **must** appear in the child Environment ``const_args`` or +``required_args`` section; if a parameter is not present in one of these +placeholders of the Environment config, it will not be propagated. This allows MLOS +users to have small immutable Environment configurations and combine and parameterize +them with external (global) configs. + +Taking it to the next level outside of the Environment configs, the parameters +can be defined in the external key-value JSON config files (usually referred to +as `global config files +<../config/index.html#globals-and-variable-substitution>`_ in MLOS lingo). +See :py:mod:`mlos_bench.config` for more details. + +We can summarize the parameter propagation rules as follows: + +1. An environment will only get the parameters defined in its ``const_args`` or + ``required_args`` sections. +2. Values of the parameters defined in the global config files will override the values of the + corresponding parameters in all environments. +3. Values of the command line parameters take precedence over values defined in the global or + environment configs. + +Examples +-------- +Here's a simple working example of a local environment config (written in Python +instead of JSON for testing) to show how variable propagation works: + +Note: this references the `dummy-tunables.jsonc +`_ +file for simplicity. + +>>> # globals.jsonc +>>> globals_json = ''' +... { +... "experiment_id": "test_experiment", +... +... "const_arg_from_globals_1": "Substituted from globals - 1", +... "const_arg_from_globals_2": "Substituted from globals - 2", +... +... "const_arg_from_cli_1": "Will be overridden from CLI", +... +... // Define reference names to represent tunable groups in the Environment configs. +... "tunable_params_map": { +... "tunables_ref1": ["dummy_params_group1", "dummy_params_group2"], +... "tunables_ref2": [], // Useful to disable all tunables for the Environment. +... } +... } +... ''' + +>>> # environment.jsonc +>>> environment_json = ''' +... { +... "class": "mlos_bench.environments.local.local_env.LocalEnv", +... "name": "test_env1", +... "include_tunables": [ +... "tunables/dummy-tunables.jsonc" // For simplicity, include all tunables available. +... ], +... "config": { +... "tunable_params": [ +... "$tunables_ref1", // Includes "dummy_params_group1", "dummy_params_group2" +... "$tunables_ref2", // A no-op +... "dummy_params_group3" // Can still refer to a group directly. +... ], +... "const_args": { +... // Environment-specific non-tunable constant parameters: +... "const_arg_1": "Default value of const_arg_1", +... "const_arg_from_globals_1": "To be replaced from global config", +... "const_arg_from_cli_1": "To be replaced from CLI" +... }, +... "required_args": [ +... // These parameters always come from elsewhere: +... "const_arg_from_globals_2", +... "const_arg_from_cli_2", +... // We already define these parameters in "const_args" section above; +... // mentioning them here is optional, but can be used for clarity: +... "const_arg_from_globals_1", +... "const_arg_from_cli_1" +... ], +... "run": [ +... "echo Hello world" +... ] +... } +... } +... ''' + +Now that we have our environment and global configurations, we can instantiate the +:py:class:`~.Environment` and inspect it. In this example we will simulate the command line execution to demonstrate how CLI parameters propagate to the environment. + +>>> # Load the globals and environment configs defined above via the Launcher as +>>> # if we were calling `mlos_bench` directly on the CLI. +>>> from mlos_bench.launcher import Launcher +>>> argv = [ +... "--environment", environment_json, +... "--globals", globals_json, +... # Override some values via CLI directly: +... "--const_arg_from_cli_1", "Substituted from CLI - 1", +... "--const_arg_from_cli_2", "Substituted from CLI - 2", +... ] +>>> launcher = Launcher("sample_launcher", argv=argv) +>>> env = launcher.root_environment +>>> env.name +'test_env1' + +``env`` is an instance of :py:class:`~.Environment` class that we can use to setup, run, and tear +down the environment. It also has a set of properties and methods that we can use to access the +object's parameters. This way we can check the actual runtime configuration of the environment. + +First, let's check the tunable parameters: + +>>> assert env.tunable_params.get_param_values() == { +... "dummy_param": "dummy", +... "dummy_param_int": 0, +... "dummy_param_float": 0.5, +... "dummy_param3": 0.0 +... } + +We can see the tunables from ``dummy_params_group1`` and ``dummy_params_group2`` groups specified +via ``$tunables_ref1``, as well as the tunables from ``dummy_params_group3`` that we specified +directly in the Environment config. All tunables are initialized to their default values. + +Now let's see how the variable propagation works. + +>>> env.const_args["const_arg_1"] +'Default value of const_arg_1' + +``const_arg_1`` has the value we have assigned in the ``"const_args"`` section of the +Environment config. No surprises here. -Since :py:mod:`json configs ` also support ``$variable`` -substitution in the values using the `globals` mechanism, this setting can used to -dynamically change the set of active TunableGroups for a given Experiment using only -`globals`, allowing for configs to be more modular and composable. +>>> env.const_args["const_arg_from_globals_1"] +'Substituted from globals - 1' +>>> env.const_args["const_arg_from_globals_2"] +'Substituted from globals - 2' + +``const_arg_from_globals_1`` and ``const_arg_from_globals_2`` were declared in the Environment's +``const_args`` and ``required_args`` sections, respectively. Their values were overridden by the +values from the global config. + +>>> env.const_args["const_arg_from_cli_1"] +'Substituted from CLI - 1' +>>> env.const_args["const_arg_from_cli_2"] +'Substituted from CLI - 2' + +Likewise, ``const_arg_from_cli_1`` and ``const_arg_from_cli_2`` got their values from the +command line. Note that for ``const_arg_from_cli_1`` the value from the command line takes +precedence over the values specified in the Environment's ``const_args`` section **and** the one +in the global config. + +Now let's set up the environment and see how the constant and tunable parameters get combined. +We'll also assign some non-default values to the tunables, as the optimizer would do on each +trial. + +>>> env.tunable_params["dummy_param_int"] = 99 +>>> env.tunable_params["dummy_param3"] = 0.999 +>>> with env: +... assert env.setup(env.tunable_params) +... assert env.parameters == { +... "const_arg_1": "Default value of const_arg_1", +... "const_arg_from_globals_1": "Substituted from globals - 1", +... "const_arg_from_globals_2": "Substituted from globals - 2", +... "const_arg_from_cli_1": "Substituted from CLI - 1", +... "const_arg_from_cli_2": "Substituted from CLI - 2", +... "trial_id": 1, +... "trial_runner_id": 1, +... "experiment_id": "test_experiment", +... "dummy_param": "dummy", +... "dummy_param_int": 99, +... "dummy_param_float": 0.5, +... "dummy_param3": 0.999 +... } + +These are the values visible to the implementations of the :py:meth:`~.Environment.setup`, +:py:meth:`~.Environment.run`, and :py:meth:`~.Environment.teardown` methods. We can see both +the constant and tunable parameters combined into a single dictionary +:py:attr:`~.Environment.parameters` with proper values assigned to each of them on each iteration. +When implementing a new :py:class:`~.Environment`-derived class, developers can rely on the +:py:attr:`~.Environment.parameters` data in their versions of :py:meth:`~.Environment.setup` and +other methods. For example, :py:class:`~mlos_bench.environments.remote.vm_env.VMEnv` would then +pass the :py:attr:`~.Environment.parameters` into an ARM template when provisioning a new VM, +and :py:class:`~mlos_bench.environments.local.local_env.LocalEnv` can dump them into a JSON file +specified in the ``dump_params_file`` config property, or/and cherry-pick some of these values +and make them shell variables with the ``shell_env_params``. + +A few `Well Known Parameters <../config/index.html#well-known-variables>`_ +parameters like ``trial_id`` and ``trial_runner_id`` are added by the +:py:mod:`Scheduler ` and used for trials parallelization +and storage of the results. It is sometimes useful to add them, for example, to +the paths used by the Environment, as in, e.g., +``"/storage/$experiment_id/$trial_id/data/"``, to prevent conflicts when running +multiple Experiments and Trials in parallel. + +We will discuss passing the parameters to external scripts and using them in referencing files +and directories in local and shared storage in the documentation of the concrete +:py:class:`~.Environment` implementations, especially +:py:class:`~mlos_bench.environments.script_env.ScriptEnv` and +:py:class:`~mlos_bench.environments.local.local_env.LocalEnv`. Environment Services ++++++++++++++++++++ @@ -86,10 +372,22 @@ used in different settings (e.g., local machine, SSH accessible machine, Azure VM, etc.) without having to change the Environment config. -Variable Propagation -++++++++++++++++++++ -TODO: Document how variable propagation works in the script environments using -required_args, const_args, etc. +Variable propagation rules described in the previous section for the environment +configs also apply to the :py:mod:`Service ` +configurations. + +That is, every parameter defined in the Service config can be overridden by a +corresponding parameter from the global config or the command line. + +All global configs, command line parameters, Environment ``const_args`` and +``required_args`` sections, and Service config parameters thus form one flat +name space of parameters. This imposes a certain risk of name clashes, but also +simplifies the configuration process and allows users to keep all +experiment-specific data in a few human-readable files. + +We will discuss the examples of such global and local configuration parameters in the +documentation of the concrete :py:mod:`~mlos_bench.services` and +:py:mod:`~mlos_bench.environments`. Examples -------- @@ -122,7 +420,7 @@ Overview of the Services available to the Environments and their configurations. :py:mod:`mlos_bench.tunables` : Overview of the Tunables available to the Environments and their configurations. -""" +""" # pylint: disable=line-too-long # noqa: E501 from mlos_bench.environments.base_environment import Environment from mlos_bench.environments.composite_env import CompositeEnv diff --git a/mlos_bench/mlos_bench/environments/remote/remote_env.py b/mlos_bench/mlos_bench/environments/remote/remote_env.py index d74b7fbbbf4..184df75cf6b 100644 --- a/mlos_bench/mlos_bench/environments/remote/remote_env.py +++ b/mlos_bench/mlos_bench/environments/remote/remote_env.py @@ -7,7 +7,7 @@ e.g. Application Environment -TODO: Documentat how variable propogation works in the remote environments. +TODO: Document how variable propagation works in the remote environments. """ import logging diff --git a/mlos_bench/mlos_bench/launcher.py b/mlos_bench/mlos_bench/launcher.py index d64045c823f..c728ed7fb20 100644 --- a/mlos_bench/mlos_bench/launcher.py +++ b/mlos_bench/mlos_bench/launcher.py @@ -609,7 +609,7 @@ def _load_scheduler(self, args_scheduler: str | None) -> Scheduler: return SyncScheduler( # All config values can be overridden from global config config={ - "experiment_id": "UNDEFINED - override from global config", + "experiment_id": "DEFAULT_EXPERIMENT_ID", "trial_id": 0, "config_id": -1, "trial_config_repeat_count": 1, diff --git a/mlos_bench/mlos_bench/services/config_persistence.py b/mlos_bench/mlos_bench/services/config_persistence.py index 3fea50a7e65..a674e6ce758 100644 --- a/mlos_bench/mlos_bench/services/config_persistence.py +++ b/mlos_bench/mlos_bench/services/config_persistence.py @@ -457,6 +457,9 @@ def build_environment( if env_services_path is not None: service = self.load_services(env_services_path, global_config, service) + if service is None: + service = Service(parent=self) + env_tunables_path = config.get("include_tunables") if env_tunables_path is not None: tunables = self.load_tunables(env_tunables_path, tunables) diff --git a/mlos_bench/mlos_bench/tunables/__init__.py b/mlos_bench/mlos_bench/tunables/__init__.py index 0e163d4b1ae..ae6bbc3187f 100644 --- a/mlos_bench/mlos_bench/tunables/__init__.py +++ b/mlos_bench/mlos_bench/tunables/__init__.py @@ -93,7 +93,8 @@ Environment config. Then individual covariant groups can be enabled via the ``tunable_params`` and -``tunable_params_map`` properties, possibly via ``globals`` variable expansion. +``tunable_params_map`` properties, possibly via ``globals`` `Variable Expansion +<../config/index.html#globals-and-variable-substitution>`_. See the :py:mod:`mlos_bench.config` and :py:mod:`mlos_bench.environments` module documentation for more information.