Track usage, errors, and create disable_telemetry flag

README.md

@@ -114,6 +114,9 @@ The sample chat app to run is found as [model-chat.py](https://github.com/micros
 - [Documentation](https://microsoft.github.io/Olive)
 - [Recipes](https://github.com/microsoft/olive-recipes)
 
+## Data/Telemetry
+Distributions of this project may collect usage data and send it to Microsoft to help improve our products and services. See the [privacy statement](docs/Privacy.md) for more details.
+
 ## 🤝 Contributions and Feedback
 - We welcome contributions! Please read the [contribution guidelines](./CONTRIBUTING.md) for more details on how to contribute to the Olive project.
 - For feature requests or bug reports, file a [GitHub Issue](https://github.com/microsoft/Olive/issues).

docs/Privacy.md

@@ -0,0 +1,16 @@
+# Privacy
+
+## Data Collection
+The software may collect information about you and your use of the software and send it to Microsoft. Microsoft may use this information to provide services and improve our products and services. You may turn off the telemetry as described in the repository. There are also some features in the software that may enable Microsoft to collect data from users of your applications. If you use these features, you must comply with applicable law, including providing appropriate notices to users of your applications together with a copy of Microsoft's privacy statement. Our privacy statement can be found [here](https://go.microsoft.com/fwlink/?LinkID=824704). You can learn more about data collection and use in the help documentation and our privacy statement. Your use of the software operates as your consent to these practices.
+
+***
+
+## Technical Details
+Olive uses the [OpenTelemetry](https://opentelemetry.io/) API for its implementation. Telemetry is turned ON by default. Based on user consent, this data may be periodically sent to Microsoft servers following GDPR and privacy regulations for anonymity and data access controls. Application, device, and version information is collected automatically.
+
+In addition, Olive may collect additional telemetry data such as:
+- Invoked commands
+- Performance data
+- Exception information
+
+Collection of this additional telemetry can be disabled by adding the `--disable_telemetry` flag to any Olive CLI command, or by setting the `OLIVE_DISABLE_TELEMETRY` environment variable to `1` before running. If telemetry is enabled, but cannot be sent to Microsoft, it will be stored locally and sent when a connection is available. You can override the default cache location by setting the `OLIVE_TELEMETRY_CACHE_PATH` environment variable to a valid file path.

olive/__init__.py

@@ -14,8 +14,6 @@
 _logger.addHandler(_sc)
 _logger.propagate = False
 
-__version__ = "0.11.0.dev0"
-
 # pylint: disable=C0413
 
 # Import Python API functions
@@ -33,10 +31,12 @@
     tune_session_params,
 )
 from olive.engine.output import ModelOutput, WorkflowOutput  # noqa: E402
+from olive.version import __version__  # noqa: E402
 
 __all__ = [
     "ModelOutput",
     "WorkflowOutput",
+    "__version__",
     # Python API functions
     "benchmark",
     "capture_onnx_graph",

olive/cli/auto_opt.py

@@ -14,6 +14,7 @@
     add_logging_options,
     add_save_config_file_options,
     add_shared_cache_options,
+    add_telemetry_options,
     get_input_model_config,
     update_accelerator_options,
     update_shared_cache_options,
@@ -22,6 +23,7 @@
 from olive.constants import Precision
 from olive.hardware.constants import ExecutionProvider
 from olive.package_config import OlivePackageConfig
+from olive.telemetry import action
 
 
 class AutoOptCommand(BaseOliveCLICommand):
@@ -167,8 +169,10 @@ def register_subcommand(parser: ArgumentParser):
         add_shared_cache_options(sub_parser)
         add_logging_options(sub_parser)
         add_save_config_file_options(sub_parser)
+        add_telemetry_options(sub_parser)
         sub_parser.set_defaults(func=AutoOptCommand)
 
+    @action
     def run(self):
         return self._run_workflow()
 

olive/cli/base.py

@@ -631,6 +631,12 @@ def add_search_options(sub_parser: ArgumentParser):
     search_strategy_group.add_argument("--seed", type=int, default=0, help="Random seed for search sampler")
 
 
+def add_telemetry_options(sub_parser: ArgumentParser):
+    """Add telemetry options to the sub_parser."""
+    sub_parser.add_argument("--disable_telemetry", action="store_true", help="Disable telemetry for this command.")
+    return sub_parser
+
+
 def update_search_options(args, config):
     to_replace = []
     to_replace.extend(

olive/cli/benchmark.py

@@ -11,10 +11,12 @@
     add_logging_options,
     add_save_config_file_options,
     add_shared_cache_options,
+    add_telemetry_options,
     get_input_model_config,
     update_shared_cache_options,
 )
 from olive.common.utils import set_nested_dict_value
+from olive.telemetry import action
 
 
 class BenchmarkCommand(BaseOliveCLICommand):
@@ -69,8 +71,10 @@ def register_subcommand(parser: ArgumentParser):
         add_logging_options(sub_parser)
         add_save_config_file_options(sub_parser)
         add_shared_cache_options(sub_parser)
+        add_telemetry_options(sub_parser)
         sub_parser.set_defaults(func=BenchmarkCommand)
 
+    @action
     def run(self):
         return self._run_workflow()
 

olive/cli/capture_onnx.py

@@ -13,12 +13,14 @@
     add_logging_options,
     add_save_config_file_options,
     add_shared_cache_options,
+    add_telemetry_options,
     get_diffusers_input_model,
     get_input_model_config,
     update_shared_cache_options,
 )
 from olive.common.utils import set_nested_dict_value
 from olive.model.utils.diffusers_utils import is_valid_diffusers_model
+from olive.telemetry import action
 
 
 class ModelBuilderAccuracyLevel(IntEnum):
@@ -170,8 +172,10 @@ def register_subcommand(parser: ArgumentParser):
         add_logging_options(sub_parser)
         add_save_config_file_options(sub_parser)
         add_shared_cache_options(sub_parser)
+        add_telemetry_options(sub_parser)
         sub_parser.set_defaults(func=CaptureOnnxGraphCommand)
 
+    @action
     def run(self):
         return self._run_workflow()
 

olive/cli/configure_qualcomm_sdk.py

@@ -4,7 +4,8 @@
 # --------------------------------------------------------------------------
 from argparse import ArgumentParser
 
-from olive.cli.base import BaseOliveCLICommand
+from olive.cli.base import BaseOliveCLICommand, add_telemetry_options
+from olive.telemetry import action
 
 
 class ConfigureQualcommSDKCommand(BaseOliveCLICommand):
@@ -21,9 +22,10 @@ def register_subcommand(parser: ArgumentParser):
             required=True,
             choices=["3.6", "3.8"],
         )
-
+        add_telemetry_options(sub_parser)
         sub_parser.set_defaults(func=ConfigureQualcommSDKCommand)
 
+    @action
     def run(self):
         from olive.platform_sdk.qualcomm.configure.configure import configure
 

olive/cli/convert_adapters.py

@@ -6,8 +6,9 @@
 from argparse import ArgumentParser
 from typing import TYPE_CHECKING
 
-from olive.cli.base import BaseOliveCLICommand, add_logging_options
+from olive.cli.base import BaseOliveCLICommand, add_logging_options, add_telemetry_options
 from olive.common.utils import WeightsFileFormat, save_weights
+from olive.telemetry import action
 
 if TYPE_CHECKING:
     from numpy.typing import NDArray
@@ -75,8 +76,10 @@ def register_subcommand(parser: ArgumentParser):
             help="Quantization mode for int4 quantization of adapter weights. Default is symmetric.",
         )
         add_logging_options(sub_parser)
+        add_telemetry_options(sub_parser)
         sub_parser.set_defaults(func=ConvertAdaptersCommand)
 
+    @action
     def run(self):
         import torch
         from peft import LoraConfig, load_peft_weights

olive/cli/diffusion_lora.py

@@ -11,11 +11,13 @@
     add_logging_options,
     add_save_config_file_options,
     add_shared_cache_options,
+    add_telemetry_options,
     update_shared_cache_options,
 )
 from olive.common.utils import set_nested_dict_value
 from olive.constants import DiffusersModelVariant
 from olive.passes.diffusers.lora import LRSchedulerType, MixedPrecision
+from olive.telemetry import action
 
 
 class DiffusionLoraCommand(BaseOliveCLICommand):
@@ -237,8 +239,10 @@ def register_subcommand(parser: ArgumentParser):
         add_shared_cache_options(sub_parser)
         add_logging_options(sub_parser)
         add_save_config_file_options(sub_parser)
+        add_telemetry_options(sub_parser)
         sub_parser.set_defaults(func=DiffusionLoraCommand)
 
+    @action
     def run(self):
         return self._run_workflow()
 

olive/cli/extract_adapters.py

@@ -6,8 +6,9 @@
 
 from huggingface_hub.constants import HF_HUB_CACHE
 
-from olive.cli.base import BaseOliveCLICommand, add_logging_options
+from olive.cli.base import BaseOliveCLICommand, add_logging_options, add_telemetry_options
 from olive.common.utils import WeightsFileFormat, save_weights
+from olive.telemetry import action
 
 
 class ExtractAdaptersCommand(BaseOliveCLICommand):
@@ -54,8 +55,10 @@ def register_subcommand(parser: ArgumentParser):
             help="Cache dir to store temporary files in. Default is Hugging Face's default cache dir.",
         )
         add_logging_options(sub_parser)
+        add_telemetry_options(sub_parser)
         sub_parser.set_defaults(func=ExtractAdaptersCommand)
 
+    @action
     def run(self):
         # Reference: https://huggingface.co/microsoft/Phi-4-multimodal-instruct-onnx/blob/05f620b467891affcb00b464e5a73e7cf2de61f9/onnx/builder.py#L318
         import os

olive/cli/finetune.py

@@ -13,11 +13,13 @@
     add_logging_options,
     add_save_config_file_options,
     add_shared_cache_options,
+    add_telemetry_options,
     get_input_model_config,
     update_dataset_options,
     update_shared_cache_options,
 )
 from olive.common.utils import set_nested_dict_value
+from olive.telemetry import action
 
 
 class FineTuneCommand(BaseOliveCLICommand):
@@ -74,8 +76,10 @@ def register_subcommand(parser: ArgumentParser):
         add_shared_cache_options(sub_parser)
         add_logging_options(sub_parser)
         add_save_config_file_options(sub_parser)
+        add_telemetry_options(sub_parser)
         sub_parser.set_defaults(func=FineTuneCommand)
 
+    @action
     def run(self):
         return self._run_workflow()
 

olive/cli/generate_adapter.py

@@ -11,11 +11,13 @@
     add_logging_options,
     add_save_config_file_options,
     add_shared_cache_options,
+    add_telemetry_options,
     get_input_model_config,
     update_shared_cache_options,
 )
 from olive.common.utils import WeightsFileFormat, set_nested_dict_value
 from olive.passes.onnx.common import AdapterType
+from olive.telemetry import action
 
 
 class GenerateAdapterCommand(BaseOliveCLICommand):
@@ -45,8 +47,10 @@ def register_subcommand(parser: ArgumentParser):
         add_logging_options(sub_parser)
         add_save_config_file_options(sub_parser)
         add_shared_cache_options(sub_parser)
+        add_telemetry_options(sub_parser)
         sub_parser.set_defaults(func=GenerateAdapterCommand)
 
+    @action
     def run(self):
         return self._run_workflow()
 

olive/cli/generate_cost_model.py

@@ -5,8 +5,9 @@
 import logging
 from pathlib import Path
 
-from olive.cli.base import BaseOliveCLICommand, add_input_model_options, get_input_model_config
+from olive.cli.base import BaseOliveCLICommand, add_input_model_options, add_telemetry_options, get_input_model_config
 from olive.model import ModelConfig
+from olive.telemetry import action
 
 logger = logging.getLogger(__name__)
 
@@ -34,8 +35,10 @@ def register_subcommand(parser):
             choices=PRECISON_TO_BYTES.keys(),
             help="Weight precision",
         )
+        add_telemetry_options(sub_parser)
         sub_parser.set_defaults(func=GenerateCostModelCommand)
 
+    @action
     def run(self):
         import torch
 

olive/cli/launcher.py

@@ -22,6 +22,7 @@
 from olive.cli.run_pass import RunPassCommand
 from olive.cli.session_params_tuning import SessionParamsTuningCommand
 from olive.cli.shared_cache import SharedCacheCommand
+from olive.telemetry import Telemetry
 
 
 def get_cli_parser(called_as_console_script: bool = True) -> ArgumentParser:
@@ -61,13 +62,18 @@ def main(raw_args=None, called_as_console_script: bool = True):
 
     args, unknown_args = parser.parse_known_args(raw_args)
 
+    telemetry = Telemetry()
+    if args.disable_telemetry:
+        telemetry.disable_telemetry()
+
     if not hasattr(args, "func"):
         parser.print_help()
         sys.exit(1)
 
     # Run the command
     service = args.func(parser, args, unknown_args)
     service.run()
+    telemetry.shutdown()
 
 
 def legacy_call(deprecated_module: str, command_name: str, *args):

olive/cli/optimize.py

@@ -15,11 +15,13 @@
     add_input_model_options,
     add_logging_options,
     add_save_config_file_options,
+    add_telemetry_options,
     get_input_model_config,
 )
 from olive.common.utils import set_nested_dict_value
 from olive.constants import Precision, precision_bits_from_precision
 from olive.hardware.constants import ExecutionProvider
+from olive.telemetry import action
 
 
 class OptimizeCommand(BaseOliveCLICommand):
@@ -184,6 +186,7 @@ def register_subcommand(parser: ArgumentParser):
 
         add_logging_options(sub_parser)
         add_save_config_file_options(sub_parser)
+        add_telemetry_options(sub_parser)
         sub_parser.set_defaults(func=OptimizeCommand)
 
     def __init__(self, parser: ArgumentParser, args: Namespace, unknown_args: Optional[list] = None):
@@ -216,6 +219,7 @@ def __init__(self, parser: ArgumentParser, args: Namespace, unknown_args: Option
         self.enable_compose_onnx_models = False
         self.enable_openvino_encapsulation = False
 
+    @action
     def run(self):
         return self._run_workflow()
 

olive/cli/quantize.py

@@ -17,13 +17,15 @@
     add_logging_options,
     add_save_config_file_options,
     add_shared_cache_options,
+    add_telemetry_options,
     update_dataset_options,
     update_input_model_options,
     update_shared_cache_options,
 )
 from olive.common.utils import StrEnumBase, set_nested_dict_value
 from olive.constants import Precision, QuantAlgorithm, precision_bits_from_precision
 from olive.package_config import OlivePackageConfig
+from olive.telemetry import action
 
 
 class ImplName(StrEnumBase):
@@ -94,6 +96,7 @@ def register_subcommand(parser: ArgumentParser):
         add_shared_cache_options(sub_parser)
         add_logging_options(sub_parser)
         add_save_config_file_options(sub_parser)
+        add_telemetry_options(sub_parser)
         sub_parser.set_defaults(func=QuantizeCommand)
 
     def _check_data_name_arg(self, pinfo):
@@ -210,6 +213,7 @@ def _get_run_config(self, tempdir: str) -> dict[str, Any]:
         self._customize_config(config)
         return config
 
+    @action
     def run(self):
         return self._run_workflow()