Harden telemetry delivery and refine recipe metadata

docs/Privacy.md

@@ -13,4 +13,4 @@ In addition, Olive may collect additional telemetry data such as:
 - Performance data
 - Exception information
 
-Collection of this additional telemetry can be disabled by adding the `--disable_telemetry` flag to any Olive CLI command, or by setting the `OLIVE_DISABLE_TELEMETRY` environment variable to `1` before running. Telemetry is also automatically disabled when a CI/CD environment is detected (e.g., GitHub Actions, Azure Pipelines, Jenkins). If telemetry is enabled, but cannot be sent to Microsoft, it will be stored locally and sent when a connection is available. You can override the default cache location by setting the `OLIVE_TELEMETRY_CACHE_DIR` environment variable to a valid directory path.
+Collection of this additional telemetry can be disabled by adding the `--disable_telemetry` flag to any Olive CLI command, or by setting the `OLIVE_DISABLE_TELEMETRY` environment variable to `1` before running. In CI/CD environments (e.g., GitHub Actions, Azure Pipelines, Jenkins), Olive suppresses the general heartbeat/action/error events and only emits the `OliveRecipe` event. The `OliveRecipe` event may include recipe metadata such as pass types, explicitly configured target settings, the host system type (including the default `LocalSystem` host) and any explicitly configured host accelerator settings, whether a custom package config was provided, a redacted snapshot of custom package-config overrides, and a redacted snapshot of explicitly supplied config overrides. Outside CI/CD environments, if telemetry is enabled but cannot be sent to Microsoft, it will be stored locally and sent when a connection is available. You can override the default cache location by setting the `OLIVE_TELEMETRY_CACHE_DIR` environment variable to a valid directory path.

olive/cli/base.py

@@ -115,7 +115,7 @@ def _run_workflow(self):
                     mark_test_output_path(self.args.output_path)
                 print("Dry run mode enabled. Configuration file is generated but no optimization is performed.")
                 return None
-            workflow_output = olive_run(run_config)
+            workflow_output = olive_run(run_config, recipe_telemetry_metadata=self._get_recipe_telemetry_metadata())
             if getattr(self.args, "test", None) not in (None, False):
                 mark_test_output_path(self.args.output_path)
             if not workflow_output.has_output_model():
@@ -124,6 +124,17 @@ def _run_workflow(self):
                 print(f"Model is saved at {self.args.output_path}")
             return workflow_output
 
+    def _get_recipe_telemetry_metadata(self) -> dict[str, str]:
+        recipe_name = self.__class__.__name__
+        if recipe_name.endswith("Command"):
+            recipe_name = recipe_name[: -len("Command")]
+        return {
+            "recipe_name": recipe_name,
+            "recipe_command": recipe_name,
+            "recipe_source": "generated_cli",
+            "recipe_format": "generated",
+        }
+
     @staticmethod
     def _parse_extra_options(kv_items):
         from onnxruntime_genai import __version__ as OrtGenaiVersion

olive/cli/run.py

@@ -53,16 +53,22 @@ def register_subcommand(parser: ArgumentParser):
 
     @action
     def run(self):
+        from copy import deepcopy
+        from pathlib import Path
+
         from olive.common.config_utils import load_config_file
         from olive.workflows import run as olive_run
 
         # allow the run_config to be a dict already (for api use)
-        run_config = self.args.run_config
-        if not isinstance(run_config, dict):
-            run_config = load_config_file(run_config)
+        run_config_input = self.args.run_config
+        run_config = (
+            deepcopy(run_config_input) if isinstance(run_config_input, dict) else load_config_file(run_config_input)
+        )
+        config_overrides = {}
         if input_model_config := get_input_model_config(self.args, required=False):
             print("Replacing input model config in run config")
             run_config["input_model"] = input_model_config
+            config_overrides["input_model"] = input_model_config
         elif self.args.test not in (None, False):
             input_model = run_config.get("input_model")
             if not isinstance(input_model, dict) or input_model.get("type", "").lower() != "hfmodel":
@@ -79,6 +85,19 @@ def run(self):
                 run_config.get("engine", {}).pop(rc_key, None)
                 # add value to run config directly
                 run_config[rc_key] = arg_value
+                config_overrides[rc_key] = arg_value
+
+        recipe_telemetry_metadata = {
+            "recipe_command": "WorkflowRun",
+            "recipe_source": "config_dict" if isinstance(run_config_input, dict) else "config_file",
+            "recipe_format": "dict"
+            if isinstance(run_config_input, dict)
+            else Path(run_config_input).suffix.lstrip(".").lower() or "unknown",
+            "execution_mode": "list_required_packages" if self.args.list_required_packages else "run",
+            "package_config_provided": bool(self.args.package_config),
+        }
+        if config_overrides:
+            recipe_telemetry_metadata["config_overrides"] = config_overrides
 
         output_path = run_config.get("output_dir") or run_config.get("engine", {}).get("output_dir")
         validate_test_output_path(output_path, self.args.test)
@@ -89,6 +108,7 @@ def run(self):
             list_required_packages=self.args.list_required_packages,
             tempdir=self.args.tempdir,
             package_config=self.args.package_config,
+            recipe_telemetry_metadata=recipe_telemetry_metadata,
         )
         if self.args.test not in (None, False):
             mark_test_output_path(output_path)

olive/systems/docker/docker_system.py

@@ -232,6 +232,8 @@ def _prepare_run_params(self) -> dict:
 
     def _prepare_environment(self, base_env) -> dict:
         """Prepare environment variables for container."""
+        from olive.telemetry.telemetry import is_ci_environment
+
         # Convert list to dict if needed
         if isinstance(base_env, list):
             environment = {env.split("=")[0]: env.split("=")[1] for env in base_env}
@@ -241,6 +243,8 @@ def _prepare_environment(self, base_env) -> dict:
         # Add default environment variables
         environment.setdefault("PYTHONPYCACHEPREFIX", "/tmp")
         environment["OLIVE_LOG_LEVEL"] = logging.getLevelName(logger.getEffectiveLevel())
+        if is_ci_environment():
+            environment["CI"] = "1"
 
         # Add HuggingFace token if needed
         if self.hf_token:

olive/systems/docker/workflow_runner.py

@@ -20,7 +20,7 @@ def runner_entry(config):
         config = json.load(f)
 
     logger.info("Running workflow with config: %s", config)
-    olive_run(config)
+    olive_run(config, emit_recipe_telemetry=False)
 
 
 if __name__ == "__main__":

olive/telemetry/constants.py

@@ -3,6 +3,6 @@
 # Licensed under the MIT License.
 # --------------------------------------------------------------------------
 
-"""OneCollector connection string."""
+"""Telemetry constants."""
 
-CONNECTION_STRING = "SW5zdHJ1bWVudGF0aW9uS2V5PTlkNWRkYWVjNjFlMjQ1NjdiNzg4YTIwYWVhMzI0NjMxLTcyMzdkN2M2LWVlNjEtNGNmZC1iYjdiLTU5MDNhOTcyYzJlNC03MDQ3"
+CONNECTION_STRING = "SW5zdHJ1bWVudGF0aW9uS2V5PTYyMTUwOTExZGMwMDRmYzliYjY3YmE5NjA2NDI3ZTU2LWVjNjFmOWFmLTVkN2EtNGQxOS1hZjMxLWI5Y2Q2OWU5ODdmMS02OTE1"

olive/telemetry/library/options.py

@@ -62,6 +62,7 @@ class OneCollectorExporterOptions:
     """Configuration options for OneCollector exporter."""
 
     connection_string: Optional[str] = None
+    service_name: Optional[str] = None
     transport_options: OneCollectorTransportOptions = field(default_factory=OneCollectorTransportOptions)
 
     # Internal fields populated during validation

olive/telemetry/library/telemetry_logger.py

@@ -6,6 +6,7 @@
 """High-level telemetry logger facade for easy usage."""
 
 import logging
+import threading
 import uuid
 from typing import Any, Callable, Optional
 
@@ -28,6 +29,8 @@ class TelemetryLogger:
 
     _instance: Optional["TelemetryLogger"] = None
     _default_logger: Optional["TelemetryLogger"] = None
+    _instance_lock = threading.RLock()
+    _default_logger_lock = threading.RLock()
     _logger: Optional[logging.Logger] = None
     _logger_exporter: Optional[OneCollectorLogExporter] = None
     _logger_provider: Optional[LoggerProvider] = None
@@ -40,8 +43,10 @@ def __new__(cls, options: Optional[OneCollectorExporterOptions] = None):
 
         """
         if cls._instance is None:
-            cls._instance = super().__new__(cls)
-            cls._instance._initialize(options)
+            with cls._instance_lock:
+                if cls._instance is None:
+                    cls._instance = super().__new__(cls)
+                    cls._instance._initialize(options)
 
         return cls._instance
 
@@ -57,10 +62,13 @@ def _initialize(self, options: Optional[OneCollectorExporterOptions]) -> None:
             self._logger_exporter = OneCollectorLogExporter(options=options)
 
             # Create logger provider
+            service_name = (
+                options.service_name if options and options.service_name else __name__.split(".", maxsplit=1)[0]
+            )
             self._logger_provider = LoggerProvider(
                 resource=Resource.create(
                     {
-                        "service.name": __name__.split(".", maxsplit=1)[0],
+                        "service.name": service_name,
                         "service.version": VERSION,
                         "service.instance.id": str(uuid.uuid4()),  # Unique instance ID; can double as session ID
                     }
@@ -141,43 +149,54 @@ def shutdown(self) -> None:
             self._logger_provider.shutdown()
 
     @classmethod
-    def get_default_logger(cls, connection_string: Optional[str] = None) -> "TelemetryLogger":
+    def get_default_logger(
+        cls, connection_string: Optional[str] = None, service_name: Optional[str] = None
+    ) -> "TelemetryLogger":
         """Get or create the default telemetry logger.
 
         Args:
             connection_string: OneCollector connection string (only used on first call)
+            service_name: Logical application/service name for emitted telemetry (only used on first call)
 
         Returns:
             TelemetryLogger instance
 
         """
         if cls._default_logger is None:
-            options = None
-            if connection_string:
-                options = OneCollectorExporterOptions(connection_string=connection_string)
-            cls._default_logger = cls(options=options)
+            with cls._default_logger_lock:
+                if cls._default_logger is None:
+                    options = None
+                    if connection_string:
+                        options = OneCollectorExporterOptions(
+                            connection_string=connection_string, service_name=service_name
+                        )
+                    cls._default_logger = cls(options=options)
 
         return cls._default_logger
 
     @classmethod
     def shutdown_default_logger(cls) -> None:
         """Shutdown the default telemetry logger."""
-        if cls._default_logger:
-            cls._default_logger.shutdown()
-            cls._default_logger = None
+        with cls._default_logger_lock:
+            if cls._default_logger:
+                cls._default_logger.shutdown()
+                cls._default_logger = None
 
 
-def get_telemetry_logger(connection_string: Optional[str] = None) -> TelemetryLogger:
+def get_telemetry_logger(
+    connection_string: Optional[str] = None, service_name: Optional[str] = None
+) -> TelemetryLogger:
     """Get or create the default telemetry logger.
 
     Args:
         connection_string: OneCollector connection string (only used on first call)
+        service_name: Logical application/service name for emitted telemetry (only used on first call)
 
     Returns:
         TelemetryLogger instance
 
     """
-    return TelemetryLogger.get_default_logger(connection_string=connection_string)
+    return TelemetryLogger.get_default_logger(connection_string=connection_string, service_name=service_name)
 
 
 def log_event(event_name: str, attributes: Optional[dict[str, Any]] = None) -> None: