feat(clp-package)!: Add telemetry consent, configuration, and docs. (#2251)

Co-authored-by: Junhao Liao <junhao@junhao.ca>
Co-authored-by: Quinn Taylor Mitchell <q.mitchell@mail.utoronto.ca>
Co-authored-by: Junhao Liao <junhao.liao@yscope.com>

Author: 3 people

components/clp-package-utils/clp_package_utils/controller.py

@@ -1155,7 +1155,7 @@ def get_or_create_instance_id(clp_config: ClpConfig) -> str:
         with open(resolved_instance_id_file_path, "r") as f:
             instance_id = f.readline()
     else:
-        instance_id = str(uuid.uuid4())[-4:]
+        instance_id = str(uuid.uuid4())
         with open(resolved_instance_id_file_path, "w") as f:
             f.write(instance_id)
 

components/clp-package-utils/clp_package_utils/general.py

@@ -1,6 +1,7 @@
 import enum
 import errno
 import json
+import logging
 import os
 import pathlib
 import re
@@ -11,6 +12,7 @@
 from enum import auto
 
 import yaml
+from ruamel.yaml import YAML
 from clp_py_utils.clp_config import (
     CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
     CLP_DEFAULT_CREDENTIALS_FILE_PATH,
@@ -54,6 +56,8 @@
 S3_KEY_PREFIX_COMPRESSION = "s3-key-prefix"
 S3_OBJECT_COMPRESSION = "s3-object"
 
+logger = logging.getLogger(__name__)
+
 
 class DockerDependencyError(OSError):
     """Base class for errors related to Docker dependencies."""
@@ -781,6 +785,49 @@ def _is_docker_compose_project_running(project_name: str) -> bool:
         ) from e
 
 
+def set_yaml_key(file_path: pathlib.Path, key: str, value: str) -> None:
+    """
+    Sets a possibly nested key in a YAML file, preserving formatting and comments.
+
+    :param file_path:
+    :param key: Dot-separated key path (e.g., ``"telemetry.disable"``).
+    :param value:
+    """
+    ryaml = YAML()
+    section, _, sub_key = key.partition(".")
+    parsed_value = ryaml.load(value)
+
+    if file_path.exists():
+        content = ryaml.load(file_path)
+        if content is None:
+            # File has only comments so can't round-trip through ruamel.yaml.
+            # Append as text to preserve existing comments.
+            text = file_path.read_text().rstrip("\n")
+            if sub_key:
+                text += f"\n\n{section}:\n  {sub_key}: {value}\n"
+            else:
+                text += f"\n\n{key}: {value}\n"
+            try:
+                file_path.write_text(text)
+                logger.info("Config updated in %s", file_path)
+            except OSError:
+                logger.warning("Failed to update config %s", file_path, exc_info=True)
+            return
+    else:
+        content = {}
+
+    if sub_key:
+        content.setdefault(section, {})[sub_key] = parsed_value
+    else:
+        content[section] = parsed_value
+
+    try:
+        ryaml.dump(content, file_path)
+        logger.info("Config updated in %s", file_path)
+    except OSError:
+        logger.warning("Failed to update config %s", file_path, exc_info=True)
+
+
 def _validate_data_directory(data_dir: pathlib.Path, component_name: str) -> None:
     try:
         validate_path_could_be_dir(resolve_host_path_in_container(data_dir))

components/clp-package-utils/clp_package_utils/scripts/start_clp.py

@@ -2,18 +2,21 @@
 """Script to start the CLP Package."""
 
 import logging
+import os
 import pathlib
 import sys
 
 import click
-from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH
+
+from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ClpConfig
 from clp_py_utils.core import resolve_host_path_in_container
 
 from clp_package_utils.cli_utils import RESTART_POLICY
 from clp_package_utils.controller import DockerComposeController, get_or_create_instance_id
 from clp_package_utils.general import (
     get_clp_home,
     load_config_file,
+    set_yaml_key,
     validate_and_load_db_credentials_file,
     validate_and_load_queue_credentials_file,
     validate_and_load_redis_credentials_file,
@@ -23,6 +26,29 @@
 
 logger = logging.getLogger(__name__)
 
+# Values accepted by both CLP_DISABLE_TELEMETRY and DO_NOT_TRACK to disable telemetry.
+_TELEMETRY_DISABLE_VALUES = ("1", "true", "yes", "y")
+
+
+TELEMETRY_NOTICE = """
+================================================================================
+CLP collects anonymous operational metrics to help improve the software. This
+includes: CLP version, OS/architecture, deployment method, bytes ingested,
+compression ratios, query volume, and more. It does NOT include: log content,
+queries, hostnames, IP addresses, or any other Personally Identifiable
+Information (PII).
+
+Telemetry is sent to: https://telemetry.yscope.io
+For details, see: https://docs.yscope.com/clp/main/user-docs/reference-telemetry
+
+You can disable metrics at any time by setting the environment variable
+CLP_DISABLE_TELEMETRY=true. Network admins can also block
+https://telemetry.yscope.io at the firewall level.
+================================================================================
+"""
+
+TELEMETRY_PROMPT = "Enable anonymous telemetry to help improve CLP? [Y/n] "
+
 
 @click.command()
 @click.option(
@@ -85,6 +111,8 @@ def main(
         logger.exception("Failed to load config.")
         sys.exit(1)
 
+    _handle_telemetry_consent(clp_config, resolved_config_path)
+
     try:
         # Create necessary directories.
         resolve_host_path_in_container(clp_config.data_directory).mkdir(parents=True, exist_ok=True)
@@ -115,5 +143,48 @@ def main(
         sys.exit(1)
 
 
+def _handle_telemetry_consent(clp_config: ClpConfig, config_file_path: pathlib.Path) -> None:
+    """
+    Handles telemetry consent and prompts the user on first run if needed.
+
+    Priority order for handling telemetry preference:
+    1. Session-only environment variable overrides. e.g.,
+       a. CLP_DISABLE_TELEMETRY=true
+       b. DO_NOT_TRACK=true
+    2. Config file opt-out
+    3. Previously confirmed telemetry preference
+    4. First run consent prompt in interactive sessions
+
+    :param config_file_path: for persisting consent.
+    """
+    clp_disable_val = os.environ.get("CLP_DISABLE_TELEMETRY", "").strip().lower()
+    dnt_val = os.environ.get("DO_NOT_TRACK", "").strip().lower()
+    if clp_disable_val in _TELEMETRY_DISABLE_VALUES or dnt_val in _TELEMETRY_DISABLE_VALUES:
+        clp_config.telemetry.disable = True
+        return
+
+    if clp_config.telemetry.disable:
+        return
+
+    # Skip prompt if not the first run. i.e., telemetry preference has been confirmed previously.
+    instance_id_file = clp_config.logs_directory / "instance-id"
+    if resolve_host_path_in_container(instance_id_file).exists():
+        return
+
+    if not sys.stdin.isatty():
+        return
+
+    print(TELEMETRY_NOTICE)
+    try:
+        response = input(TELEMETRY_PROMPT).strip().lower()
+    except EOFError:
+        # e.g., Ctrl+D
+        response = "n"
+
+    if response.startswith("n"):
+        clp_config.telemetry.disable = True
+        set_yaml_key(config_file_path, "telemetry.disable", "true")
+
+
 if "__main__" == __name__:
     main()

components/clp-package-utils/pyproject.toml

@@ -13,6 +13,7 @@ dependencies = [
     "pydantic>=2.12.5",
     "pymongo>=4.16.0",
     "PyYAML>=6.0.3",
+    "ruamel.yaml>=0.19.1",
 ]
 
 [build-system]