feat(AIC-3449): refactor AI Config reference into a dataclass

packages/client/src/launchdarkly_ai_server/__init__.py

@@ -26,6 +26,7 @@
     set_conversation_id_if_absent,
 )
 from .evaluations import (
+    AIConfig,
     Criterion,
     DatasetRow,
     EvalRunResult,
@@ -184,6 +185,7 @@
     "to_semconv_finish_reason",
     "VariationMeta",
     # evaluations
+    "AIConfig",
     "EvalRunResult",
     "Criterion",
     "DatasetRow",

packages/client/src/launchdarkly_ai_server/evaluations/__init__.py

@@ -11,10 +11,18 @@
 )
 from .criteria import Criterion, Judge, Scorer, SuccessDirection
 from .module import EvaluationsModule, init_evaluations
-from .types import DatasetRow, EvalRunResult, GenerationConfig, RunSummary, Usage
+from .types import (
+    AIConfig,
+    DatasetRow,
+    EvalRunResult,
+    GenerationConfig,
+    RunSummary,
+    Usage,
+)
 
 __all__ = [
     "DEFAULT_BASE_URI",
+    "AIConfig",
     "Criterion",
     "DatasetRow",
     "EvalRunResult",

packages/client/src/launchdarkly_ai_server/evaluations/module.py

@@ -25,7 +25,7 @@
     _provides_for,
     _segment,
 )
-from .types import EvalRunResult, GenerationConfig, RunSummary
+from .types import AIConfig, EvalRunResult, GenerationConfig, RunSummary
 
 logger = logging.getLogger(__name__)
 
@@ -128,8 +128,7 @@ async def run(
         dataset: str,
         handler: EvalHandler,
         generation: GenerationConfig | None = None,
-        ai_config: str | None = None,
-        variation: str | None = None,
+        ai_config: AIConfig | None = None,
         tools: Mapping[str, ToolImplementation] | None = None,
         criteria: list[Criterion] | None = None,
         judge_handlers: list[EvalHandler] | None = None,
@@ -158,7 +157,7 @@ async def run(
         and a wider ``poll_interval_seconds``; both default to
         ``SUMMARY_POLL_TIMEOUT_SECONDS`` / ``SUMMARY_POLL_INTERVAL_SECONDS``.
 
-        Pass ``ai_config`` and ``variation`` to start from an existing AI Config
+        Pass ``ai_config`` to start from an existing AI Config
         variation instead of a hand-built ``generation``. Its model, provider,
         parameters, prompt and output format become the defaults, and anything
         set in ``generation`` overrides them field by field (``parameters``
@@ -180,18 +179,21 @@ async def run(
             poll_interval_seconds=poll_interval_seconds,
             poll_timeout_seconds=poll_timeout_seconds,
         )
-        self._validate_config_source(
-            generation=generation, ai_config=ai_config, variation=variation
-        )
+        self._validate_config_source(generation=generation, ai_config=ai_config)
         pinned_tool_versions: dict[str, int] = {}
-        if ai_config is not None and variation is not None:
+        config_label = ""
+        if ai_config is not None:
+            config_label = f"{ai_config.key!r}/{ai_config.variation!r}"
             ai_config_variation = await asyncio.to_thread(
-                self._runner._fetch_config_variation, project_key, ai_config, variation
+                self._runner._fetch_config_variation,
+                project_key,
+                ai_config.key,
+                ai_config.variation,
             )
             generation = _merge_generation(ai_config_variation.generation, generation)
             if tools is None and ai_config_variation.tool_versions:
                 raise EvaluationsError(
-                    f"AI Config variation {ai_config!r}/{variation!r} uses tools "
+                    f"AI Config variation {config_label} uses tools "
                     "with no implementation: "
                     + ", ".join(
                         repr(name) for name in ai_config_variation.tool_versions
@@ -226,10 +228,9 @@ async def run(
             resolved_tool = resolved_tools.get(tool_key)
             if resolved_tool is not None and resolved_tool.version != pinned_version:
                 logger.warning(
-                    "AI Config variation %r/%r pins tool %r at version %d; "
+                    "AI Config variation %s pins tool %r at version %d; "
                     "evaluating against the latest version %d.",
-                    ai_config,
-                    variation,
+                    config_label,
                     tool_key,
                     pinned_version,
                     resolved_tool.version,
@@ -461,22 +462,20 @@ def _validate_run_args(
     def _validate_config_source(
         *,
         generation: GenerationConfig | None,
-        ai_config: str | None,
-        variation: str | None,
+        ai_config: AIConfig | None,
     ) -> None:
         """Require a generation source before any request is made."""
-        if ai_config is None and variation is None:
+        if ai_config is None:
             if generation is None:
                 raise EvaluationsError(
-                    "Pass generation, or ai_config and variation to evaluate an "
-                    "existing AI Config variation"
+                    "Pass generation, or ai_config to evaluate an existing AI "
+                    "Config variation"
                 )
             return
-        if ai_config is None:
-            raise EvaluationsError("variation requires ai_config")
-        if variation is None:
-            raise EvaluationsError("ai_config requires variation")
-        for name, value in (("ai_config", ai_config), ("variation", variation)):
+        for name, value in (
+            ("ai_config.key", ai_config.key),
+            ("ai_config.variation", ai_config.variation),
+        ):
             if not value.strip():
                 raise EvaluationsError(f"{name} must not be blank")
 
@@ -485,8 +484,8 @@ def _validate_generation(generation: GenerationConfig | None) -> GenerationConfi
         """Check the final generation settings, after any fetched variation is merged."""
         if generation is None:
             raise EvaluationsError(
-                "Pass generation, or ai_config and variation to evaluate an "
-                "existing AI Config variation"
+                "Pass generation, or ai_config to evaluate an existing AI "
+                "Config variation"
             )
         provider = generation.get("provider")
         model = generation.get("model")

packages/client/src/launchdarkly_ai_server/evaluations/types.py

@@ -67,6 +67,21 @@ class ResolvedTool:
     schema: dict[str, Any] = field(default_factory=dict)
 
 
+@dataclass(frozen=True)
+class AIConfig:
+    """Identifies an existing AI Config variation to seed an evaluation run.
+
+    ``key`` is the AI Config key and ``variation`` is the variation key. The two
+    are only meaningful together, since a variation key is scoped to its config.
+    ``run()`` reads the variation's latest version from the management API.
+    This is a reference, not the config itself; see ``AiConfigRep`` for the
+    evaluated payload.
+    """
+
+    key: str
+    variation: str
+
+
 @dataclass
 class AIConfigVariation:
     """An AI Config variation read from the management API as run() defaults.

packages/client/tests/test_evaluations_run.py

@@ -10,6 +10,7 @@
 
 from launchdarkly_ai_server import create_handler
 from launchdarkly_ai_server.evaluations import (
+    AIConfig,
     DatasetRow,
     EvaluationsError,
     HttpResponse,
@@ -2340,8 +2341,7 @@ async def handler(config: dict[str, Any], *args: object) -> dict[str, Any]:
         key="eval-key",
         dataset="golden",
         handler=handler,
-        ai_config="support-agent",
-        variation="control",
+        ai_config=AIConfig(key="support-agent", variation="control"),
     )
 
     assert result.passed is True
@@ -2378,8 +2378,7 @@ async def handler(*args: object) -> dict[str, Any]:
         key="eval-key",
         dataset="golden",
         handler=handler,
-        ai_config="support-agent",
-        variation="control",
+        ai_config=AIConfig(key="support-agent", variation="control"),
         generation={
             "model": "gpt-4o-mini",
             "parameters": {"temperature": 0.1},
@@ -2417,8 +2416,7 @@ async def test_variation_tools_without_implementations_fail_before_mutating_requ
             key="eval-key",
             dataset="golden",
             handler=successful_handler,
-            ai_config="support-agent",
-            variation="control",
+            ai_config=AIConfig(key="support-agent", variation="control"),
         )
 
     assert [request["method"] for request in transport.requests] == ["GET", "GET"]
@@ -2465,8 +2463,7 @@ async def handler(*args: object) -> dict[str, Any]:
             key="eval-key",
             dataset="golden",
             handler=handler,
-            ai_config="support-agent",
-            variation="control",
+            ai_config=AIConfig(key="support-agent", variation="control"),
         )
 
     assert [request["method"] for request in transport.requests] == ["GET", "GET"]
@@ -2485,8 +2482,7 @@ async def test_unknown_variation_fails_before_any_records_are_created() -> None:
             key="eval-key",
             dataset="golden",
             handler=successful_handler,
-            ai_config="support-agent",
-            variation="missing",
+            ai_config=AIConfig(key="support-agent", variation="missing"),
         )
 
     assert [request["method"] for request in transport.requests] == ["GET"]
@@ -2497,13 +2493,18 @@ async def test_unknown_variation_fails_before_any_records_are_created() -> None:
     ("source", "message"),
     [
         ({}, "Pass generation"),
-        ({"variation": "control"}, "variation requires ai_config"),
-        ({"ai_config": "support-agent"}, "ai_config requires variation"),
-        ({"ai_config": " ", "variation": "control"}, "ai_config must not be blank"),
+        (
+            {"ai_config": AIConfig(key=" ", variation="control")},
+            "ai_config.key must not be blank",
+        ),
+        (
+            {"ai_config": AIConfig(key="support-agent", variation="")},
+            "ai_config.variation must not be blank",
+        ),
     ],
 )
 async def test_config_source_is_validated_before_network_io(
-    source: dict[str, str], message: str
+    source: dict[str, AIConfig], message: str
 ) -> None:
     transport = SequencedTransport([])
     evals = init_evaluations(api_token="token", transport=transport)
@@ -2546,8 +2547,7 @@ async def handler(*args: object) -> dict[str, Any]:
         key="eval-key",
         dataset="golden",
         handler=handler,
-        ai_config="support-agent",
-        variation="control",
+        ai_config=AIConfig(key="support-agent", variation="control"),
         tools={"lookup_order": lookup_order},
     )
 
@@ -2574,8 +2574,7 @@ async def test_non_string_model_config_key_fails_loudly(
             key="eval-key",
             dataset="golden",
             handler=successful_handler,
-            ai_config="support-agent",
-            variation="control",
+            ai_config=AIConfig(key="support-agent", variation="control"),
         )
 
     assert [request["method"] for request in transport.requests] == ["GET"]
@@ -2598,8 +2597,7 @@ async def test_variation_without_a_model_config_needs_an_explicit_provider(
             key="eval-key",
             dataset="golden",
             handler=successful_handler,
-            ai_config="support-agent",
-            variation="control",
+            ai_config=AIConfig(key="support-agent", variation="control"),
         )
 
     assert [request["method"] for request in transport.requests] == ["GET"]