Add an inspect_ai eval engine behind --engine

.github/workflows/selftest.yml

@@ -44,6 +44,116 @@ jobs:
       - name: Run the suite
         run: python -m unittest discover -s tests -t . --verbose
 
+  engine:
+    name: Inspect engine (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # Both sandboxes a run can get. Ubuntu has Docker, which is the default
+        # and what CI uses; Windows has none, which is the other supported
+        # shape. The Windows leg is the only place the cross-platform tools are
+        # exercised at all -- inspect's own assume a POSIX guest.
+        os: [ubuntu-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          # inspect-ai pulls in the order of eighty packages, so the download
+          # is most of this step. Keyed on pyproject, which is where the extra
+          # is declared and the only thing that changes what gets installed.
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install with the inspect extra
+        run: python -m pip install --upgrade pip && python -m pip install ".[inspect]"
+
+      # `mockllm` reaches no provider, so this needs no key and runs on a pull
+      # request from a fork. It cannot tell us whether a skill is any good --
+      # the mock does no work and every judged expectation fails. What it does
+      # tell us is whether the machinery around the model still holds together
+      # on both platforms, which is where every bug so far has been.
+      - name: Build a repo to test
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p fixture/demo-skill/evals fixture/demo-skill/fixtures
+          cat > fixture/demo-skill/SKILL.md <<'EOF'
+          ---
+          name: demo-skill
+          description: Does demonstrable things, for a test that needs a skill.
+          ---
+          EOF
+          echo "seeded by the case, not produced by the agent" \
+            > fixture/demo-skill/fixtures/seeded.txt
+          cat > fixture/demo-skill/evals/evals.json <<'EOF'
+          {
+            "evaluations": [
+              {"id": "demo-a", "skill_should_trigger": true, "prompt": "do the demo thing",
+               "workspace": "fixtures", "files_exist": ["seeded.txt"]},
+              {"id": "demo-b", "skill_should_trigger": true, "prompt": "another way to ask"},
+              {"id": "demo-c", "skill_should_trigger": true, "prompt": "a third phrasing"},
+              {"id": "demo-d", "skill_should_trigger": false, "prompt": "something adjacent"},
+              {"id": "demo-e", "skill_should_trigger": false, "prompt": "something else entirely"}
+            ]
+          }
+          EOF
+
+      # `SKILLSCOPE_REPO` rather than `working-directory`, because a `--skills-dir`
+      # glob that was passed is resolved against the repo root and `find_root`
+      # takes that from the nearest `.git`. Running from a subdirectory of a
+      # checkout would therefore still glob the checkout. Same thing `repo:`
+      # does for the action.
+      #
+      # Exits non-zero because the mock satisfies nothing; the report is the
+      # artifact under test, not the exit code.
+      - name: Grade the fixture on the inspect engine
+        continue-on-error: true
+        env:
+          SKILLSCOPE_REPO: fixture
+        run: >
+          python -m skillscope behavioral --engine inspect
+          --model mockllm/model --skills-dir '*' --skill demo-skill
+          --output engine-report.json
+
+      - name: Check the machinery held
+        shell: python
+        env:
+          EXPECTED_SANDBOX: ${{ matrix.os == 'windows-latest' && 'local' || 'docker' }}
+        run: |
+          import json
+          import os
+
+          report = json.load(open("engine-report.json", encoding="utf-8"))
+          meta, totals = report["meta"], report["totals"]
+          print(json.dumps(meta, indent=2))
+
+          # An infrastructure failure is not a graded result. This is what
+          # catches a sandbox that would not start.
+          assert totals["errors"] == 0, report["cases"]
+
+          expected = os.environ["EXPECTED_SANDBOX"]
+          assert meta["sandbox"] == expected, f"ran in {meta['sandbox']!r}, wanted {expected!r}"
+
+          checks = [c for case in report["cases"] for c in case["checks"]]
+          assert checks, "nothing was graded, so nothing was proven"
+
+          # A sandbox that cannot be listed reports the same shape as an agent
+          # that produced nothing, so the difference is asserted explicitly.
+          broken = [c for c in checks if "could not list the sandbox" in (c["detail"] or "")]
+          assert not broken, broken
+
+          # The seeded file exists because the case put it there, not because
+          # the mock did anything. It passing proves the whole path: the fixture
+          # was staged into the working directory, the directory was listed, and
+          # the listing was matched against what the case asked for.
+          seeded = [c for c in checks if c["kind"] == "files_exist"]
+          assert seeded, "the seeded-file check did not run"
+          assert all(c["passed"] for c in seeded), seeded
+          print(f"{len(checks)} checks graded, seeded fixture found, sandbox {expected}.")
+
   action:
     name: Action against a throwaway repo
     runs-on: ubuntu-latest

docs/usage.md

@@ -208,6 +208,69 @@ Legs with a scoped environment run as a separate job, because a job's
 credentials are fixed before its matrix expands. A repo that declares no scoped
 environment gets one matrix, labels and all.
 
+## Which engine grades a run
+
+`--engine` chooses what actually runs the cases. The dataset, the CLI and the
+reports are identical whichever you pick; only the thing driving the agent
+changes.
+
+| `--engine` | What runs | Needs |
+| --- | --- | --- |
+| `legacy` (default) | The `claude` CLI, driven directly | the CLI on `PATH` |
+| `inspect` | A harness-independent agent through `inspect_ai` | `pip install 'skillscope[inspect]'` |
+| `claude-code` | Real Claude Code inside the sandbox, to cross-check the other two | `skillscope[verify]`, Linux only |
+
+`inspect` grades a skill on whether its *instructions* work rather than on how
+one product reads them, which is the stronger claim and the one a product repo
+can adopt. It is also much cheaper: a routing case is a single model call,
+because the decision is visible in the first reply and nothing needs executing.
+
+`claude-code` is a reporting leg, never a gate. Harness runs are
+nondeterministic and the harness is not what is being graded, so a divergence
+there is a question about the skill rather than a build failure.
+
+### Where an `inspect` run is sandboxed
+
+Two separate decisions, made by different people.
+
+**Which provider** is a property of the runner, chosen with
+`SKILLSCOPE_SANDBOX`. Docker by default; `podman` on a host that has that
+instead; `local` to skip the container. `local` is for working locally rather
+than for CI, because a graded run that quietly dropped its sandbox would report
+the same numbers with none of the isolation.
+
+Podman needs three things, and each was discovered by the next one failing:
+
+* `pip install 'skillscope[podman]'`. The provider is registered by a separate
+  package through an entry point, so the podman binary alone is not enough.
+* `podman-compose`, and `INSPECT_PODMAN_COMPOSE=podman-compose`. Bare
+  `podman compose` is a shim that delegates to whichever compose provider it
+  finds, which on a host that also has Docker is Docker's -- and that then
+  talks to a daemon podman was chosen to avoid.
+* A search registry, because podman will not guess one. Docker assumes Docker
+  Hub for an image name with no registry; podman refuses, and the default
+  sandbox image is named without one. `unqualified-search-registries =
+  ["docker.io"]` in `/etc/containers/registries.conf`.
+
+Podman is worth the setup where the runner's user cannot reach the Docker
+socket, since it is daemonless and rootless and needs neither that nor group
+membership.
+
+**What the sandbox must provide** is a property of the skill, declared as
+`sandbox: compose.yaml` in its `evals/machine.yml`. Skills get a container with
+no network by default; one that installs a server or pulls a model cannot run
+that way and says so. Selecting a provider does not discard what a skill asked
+for -- the compose file rides along.
+
+Windows is the exception to both: inspect's sandbox layer and every tool built
+on it assume a POSIX guest, so those legs run unsandboxed and trade isolation
+for running on the platform they are meant to test.
+
+To see what changing engine would do to your own datasets before changing it,
+[`tools/benchmark_engines.py`](../tools/benchmark_engines.py) runs the same
+cases through two engines and reports per-case agreement, measured against how
+much one engine already disagrees with itself.
+
 ## In CI: one job
 
 [`reusable.yml`](../.github/workflows/reusable.yml) grades a repo's skills with

pyproject.toml

@@ -15,16 +15,34 @@ requires-python = ">=3.10"
 license = { text = "MIT" }
 authors = [{ name = "Advanced Micro Devices, Inc." }]
 
-# The runner itself is standard library only, so a run needs no wheels beyond
-# this package. PyYAML is the one exception: it reads the optional
+# The legacy runner is standard library only, so a graded run needs no wheels
+# beyond this package. PyYAML is the one exception: it reads the optional
 # evals/machine.yml, which only CI planning touches.
 dependencies = ["pyyaml>=6.0"]
 
+# The inspect-backed engine (`--engine inspect`). Kept an extra so the default
+# path keeps the stdlib-only property while both engines are shipped: a repo
+# that has not migrated installs nothing new.
+[project.optional-dependencies]
+# `anthropic` is listed explicitly: inspect-ai treats every model provider as
+# optional, so installing it alone gets you a harness that cannot reach a model.
+inspect = ["inspect-ai>=0.3.263", "anthropic>=0.40"]
+
+# For a runner that has podman rather than docker. The provider registers
+# itself through an `inspect_ai` entry point, so installing it is the whole
+# setup; `SKILLSCOPE_SANDBOX=podman` then selects it.
+podman = ["skillscope[inspect]", "inspect-podman"]
+
+# The Claude Code verification leg (`--engine claude-code`). Separate from
+# `inspect` because it is a reporting-only cross-check, not something a graded
+# run needs -- and because it only works on a POSIX guest.
+verify = ["skillscope[inspect]", "inspect-swe>=0.2.70"]
+
 [project.scripts]
 skillscope = "skillscope.cli:main"
 
 [tool.setuptools]
-packages = ["skillscope"]
+packages = ["skillscope", "skillscope.engine"]
 
 [tool.setuptools.package-data]
 skillscope = ["data/*.json", "schema/*.json"]

skillscope/agent.py

@@ -41,7 +41,7 @@
 from dataclasses import dataclass
 from pathlib import Path
 
-from . import datasets, deadline
+from . import datasets, deadline, usage
 
 DEFAULT_MODEL = os.environ.get("SKILLSCOPE_MODEL", "opus")
 DEFAULT_EFFORT = os.environ.get("SKILLSCOPE_EFFORT", "high")
@@ -70,10 +70,18 @@ def is_automated_env() -> bool:
     )
 
 
+# Model providers that reach no cloud service. The CI pin exists to keep paid
+# runs comparable between runs; one of these grades nothing and costs nothing,
+# so pinning it only turns a free wiring check into a run that needs a key.
+NO_PROVIDER_PREFIXES = ("mockllm",)
+
+
 def enforce_model_policy(model: str | None) -> str | None:
     """Coerce non-opus models to opus in CI; pass through otherwise."""
     if model is None or not is_automated_env() or "opus" in model.lower():
         return model
+    if model.lower().startswith(NO_PROVIDER_PREFIXES):
+        return model
     _safe_print(
         f"[skillscope] automated run: coercing model '{model}' -> "
         f"'{AUTOMATED_MODEL}' to pin the CI model."
@@ -387,6 +395,9 @@ def __init__(self, *, workspace: Path, events: list[dict], judge_model: str | No
 
         result_text = ""
         for ev in events:
+            # Recording what the run spent is what lets it be compared against
+            # the same cases on the other engine.
+            usage.record_stream_event(ev)
             if ev.get("type") == "result" and isinstance(ev.get("result"), str):
                 result_text = ev["result"]
 

skillscope/behavior.py

@@ -210,6 +210,25 @@ def summarize(outcomes: list[BehaviorOutcome], meta: dict) -> dict:
     }
 
 
+def _isolation_note(meta: dict) -> str:
+    """One line saying whether the agent was contained while it worked.
+
+    Behavioral runs the agent to completion with permissions bypassed, so
+    whether it was isolated changes what the numbers cost to obtain. A report
+    that omits it reads as though it were, and the answer differs per platform:
+    the Windows legs have no sandbox available at all.
+    """
+    where = meta.get("sandbox")
+    if where is None:
+        return ""
+    if meta.get("sandbox_isolated"):
+        return f"Cases ran isolated, in `{where}`."
+    return (
+        f"**Cases ran unsandboxed** (`{where}`): the agent worked directly in "
+        "the harness's own filesystem, with permissions bypassed."
+    )
+
+
 def render_markdown(summary: dict) -> str:
     totals = summary["totals"]
     meta = summary["meta"]
@@ -220,6 +239,8 @@ def render_markdown(summary: dict) -> str:
         f"({totals['checks_passed']}/{totals['checks']} individual expectations) "
         f"on `{meta['model']}` (effort `{meta['effort']}`).",
         "",
+        _isolation_note(meta),
+        "",
         "| Skill | Cases | Passed | Expectations | Met |",
         "| --- | --- | --- | --- | --- |",
     ]