From 96cdf7723d1dd0bee8d1a3090fae15f0598dee19 Mon Sep 17 00:00:00 2001 From: Ike Nwankwo Date: Mon, 5 Oct 2026 22:58:38 +0000 Subject: [PATCH 1/4] feat(models): forward MCP OAuth scopes in claude_code and codex_cli generators --- docs/claude_code_agent_testing.md | 2 +- docs/codex_cli_agent_testing.md | 2 +- evalbench/generators/models/claude_code.py | 56 +++++++++++-- evalbench/generators/models/codex_cli.py | 43 ++++++++-- evalbench/test/claude_code_test.py | 60 +++++++++++++ evalbench/test/codex_cli_test.py | 97 ++++++++++++++++++++++ 6 files changed, 244 insertions(+), 16 deletions(-) diff --git a/docs/claude_code_agent_testing.md b/docs/claude_code_agent_testing.md index ea20ec1e..f6ca2f7e 100644 --- a/docs/claude_code_agent_testing.md +++ b/docs/claude_code_agent_testing.md @@ -284,7 +284,7 @@ EvalBench accepts the **same MCP server config schema as Gemini CLI** for HTTP s |---|---| | `httpUrl` | → `url` + auto-adds `type: "http"` | | `authProviderType: google_credentials` | → injects `Authorization: Bearer ` (from `gcloud auth application-default print-access-token`, falling back to `gcloud auth print-access-token`) **and** sets a `headersHelper` so Claude Code re-mints a fresh ADC token on every connection (avoids ~1h expiry). Google API MCP endpoints reject the plain user token on tool calls — see [Troubleshooting](#mcp-tool-call-fails-with-incompatible-auth-server-does-not-support-dynamic-client-registration). | -| `oauth.scopes` | (dropped — Claude Code doesn't use Gemini's OAuth delegation) | +| `oauth.scopes` | → forwarded as `--scopes` to `gcloud auth application-default print-access-token` in both `headersHelper` and the initial static `Authorization` header | | `headers` | → passed through as-is | | `command` / `args` (stdio) | → passed through as-is | diff --git a/docs/codex_cli_agent_testing.md b/docs/codex_cli_agent_testing.md index c634e4e9..ca857aad 100644 --- a/docs/codex_cli_agent_testing.md +++ b/docs/codex_cli_agent_testing.md @@ -305,7 +305,7 @@ EvalBench accepts the **same MCP server config schema as Gemini CLI and Claude C | `httpUrl` | → `url` (TOML, streamable HTTP server) | | `headers` | → `http_headers` (TOML inline table) | | `authProviderType: google_credentials` | → mints an **ADC** token (`gcloud auth application-default print-access-token`, falling back to `gcloud auth print-access-token`) and passes it to Codex via `bearer_token_env_var` (env var `EVALBENCH_GCLOUD_MCP_TOKEN`). The generator re-mints a fresh token before **every turn** (each `codex exec` re-reads the env var), so it doesn't expire mid-suite. `X-Goog-User-Project` stays a static `http_headers` entry. Google API MCP endpoints reject the plain user token on tool calls — see [Troubleshooting](#mcp-server-fails-with-401-unauthorized-real-cloud-sql-endpoint). | -| `oauth.scopes` | (dropped — Codex doesn't use Gemini's OAuth delegation) | +| `oauth.scopes` | → forwarded as `--scopes` to `gcloud auth application-default print-access-token` when minting `bearer_token_env_var` (`EVALBENCH_GCLOUD_MCP_TOKEN`) | | `command` / `args` / `env` / `cwd` (stdio) | → passed through as-is into a `[mcp_servers.NAME]` stdio block | ### HTTP MCP server (Cloud SQL Managed) diff --git a/evalbench/generators/models/claude_code.py b/evalbench/generators/models/claude_code.py index 012c7a6c..e0731526 100644 --- a/evalbench/generators/models/claude_code.py +++ b/evalbench/generators/models/claude_code.py @@ -290,8 +290,24 @@ def _translate_mcp_config(self, server_name: str, config: dict) -> dict: # Translate `authProviderType: google_credentials` into Bearer header auth_provider = config.pop("authProviderType", None) - # Gemini-style `oauth.scopes` is ignored by Claude Code; drop it - config.pop("oauth", None) + + # Gemini-style `oauth.scopes` is forwarded to gcloud, but the `oauth` + # block itself must be removed so Claude Code doesn't treat it as an + # interactive MCP OAuth 2.0 dynamic registration config. + oauth_config = config.pop("oauth", None) + raw_scopes = None + if isinstance(oauth_config, dict): + raw_scopes = oauth_config.get("scopes") + elif isinstance(oauth_config, (list, str)): + raw_scopes = oauth_config + if not raw_scopes and "scopes" in config: + raw_scopes = config.pop("scopes") + + scopes: list[str] = [] + if isinstance(raw_scopes, list): + scopes = [str(s).strip() for s in raw_scopes if str(s).strip()] + elif isinstance(raw_scopes, str) and raw_scopes.strip(): + scopes = [s.strip() for s in raw_scopes.split(",") if s.strip()] if auth_provider == "google_credentials": headers = config.get("headers", {}) or {} @@ -304,12 +320,12 @@ def _translate_mcp_config(self, server_name: str, config: dict) -> dict: # Code's env), it prints nothing and Claude Code falls back to the # static header below. config.setdefault( - "headersHelper", self._google_credentials_headers_helper()) + "headersHelper", self._google_credentials_headers_helper(scopes=scopes)) # Baked static token: the initial value and the fallback for when # headersHelper can't run. headersHelper output takes precedence. if "Authorization" not in headers: - token = self._fetch_gcloud_access_token() + token = self._fetch_gcloud_access_token(scopes=scopes) if token: headers["Authorization"] = f"Bearer {token}" else: @@ -322,7 +338,9 @@ def _translate_mcp_config(self, server_name: str, config: dict) -> dict: return config @staticmethod - def _google_credentials_headers_helper() -> str: + def _google_credentials_headers_helper( + scopes: Optional[list[str]] = None, + ) -> str: """Shell command Claude Code executes on each MCP connection to mint a fresh Google bearer token. @@ -333,13 +351,21 @@ def _google_credentials_headers_helper() -> str: prints nothing (non-zero exit) when neither is available so Claude Code keeps using the baked static header. """ + scoped_cmd = "" + if scopes: + scopes_str = ",".join(scopes) + scoped_cmd = ( + f'gcloud auth application-default print-access-token --scopes="{scopes_str}" 2>/dev/null || ' + ) return ( - 'tok="$(gcloud auth application-default print-access-token ' + f'tok="$({scoped_cmd}gcloud auth application-default print-access-token ' '2>/dev/null || gcloud auth print-access-token 2>/dev/null)"; ' '[ -n "$tok" ] && printf \'{"Authorization":"Bearer %s"}\' "$tok"' ) - def _fetch_gcloud_access_token(self) -> str: + def _fetch_gcloud_access_token( + self, scopes: Optional[list[str]] = None + ) -> str: """Fetches a Google Cloud access token for MCP `google_credentials` auth. Prefers Application Default Credentials (``gcloud auth @@ -372,10 +398,22 @@ def _fetch_gcloud_access_token(self) -> str: token_env["GOOGLE_APPLICATION_CREDENTIALS"] = adc # ADC first (works for these endpoints); user token as a fallback for # any MCP server that happens to accept it. - commands = [ + commands = [] + if scopes: + scopes_str = ",".join(scopes) + commands.append( + [ + "gcloud", + "auth", + "application-default", + "print-access-token", + f"--scopes={scopes_str}", + ] + ) + commands.extend([ ["gcloud", "auth", "application-default", "print-access-token"], ["gcloud", "auth", "print-access-token"], - ] + ]) for cmd in commands: try: result = subprocess.run( diff --git a/evalbench/generators/models/codex_cli.py b/evalbench/generators/models/codex_cli.py index 052ef6b4..47c1f650 100644 --- a/evalbench/generators/models/codex_cli.py +++ b/evalbench/generators/models/codex_cli.py @@ -123,6 +123,7 @@ def __init__(self, querygenerator_config): self.config_path = os.path.join(self.codex_config_dir, "config.toml") self.inline_mcp_servers = {} self.enabled_plugins = {} + self._gcloud_mcp_scopes = [] self._setup() @staticmethod @@ -607,6 +608,11 @@ def _translate_mcp_config(self, server_name: str, config: dict) -> dict: out: dict = {"url": url} headers = dict(config.get("headers") or {}) + oauth = config.pop("oauth", None) or {} + scopes = oauth.get("scopes") or config.get("scopes") or [] + if isinstance(scopes, str): + scopes = [scopes] + auth_provider = config.get("authProviderType") if auth_provider == "google_credentials" and "Authorization" not in headers: # Supply the bearer token via `bearer_token_env_var` rather than a @@ -618,7 +624,12 @@ def _translate_mcp_config(self, server_name: str, config: dict) -> dict: # stays a static header. out["bearer_token_env_var"] = self._GCLOUD_MCP_TOKEN_ENV self._needs_gcloud_mcp_token = True - token = self._fetch_gcloud_access_token() + if not hasattr(self, "_gcloud_mcp_scopes"): + self._gcloud_mcp_scopes = [] + for s in scopes: + if s and s not in self._gcloud_mcp_scopes: + self._gcloud_mcp_scopes.append(s) + token = self._fetch_gcloud_access_token(scopes=scopes) if token: self.env[self._GCLOUD_MCP_TOKEN_ENV] = token # initial value else: @@ -630,7 +641,9 @@ def _translate_mcp_config(self, server_name: str, config: dict) -> dict: out["http_headers"] = headers return out - def _fetch_gcloud_access_token(self) -> str: + def _fetch_gcloud_access_token( + self, scopes: Optional[list[str]] = None + ) -> str: """Fetches a Google Cloud access token for MCP `google_credentials` auth. Prefers Application Default Credentials (``gcloud auth @@ -650,12 +663,30 @@ def _fetch_gcloud_access_token(self) -> str: """ token_env = os.environ.copy() adc = self.env.get("GOOGLE_APPLICATION_CREDENTIALS") + if not adc and os.path.exists("/etc/evalbench-sa-key/key.json"): + adc = "/etc/evalbench-sa-key/key.json" if adc and os.path.exists(adc): token_env["GOOGLE_APPLICATION_CREDENTIALS"] = adc - commands = [ + + if scopes is None: + scopes = getattr(self, "_gcloud_mcp_scopes", None) + + commands = [] + if scopes: + scopes_str = ",".join(scopes) + commands.append( + [ + "gcloud", + "auth", + "application-default", + "print-access-token", + f"--scopes={scopes_str}", + ] + ) + commands.extend([ ["gcloud", "auth", "application-default", "print-access-token"], ["gcloud", "auth", "print-access-token"], - ] + ]) for cmd in commands: try: result = subprocess.run( @@ -843,7 +874,9 @@ def _run_codex_cli(self, cli_cmd: CLICommand, timeout_seconds=None): # is a fresh process that re-reads `bearer_token_env_var`, so minting a # new ADC token here keeps it from expiring across a long suite. if getattr(self, "_needs_gcloud_mcp_token", False): - token = self._fetch_gcloud_access_token() + token = self._fetch_gcloud_access_token( + scopes=getattr(self, "_gcloud_mcp_scopes", None) + ) if token: env[self._GCLOUD_MCP_TOKEN_ENV] = token diff --git a/evalbench/test/claude_code_test.py b/evalbench/test/claude_code_test.py index d057822c..f156b79f 100644 --- a/evalbench/test/claude_code_test.py +++ b/evalbench/test/claude_code_test.py @@ -240,3 +240,63 @@ def test_extract_skills_strips_plugin_namespace(): with patch.object(ClaudeCodeGenerator, '_get_installed_skills', return_value=set()): assert generator.extract_skills(stdout) == ["cloud-sql-postgres-admin"] + + +def test_translate_mcp_config_forwards_oauth_scopes(): + generator = object.__new__(ClaudeCodeGenerator) + generator.env = {} + + mcp_config = { + "httpUrl": "https://test-dfareporting.sandbox.googleapis.com/mcp", + "authProviderType": "google_credentials", + "oauth": { + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/dfareporting", + ] + }, + "headers": { + "X-Goog-User-Project": "my-project" + } + } + + with patch.object(generator, '_fetch_gcloud_access_token', return_value="fake_token") as mock_fetch: + translated = generator._translate_mcp_config("dfareporting", mcp_config) + + assert "oauth" not in translated + assert "authProviderType" not in translated + assert translated["type"] == "http" + assert translated["url"] == "https://test-dfareporting.sandbox.googleapis.com/mcp" + assert translated["headers"]["Authorization"] == "Bearer fake_token" + assert translated["headers"]["X-Goog-User-Project"] == "my-project" + assert '--scopes="https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/dfareporting"' in translated["headersHelper"] + + mock_fetch.assert_called_once_with(scopes=[ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/dfareporting", + ]) + + +def test_fetch_gcloud_access_token_passes_scopes(): + generator = object.__new__(ClaudeCodeGenerator) + generator.env = {} + + scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/dfareporting", + ] + + with patch('generators.models.claude_code.subprocess.run') as mock_run: + mock_proc = MagicMock() + mock_proc.stdout = "scoped_token_xyz\n" + mock_run.return_value = mock_proc + + token = generator._fetch_gcloud_access_token(scopes=scopes) + + assert token == "scoped_token_xyz" + first_call_cmd = mock_run.call_args_list[0][0][0] + assert first_call_cmd == [ + "gcloud", "auth", "application-default", "print-access-token", + "--scopes=https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/dfareporting", + ] + diff --git a/evalbench/test/codex_cli_test.py b/evalbench/test/codex_cli_test.py index db4755bf..14850407 100644 --- a/evalbench/test/codex_cli_test.py +++ b/evalbench/test/codex_cli_test.py @@ -127,3 +127,100 @@ def test_write_config_toml_escapes_plugin_id(monkeypatch, tmp_path): content = config_file.read_text() assert '[plugins."dak@evalbench-local-marketplace"]' in content assert '[plugins.clean_plugin]' in content + + +def test_translate_mcp_config_forwards_oauth_scopes(): + generator = object.__new__(CodexCliGenerator) + generator.env = {} + generator._gcloud_mcp_scopes = [] + + mcp_config = { + "httpUrl": "https://test-dfareporting.sandbox.googleapis.com/mcp", + "authProviderType": "google_credentials", + "oauth": { + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/dfareporting", + ] + }, + "headers": { + "X-Goog-User-Project": "my-project" + } + } + + with patch.object(generator, '_fetch_gcloud_access_token', return_value="fake_token") as mock_fetch: + translated = generator._translate_mcp_config("dfareporting", mcp_config) + + assert "oauth" not in translated + assert "authProviderType" not in translated + assert translated["url"] == "https://test-dfareporting.sandbox.googleapis.com/mcp" + assert translated["bearer_token_env_var"] == "EVALBENCH_GCLOUD_MCP_TOKEN" + assert generator.env["EVALBENCH_GCLOUD_MCP_TOKEN"] == "fake_token" + assert translated["http_headers"]["X-Goog-User-Project"] == "my-project" + assert generator._gcloud_mcp_scopes == [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/dfareporting", + ] + + mock_fetch.assert_called_once_with(scopes=[ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/dfareporting", + ]) + + +def test_fetch_gcloud_access_token_passes_scopes(): + generator = object.__new__(CodexCliGenerator) + generator.env = {} + + scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/dfareporting", + ] + + with patch('generators.models.codex_cli.subprocess.run') as mock_run: + mock_proc = MagicMock() + mock_proc.stdout = "scoped_token_xyz\n" + mock_run.return_value = mock_proc + + token = generator._fetch_gcloud_access_token(scopes=scopes) + + assert token == "scoped_token_xyz" + first_call_cmd = mock_run.call_args_list[0][0][0] + assert first_call_cmd == [ + "gcloud", "auth", "application-default", "print-access-token", + "--scopes=https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/dfareporting", + ] + + +def test_run_codex_cli_refreshes_token_with_recorded_scopes(): + generator = object.__new__(CodexCliGenerator) + generator.env = {} + generator._needs_gcloud_mcp_token = True + generator._gcloud_mcp_scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/dfareporting", + ] + generator.json_flag = "--json" + generator.sandbox_mode = "danger-full-access" + generator.approval_mode = "never" + generator.model = None + generator.profile = None + + cli_cmd = MagicMock() + cli_cmd.cli = "codex" + cli_cmd.resume = False + cli_cmd.session_id = None + cli_cmd.prompt = "test prompt" + cli_cmd.env = {} + cli_cmd.cwd = None + + mock_completed = MagicMock() + mock_completed.stdout = "" + mock_completed.stderr = "" + + with ( + patch.object(generator, '_fetch_gcloud_access_token', return_value="refreshed_token") as mock_fetch, + patch.object(generator, '_execute_cli_command', return_value=(mock_completed, {})), + ): + generator._run_codex_cli(cli_cmd) + mock_fetch.assert_called_once_with(scopes=generator._gcloud_mcp_scopes) From ca3515ab8e1ef377b0ab224bb1254ebc731a9932 Mon Sep 17 00:00:00 2001 From: Ike Nwankwo Date: Tue, 6 Oct 2026 15:16:43 +0000 Subject: [PATCH 2/4] Unify MCP OAuth scope extraction using shared helper in mcp_client --- evalbench/generators/models/claude_code.py | 17 +-- evalbench/generators/models/codex_cli.py | 7 +- evalbench/generators/models/mcp_client.py | 19 ++- evalbench/test/mcp_client_test.py | 136 +++++++++++++++++++++ 4 files changed, 160 insertions(+), 19 deletions(-) create mode 100644 evalbench/test/mcp_client_test.py diff --git a/evalbench/generators/models/claude_code.py b/evalbench/generators/models/claude_code.py index e0731526..5f6356e9 100644 --- a/evalbench/generators/models/claude_code.py +++ b/evalbench/generators/models/claude_code.py @@ -1,4 +1,5 @@ from .agent_cli import AgentCliGenerator +from . import mcp_client from .tool_naming import canonicalize_claude_tool_name import subprocess import os @@ -294,20 +295,8 @@ def _translate_mcp_config(self, server_name: str, config: dict) -> dict: # Gemini-style `oauth.scopes` is forwarded to gcloud, but the `oauth` # block itself must be removed so Claude Code doesn't treat it as an # interactive MCP OAuth 2.0 dynamic registration config. - oauth_config = config.pop("oauth", None) - raw_scopes = None - if isinstance(oauth_config, dict): - raw_scopes = oauth_config.get("scopes") - elif isinstance(oauth_config, (list, str)): - raw_scopes = oauth_config - if not raw_scopes and "scopes" in config: - raw_scopes = config.pop("scopes") - - scopes: list[str] = [] - if isinstance(raw_scopes, list): - scopes = [str(s).strip() for s in raw_scopes if str(s).strip()] - elif isinstance(raw_scopes, str) and raw_scopes.strip(): - scopes = [s.strip() for s in raw_scopes.split(",") if s.strip()] + scopes = mcp_client.extract_mcp_oauth_scopes(config) + config.pop("oauth", None) if auth_provider == "google_credentials": headers = config.get("headers", {}) or {} diff --git a/evalbench/generators/models/codex_cli.py b/evalbench/generators/models/codex_cli.py index 47c1f650..ce8651c0 100644 --- a/evalbench/generators/models/codex_cli.py +++ b/evalbench/generators/models/codex_cli.py @@ -1,3 +1,4 @@ +from . import mcp_client from .agent_cli import AgentCliGenerator from .tool_naming import canonical_tool_name import subprocess @@ -608,10 +609,8 @@ def _translate_mcp_config(self, server_name: str, config: dict) -> dict: out: dict = {"url": url} headers = dict(config.get("headers") or {}) - oauth = config.pop("oauth", None) or {} - scopes = oauth.get("scopes") or config.get("scopes") or [] - if isinstance(scopes, str): - scopes = [scopes] + scopes = mcp_client.extract_mcp_oauth_scopes(config) + config.pop("oauth", None) auth_provider = config.get("authProviderType") if auth_provider == "google_credentials" and "Authorization" not in headers: diff --git a/evalbench/generators/models/mcp_client.py b/evalbench/generators/models/mcp_client.py index 7c3bbd73..c6f4e2a9 100644 --- a/evalbench/generators/models/mcp_client.py +++ b/evalbench/generators/models/mcp_client.py @@ -27,6 +27,23 @@ def _format_error(e: BaseException) -> str: return f"{type(e).__name__}: {e}" if str(e) else type(e).__name__ +def extract_mcp_oauth_scopes(server_config: dict) -> list[str]: + """Extracts OAuth scopes from an MCP server config. + + Accepts the canonical Gemini CLI schema: + oauth: + scopes: + - https://www.googleapis.com/auth/... + """ + oauth = server_config.get("oauth") + if not isinstance(oauth, dict): + return [] + scopes = oauth.get("scopes") + if not isinstance(scopes, list): + return [] + return [str(s).strip() for s in scopes if str(s).strip()] + + def auth_headers(server_config: dict) -> dict | None: """Build request headers for an MCP server (configured headers + auth). @@ -39,7 +56,7 @@ def auth_headers(server_config: dict) -> dict | None: import google.auth import google.auth.transport.requests - scopes = (server_config.get("oauth") or {}).get("scopes") + scopes = extract_mcp_oauth_scopes(server_config) if not scopes: raise McpToolsError( "google_credentials auth requires oauth.scopes on the MCP " diff --git a/evalbench/test/mcp_client_test.py b/evalbench/test/mcp_client_test.py new file mode 100644 index 00000000..c84051b4 --- /dev/null +++ b/evalbench/test/mcp_client_test.py @@ -0,0 +1,136 @@ +"""Unit tests for mcp_client auth and scope extraction.""" + +import unittest +from unittest.mock import MagicMock, patch + +from generators.models.mcp_client import ( + McpToolsError, + auth_headers, + extract_mcp_oauth_scopes, +) + + +class ExtractMcpOauthScopesTest(unittest.TestCase): + + def test_canonical_schema(self): + config = { + "oauth": { + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/dfareporting", + ] + } + } + self.assertEqual( + extract_mcp_oauth_scopes(config), + [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/dfareporting", + ], + ) + + def test_strips_whitespace_and_drops_empty(self): + config = { + "oauth": { + "scopes": [ + " https://www.googleapis.com/auth/cloud-platform ", + "", + " ", + "https://www.googleapis.com/auth/dfareporting", + ] + } + } + self.assertEqual( + extract_mcp_oauth_scopes(config), + [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/dfareporting", + ], + ) + + def test_missing_or_none_oauth(self): + self.assertEqual(extract_mcp_oauth_scopes({}), []) + self.assertEqual(extract_mcp_oauth_scopes({"oauth": None}), []) + + def test_non_dict_oauth_does_not_raise(self): + self.assertEqual( + extract_mcp_oauth_scopes({"oauth": ["https://www.googleapis.com/auth/cloud-platform"]}), + [], + ) + self.assertEqual( + extract_mcp_oauth_scopes({"oauth": "https://www.googleapis.com/auth/cloud-platform"}), + [], + ) + self.assertEqual(extract_mcp_oauth_scopes({"oauth": 123}), []) + + def test_missing_or_non_list_scopes_does_not_raise(self): + self.assertEqual(extract_mcp_oauth_scopes({"oauth": {}}), []) + self.assertEqual(extract_mcp_oauth_scopes({"oauth": {"scopes": None}}), []) + self.assertEqual( + extract_mcp_oauth_scopes({"oauth": {"scopes": "https://www.googleapis.com/auth/cloud-platform"}}), + [], + ) + + def test_top_level_scopes_ignored(self): + self.assertEqual( + extract_mcp_oauth_scopes({"scopes": ["https://www.googleapis.com/auth/cloud-platform"]}), + [], + ) + + +class AuthHeadersTest(unittest.TestCase): + + def test_no_auth_provider_returns_none_if_no_headers(self): + self.assertIsNone(auth_headers({})) + + def test_no_auth_provider_preserves_static_headers(self): + self.assertEqual( + auth_headers({"headers": {"X-Custom": "val"}}), + {"X-Custom": "val"}, + ) + + def test_google_credentials_missing_scopes_raises_mcp_tools_error(self): + with self.assertRaises(McpToolsError) as ctx: + auth_headers({"authProviderType": "google_credentials"}) + self.assertIn("requires oauth.scopes", str(ctx.exception)) + + def test_google_credentials_non_dict_oauth_raises_mcp_tools_error(self): + with self.assertRaises(McpToolsError) as ctx: + auth_headers({ + "authProviderType": "google_credentials", + "oauth": ["https://www.googleapis.com/auth/cloud-platform"], + }) + self.assertIn("requires oauth.scopes", str(ctx.exception)) + + @patch("google.auth.default", side_effect=Exception("network error")) + def test_google_credentials_adc_failure_raises_mcp_tools_error(self, mock_default): + with self.assertRaises(McpToolsError) as ctx: + auth_headers({ + "authProviderType": "google_credentials", + "oauth": {"scopes": ["https://www.googleapis.com/auth/cloud-platform"]}, + }) + self.assertIn("Failed to acquire GCP Application Default Credentials", str(ctx.exception)) + + @patch("google.auth.default") + @patch("google.auth.transport.requests.Request") + def test_google_credentials_fetches_token(self, mock_request, mock_default): + mock_creds = MagicMock() + mock_creds.token = "token123" + mock_default.return_value = (mock_creds, "project1") + + headers = auth_headers({ + "authProviderType": "google_credentials", + "headers": {"X-Goog-User-Project": "project1"}, + "oauth": { + "scopes": ["https://www.googleapis.com/auth/cloud-platform"] + }, + }) + mock_default.assert_called_once_with( + scopes=["https://www.googleapis.com/auth/cloud-platform"] + ) + self.assertEqual(headers["Authorization"], "Bearer token123") + self.assertEqual(headers["X-Goog-User-Project"], "project1") + + +if __name__ == "__main__": + unittest.main() From dfcc66be7dfd8e08318501cbce44b124ae628331 Mon Sep 17 00:00:00 2001 From: Ike Nwankwo Date: Wed, 7 Oct 2026 16:52:28 +0000 Subject: [PATCH 3/4] style: fix pycodestyle W391 blank line at EOF in claude_code_test.py --- evalbench/test/claude_code_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/evalbench/test/claude_code_test.py b/evalbench/test/claude_code_test.py index f156b79f..414d6166 100644 --- a/evalbench/test/claude_code_test.py +++ b/evalbench/test/claude_code_test.py @@ -299,4 +299,3 @@ def test_fetch_gcloud_access_token_passes_scopes(): "gcloud", "auth", "application-default", "print-access-token", "--scopes=https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/dfareporting", ] - From 06d3ed6de269946a17bc5beb9add8e09b194d657 Mon Sep 17 00:00:00 2001 From: Ike Nwankwo Date: Fri, 9 Oct 2026 03:23:29 +0000 Subject: [PATCH 4/4] fix(models): address PR review comments for scope assertions, adc fallback, and docs --- docs/claude_code_agent_testing.md | 13 ++++++++++++- docs/codex_cli_agent_testing.md | 13 ++++++++++++- evalbench/generators/models/claude_code.py | 8 ++++++++ evalbench/generators/models/codex_cli.py | 6 ++++-- evalbench/test/claude_code_test.py | 18 ++++++++++++++++++ evalbench/test/codex_cli_test.py | 10 ++++++++++ 6 files changed, 64 insertions(+), 4 deletions(-) diff --git a/docs/claude_code_agent_testing.md b/docs/claude_code_agent_testing.md index f6ca2f7e..2c9c9493 100644 --- a/docs/claude_code_agent_testing.md +++ b/docs/claude_code_agent_testing.md @@ -284,7 +284,7 @@ EvalBench accepts the **same MCP server config schema as Gemini CLI** for HTTP s |---|---| | `httpUrl` | → `url` + auto-adds `type: "http"` | | `authProviderType: google_credentials` | → injects `Authorization: Bearer ` (from `gcloud auth application-default print-access-token`, falling back to `gcloud auth print-access-token`) **and** sets a `headersHelper` so Claude Code re-mints a fresh ADC token on every connection (avoids ~1h expiry). Google API MCP endpoints reject the plain user token on tool calls — see [Troubleshooting](#mcp-tool-call-fails-with-incompatible-auth-server-does-not-support-dynamic-client-registration). | -| `oauth.scopes` | → forwarded as `--scopes` to `gcloud auth application-default print-access-token` in both `headersHelper` and the initial static `Authorization` header | +| `oauth.scopes` | → forwarded as `--scopes` to `gcloud auth application-default print-access-token` in both `headersHelper` and the initial static `Authorization` header. If the scoped token request fails, the generator falls back to an unscoped ADC token (see [Troubleshooting](#user-adc-and-non-default-oauth-scopes-403-forbidden-on-tool-call)). | | `headers` | → passed through as-is | | `command` / `args` (stdio) | → passed through as-is | @@ -490,6 +490,17 @@ Usually a token problem (see the DCR entry above). Checklist: - Set the quota project header: `headers: { X-Goog-User-Project: }`. - Verify directly: `curl -H "Authorization: Bearer $(gcloud auth application-default print-access-token)" -H "X-Goog-User-Project: " -H "Content-Type: application/json" -H "Accept: application/json, text/event-stream" -d '{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"list_instances","arguments":{"project":""}}}' https://sqladmin.googleapis.com/mcp` +### User ADC and Non-Default OAuth Scopes (`403 Forbidden` on tool call) + +When using user credentials (via `gcloud auth application-default login`), `gcloud` only requests the default `cloud-platform` scopes by default. If an MCP server specifies a non-default OAuth scope (e.g. DFA Reporting or Google Ads), `gcloud auth application-default print-access-token --scopes=...` will fail unless that scope was explicitly granted at login time. + +When the scoped request fails, the generator quietly falls back to an **unscoped ADC token**. The MCP server connects normally, but actual tool calls will later fail with a confusing `403 Forbidden`. + +To resolve this with user ADC, re-authenticate and explicitly specify the required scopes: +```bash +gcloud auth application-default login --scopes="https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/" +``` + ### `npm exec` is slow on first run `npm exec --yes @` downloads the package on first use (~30 sec). Subsequent runs use the cache. diff --git a/docs/codex_cli_agent_testing.md b/docs/codex_cli_agent_testing.md index ca857aad..9b567e99 100644 --- a/docs/codex_cli_agent_testing.md +++ b/docs/codex_cli_agent_testing.md @@ -305,7 +305,7 @@ EvalBench accepts the **same MCP server config schema as Gemini CLI and Claude C | `httpUrl` | → `url` (TOML, streamable HTTP server) | | `headers` | → `http_headers` (TOML inline table) | | `authProviderType: google_credentials` | → mints an **ADC** token (`gcloud auth application-default print-access-token`, falling back to `gcloud auth print-access-token`) and passes it to Codex via `bearer_token_env_var` (env var `EVALBENCH_GCLOUD_MCP_TOKEN`). The generator re-mints a fresh token before **every turn** (each `codex exec` re-reads the env var), so it doesn't expire mid-suite. `X-Goog-User-Project` stays a static `http_headers` entry. Google API MCP endpoints reject the plain user token on tool calls — see [Troubleshooting](#mcp-server-fails-with-401-unauthorized-real-cloud-sql-endpoint). | -| `oauth.scopes` | → forwarded as `--scopes` to `gcloud auth application-default print-access-token` when minting `bearer_token_env_var` (`EVALBENCH_GCLOUD_MCP_TOKEN`) | +| `oauth.scopes` | → forwarded as `--scopes` to `gcloud auth application-default print-access-token` when minting `bearer_token_env_var` (`EVALBENCH_GCLOUD_MCP_TOKEN`). If the scoped token request fails, the generator falls back to an unscoped ADC token (see [Troubleshooting](#user-adc-and-non-default-oauth-scopes-403-forbidden-on-tool-call)). | | `command` / `args` / `env` / `cwd` (stdio) | → passed through as-is into a `[mcp_servers.NAME]` stdio block | ### HTTP MCP server (Cloud SQL Managed) @@ -575,6 +575,17 @@ The injected token is missing, expired, the **wrong kind**, or your principal la - `X-Goog-User-Project` header points at a project that has the Cloud SQL Admin API enabled. - Token expiry is handled automatically: the generator mints a fresh ADC token into `EVALBENCH_GCLOUD_MCP_TOKEN` before every turn and Codex reads it via `bearer_token_env_var`, so long suites don't hit stale-token 401s. (Requires a Codex build that supports `bearer_token_env_var`; if yours doesn't, the token won't be applied — fall back to a static `http_headers` `Authorization`.) +### User ADC and Non-Default OAuth Scopes (`403 Forbidden` on tool call) + +When using user credentials (via `gcloud auth application-default login`), `gcloud` only requests default scopes by default. If an MCP server specifies a non-default OAuth scope (e.g. DFA Reporting or Google Ads), `gcloud auth application-default print-access-token --scopes=...` will fail unless that scope was explicitly granted at login time. + +When the scoped request fails, the generator quietly falls back to an **unscoped ADC token**. The MCP server connects normally, but actual tool calls will subsequently fail with a confusing `403 Forbidden`. + +To resolve this with user ADC, re-authenticate and explicitly specify the required scopes: +```bash +gcloud auth application-default login --scopes="https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/" +``` + ### `Invalid TOML` when Codex starts Either a manual edit to `~/.codex/config.toml` clobbered the generated file, or a hand-written `setup.config` value contains an unsupported type. Fix: diff --git a/evalbench/generators/models/claude_code.py b/evalbench/generators/models/claude_code.py index 5f6356e9..c6e8f144 100644 --- a/evalbench/generators/models/claude_code.py +++ b/evalbench/generators/models/claude_code.py @@ -342,6 +342,10 @@ def _google_credentials_headers_helper( """ scoped_cmd = "" if scopes: + for s in scopes: + assert re.fullmatch(r"[A-Za-z0-9_.:/-]+", s), ( + f"Invalid or unsafe OAuth scope: {s!r}" + ) scopes_str = ",".join(scopes) scoped_cmd = ( f'gcloud auth application-default print-access-token --scopes="{scopes_str}" 2>/dev/null || ' @@ -389,6 +393,10 @@ def _fetch_gcloud_access_token( # any MCP server that happens to accept it. commands = [] if scopes: + for s in scopes: + assert re.fullmatch(r"[A-Za-z0-9_.:/-]+", s), ( + f"Invalid or unsafe OAuth scope: {s!r}" + ) scopes_str = ",".join(scopes) commands.append( [ diff --git a/evalbench/generators/models/codex_cli.py b/evalbench/generators/models/codex_cli.py index ce8651c0..2f1e0759 100644 --- a/evalbench/generators/models/codex_cli.py +++ b/evalbench/generators/models/codex_cli.py @@ -662,8 +662,6 @@ def _fetch_gcloud_access_token( """ token_env = os.environ.copy() adc = self.env.get("GOOGLE_APPLICATION_CREDENTIALS") - if not adc and os.path.exists("/etc/evalbench-sa-key/key.json"): - adc = "/etc/evalbench-sa-key/key.json" if adc and os.path.exists(adc): token_env["GOOGLE_APPLICATION_CREDENTIALS"] = adc @@ -672,6 +670,10 @@ def _fetch_gcloud_access_token( commands = [] if scopes: + for s in scopes: + assert re.fullmatch(r"[A-Za-z0-9_.:/-]+", s), ( + f"Invalid or unsafe OAuth scope: {s!r}" + ) scopes_str = ",".join(scopes) commands.append( [ diff --git a/evalbench/test/claude_code_test.py b/evalbench/test/claude_code_test.py index 414d6166..92f3cfca 100644 --- a/evalbench/test/claude_code_test.py +++ b/evalbench/test/claude_code_test.py @@ -3,6 +3,8 @@ import sys from unittest.mock import MagicMock, patch, ANY +import pytest + # Add parent directory to path so we can import generators sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -299,3 +301,19 @@ def test_fetch_gcloud_access_token_passes_scopes(): "gcloud", "auth", "application-default", "print-access-token", "--scopes=https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/dfareporting", ] + + +def test_headers_helper_rejects_unsafe_scopes(): + with pytest.raises(AssertionError, match="Invalid or unsafe OAuth scope"): + ClaudeCodeGenerator._google_credentials_headers_helper( + scopes=['https://example.com"; rm -rf /; "'] + ) + + +def test_fetch_gcloud_access_token_rejects_unsafe_scopes(): + generator = object.__new__(ClaudeCodeGenerator) + generator.env = {} + with pytest.raises(AssertionError, match="Invalid or unsafe OAuth scope"): + generator._fetch_gcloud_access_token( + scopes=['$(whoami)'] + ) diff --git a/evalbench/test/codex_cli_test.py b/evalbench/test/codex_cli_test.py index 14850407..e6719781 100644 --- a/evalbench/test/codex_cli_test.py +++ b/evalbench/test/codex_cli_test.py @@ -1,6 +1,7 @@ import os import sys from unittest.mock import MagicMock, patch +import pytest sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -224,3 +225,12 @@ def test_run_codex_cli_refreshes_token_with_recorded_scopes(): ): generator._run_codex_cli(cli_cmd) mock_fetch.assert_called_once_with(scopes=generator._gcloud_mcp_scopes) + + +def test_fetch_gcloud_access_token_rejects_unsafe_scopes(): + generator = object.__new__(CodexCliGenerator) + generator.env = {} + with pytest.raises(AssertionError, match="Invalid or unsafe OAuth scope"): + generator._fetch_gcloud_access_token( + scopes=['$(whoami)'] + )