From bb2482b55f2687f39326273645ad9cbdd5972a9e Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Mon, 5 Jan 2026 18:05:51 +0000 Subject: [PATCH 01/10] feat: add client conformance testing CI - Copy everything-client.ts and helpers from conformance repo - Add conformance:client npm script for running initialize scenario - Add GitHub Actions workflow (non-blocking with continue-on-error) - Use @modelcontextprotocol/client workspace package for imports --- .github/workflows/conformance.yml | 24 ++ package.json | 4 +- pnpm-lock.yaml | 3 + src/conformance/everything-client.ts | 224 ++++++++++++++++++ .../helpers/ConformanceOAuthProvider.ts | 95 ++++++++ src/conformance/helpers/logger.ts | 27 +++ src/conformance/helpers/withOAuthRetry.ts | 111 +++++++++ 7 files changed, 487 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/conformance.yml create mode 100644 src/conformance/everything-client.ts create mode 100644 src/conformance/helpers/ConformanceOAuthProvider.ts create mode 100644 src/conformance/helpers/logger.ts create mode 100644 src/conformance/helpers/withOAuthRetry.ts diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 000000000..4c7a477f1 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,24 @@ +name: Conformance Tests + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + client-conformance: + runs-on: ubuntu-latest + continue-on-error: true # Non-blocking initially + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: npm + - uses: pnpm/action-setup@v4 + with: + version: 10.24.0 + - run: pnpm install + - run: pnpm run build:all + - run: pnpm run conformance:client diff --git a/package.json b/package.json index 2633d5ef2..63a832207 100644 --- a/package.json +++ b/package.json @@ -29,9 +29,11 @@ "lint:all": "pnpm -r lint", "lint:fix:all": "pnpm -r lint:fix", "check:all": "pnpm -r typecheck && pnpm -r lint", - "test:all": "pnpm -r test" + "test:all": "pnpm -r test", + "conformance:client": "npx @modelcontextprotocol/conformance client --command 'npx tsx src/conformance/everything-client.ts' --scenario initialize" }, "devDependencies": { + "@modelcontextprotocol/client": "workspace:^", "@cfworker/json-schema": "catalog:runtimeShared", "@changesets/changelog-github": "^0.5.2", "@changesets/cli": "^2.29.8", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 92dbf8253..d5148215f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -149,6 +149,9 @@ importers: '@eslint/js': specifier: catalog:devTools version: 9.39.1 + '@modelcontextprotocol/client': + specifier: workspace:^ + version: link:packages/client '@types/content-type': specifier: catalog:devTools version: 1.1.9 diff --git a/src/conformance/everything-client.ts b/src/conformance/everything-client.ts new file mode 100644 index 000000000..f8a8ae581 --- /dev/null +++ b/src/conformance/everything-client.ts @@ -0,0 +1,224 @@ +#!/usr/bin/env node + +/** + * Everything client - a single conformance test client that handles all scenarios. + * + * Usage: everything-client + * + * The scenario name is read from the MCP_CONFORMANCE_SCENARIO environment variable, + * which is set by the conformance test runner. + * + * This client routes to the appropriate behavior based on the scenario name, + * consolidating all the individual test clients into one. + */ + +import { + Client, + StreamableHTTPClientTransport, + ElicitRequestSchema +} from '@modelcontextprotocol/client'; +import { withOAuthRetry } from './helpers/withOAuthRetry.js'; +import { logger } from './helpers/logger.js'; + +// Scenario handler type +type ScenarioHandler = (serverUrl: string) => Promise; + +// Registry of scenario handlers +const scenarioHandlers: Record = {}; + +// Helper to register a scenario handler +function registerScenario(name: string, handler: ScenarioHandler): void { + scenarioHandlers[name] = handler; +} + +// Helper to register multiple scenarios with the same handler +function registerScenarios(names: string[], handler: ScenarioHandler): void { + for (const name of names) { + scenarioHandlers[name] = handler; + } +} + +// ============================================================================ +// Basic scenarios (initialize, tools-call) +// ============================================================================ + +async function runBasicClient(serverUrl: string): Promise { + const client = new Client( + { name: 'test-client', version: '1.0.0' }, + { capabilities: {} } + ); + + const transport = new StreamableHTTPClientTransport(new URL(serverUrl)); + + await client.connect(transport); + logger.debug('Successfully connected to MCP server'); + + await client.listTools(); + logger.debug('Successfully listed tools'); + + await transport.close(); + logger.debug('Connection closed successfully'); +} + +registerScenarios(['initialize', 'tools-call'], runBasicClient); + +// ============================================================================ +// Auth scenarios - well-behaved client +// ============================================================================ + +async function runAuthClient(serverUrl: string): Promise { + const client = new Client( + { name: 'test-auth-client', version: '1.0.0' }, + { capabilities: {} } + ); + + const oauthFetch = withOAuthRetry( + 'test-auth-client', + new URL(serverUrl) + )(fetch); + + const transport = new StreamableHTTPClientTransport(new URL(serverUrl), { + fetch: oauthFetch + }); + + await client.connect(transport); + logger.debug('Successfully connected to MCP server'); + + await client.listTools(); + logger.debug('Successfully listed tools'); + + await client.callTool({ name: 'test-tool', arguments: {} }); + logger.debug('Successfully called tool'); + + await transport.close(); + logger.debug('Connection closed successfully'); +} + +// Register all auth scenarios that should use the well-behaved auth client +registerScenarios( + [ + 'auth/basic-dcr', + 'auth/basic-metadata-var1', + 'auth/basic-metadata-var2', + 'auth/basic-metadata-var3', + 'auth/2025-03-26-oauth-metadata-backcompat', + 'auth/2025-03-26-oauth-endpoint-fallback', + 'auth/scope-from-www-authenticate', + 'auth/scope-from-scopes-supported', + 'auth/scope-omitted-when-undefined', + 'auth/scope-step-up' + ], + runAuthClient +); + +// ============================================================================ +// Elicitation defaults scenario +// ============================================================================ + +async function runElicitationDefaultsClient(serverUrl: string): Promise { + const client = new Client( + { name: 'elicitation-defaults-test-client', version: '1.0.0' }, + { + capabilities: { + elicitation: { + applyDefaults: true + } + } + } + ); + + // Register elicitation handler that returns empty content + // The SDK should fill in defaults for all omitted fields + client.setRequestHandler(ElicitRequestSchema, async (request) => { + logger.debug( + 'Received elicitation request:', + JSON.stringify(request.params, null, 2) + ); + logger.debug('Accepting with empty content - SDK should apply defaults'); + + // Return empty content - SDK should merge in defaults + return { + action: 'accept' as const, + content: {} + }; + }); + + const transport = new StreamableHTTPClientTransport(new URL(serverUrl)); + + await client.connect(transport); + logger.debug('Successfully connected to MCP server'); + + // List available tools + const tools = await client.listTools(); + logger.debug( + 'Available tools:', + tools.tools.map((t) => t.name) + ); + + // Call the test tool which will trigger elicitation + const testTool = tools.tools.find( + (t) => t.name === 'test_client_elicitation_defaults' + ); + if (!testTool) { + throw new Error('Test tool not found: test_client_elicitation_defaults'); + } + + logger.debug('Calling test_client_elicitation_defaults tool...'); + const result = await client.callTool({ + name: 'test_client_elicitation_defaults', + arguments: {} + }); + + logger.debug('Tool result:', JSON.stringify(result, null, 2)); + + await transport.close(); + logger.debug('Connection closed successfully'); +} + +registerScenario('elicitation-defaults', runElicitationDefaultsClient); + +// ============================================================================ +// Main entry point +// ============================================================================ + +async function main(): Promise { + const scenarioName = process.env.MCP_CONFORMANCE_SCENARIO; + const serverUrl = process.argv[2]; + + if (!scenarioName || !serverUrl) { + console.error( + 'Usage: MCP_CONFORMANCE_SCENARIO= everything-client ' + ); + console.error( + '\nThe MCP_CONFORMANCE_SCENARIO env var is set automatically by the conformance runner.' + ); + console.error('\nAvailable scenarios:'); + for (const name of Object.keys(scenarioHandlers).sort()) { + console.error(` - ${name}`); + } + process.exit(1); + } + + const handler = scenarioHandlers[scenarioName]; + if (!handler) { + console.error(`Unknown scenario: ${scenarioName}`); + console.error('\nAvailable scenarios:'); + for (const name of Object.keys(scenarioHandlers).sort()) { + console.error(` - ${name}`); + } + process.exit(1); + } + + try { + await handler(serverUrl); + process.exit(0); + } catch (error) { + console.error('Error:', error); + process.exit(1); + } +} + +main().catch((error) => { + console.error('Unhandled error:', error); + process.exit(1); +}); diff --git a/src/conformance/helpers/ConformanceOAuthProvider.ts b/src/conformance/helpers/ConformanceOAuthProvider.ts new file mode 100644 index 000000000..b29b95796 --- /dev/null +++ b/src/conformance/helpers/ConformanceOAuthProvider.ts @@ -0,0 +1,95 @@ +import type { + OAuthClientProvider, + OAuthClientInformation, + OAuthClientInformationFull, + OAuthClientMetadata, + OAuthTokens +} from '@modelcontextprotocol/client'; + +export class ConformanceOAuthProvider implements OAuthClientProvider { + private _clientInformation?: OAuthClientInformationFull; + private _tokens?: OAuthTokens; + private _codeVerifier?: string; + private _authCode?: string; + private _authCodePromise?: Promise; + + constructor( + private readonly _redirectUrl: string | URL, + private readonly _clientMetadata: OAuthClientMetadata, + private readonly _clientMetadataUrl?: string | URL + ) {} + + get redirectUrl(): string | URL { + return this._redirectUrl; + } + + get clientMetadata(): OAuthClientMetadata { + return this._clientMetadata; + } + + get clientMetadataUrl(): string | undefined { + return this._clientMetadataUrl?.toString(); + } + + clientInformation(): OAuthClientInformation | undefined { + return this._clientInformation; + } + + saveClientInformation(clientInformation: OAuthClientInformationFull): void { + this._clientInformation = clientInformation; + } + + tokens(): OAuthTokens | undefined { + return this._tokens; + } + + saveTokens(tokens: OAuthTokens): void { + this._tokens = tokens; + } + + async redirectToAuthorization(authorizationUrl: URL): Promise { + try { + const response = await fetch(authorizationUrl.toString(), { + redirect: 'manual' // Don't follow redirects automatically + }); + + // Get the Location header which contains the redirect with auth code + const location = response.headers.get('location'); + if (location) { + const redirectUrl = new URL(location); + const code = redirectUrl.searchParams.get('code'); + if (code) { + this._authCode = code; + return; + } else { + throw new Error('No auth code in redirect URL'); + } + } else { + throw new Error( + `No redirect location received, from '${authorizationUrl.toString()}'` + ); + } + } catch (error) { + console.error('Failed to fetch authorization URL:', error); + throw error; + } + } + + async getAuthCode(): Promise { + if (this._authCode) { + return this._authCode; + } + throw new Error('No authorization code'); + } + + saveCodeVerifier(codeVerifier: string): void { + this._codeVerifier = codeVerifier; + } + + codeVerifier(): string { + if (!this._codeVerifier) { + throw new Error('No code verifier saved'); + } + return this._codeVerifier; + } +} diff --git a/src/conformance/helpers/logger.ts b/src/conformance/helpers/logger.ts new file mode 100644 index 000000000..3b1ff5c39 --- /dev/null +++ b/src/conformance/helpers/logger.ts @@ -0,0 +1,27 @@ +/** + * Simple logger with configurable log levels. + * Set to 'error' in tests to suppress debug output. + */ + +export type LogLevel = 'debug' | 'error'; + +let currentLogLevel: LogLevel = 'debug'; + +export function setLogLevel(level: LogLevel): void { + currentLogLevel = level; +} + +export function getLogLevel(): LogLevel { + return currentLogLevel; +} + +export const logger = { + debug: (...args: unknown[]): void => { + if (currentLogLevel === 'debug') { + console.log(...args); + } + }, + error: (...args: unknown[]): void => { + console.error(...args); + } +}; diff --git a/src/conformance/helpers/withOAuthRetry.ts b/src/conformance/helpers/withOAuthRetry.ts new file mode 100644 index 000000000..bba21ef73 --- /dev/null +++ b/src/conformance/helpers/withOAuthRetry.ts @@ -0,0 +1,111 @@ +import { + auth, + extractWWWAuthenticateParams, + UnauthorizedError +} from '@modelcontextprotocol/client'; +import type { FetchLike, Middleware } from '@modelcontextprotocol/client'; +import { ConformanceOAuthProvider } from './ConformanceOAuthProvider.js'; + +export const handle401 = async ( + response: Response, + provider: ConformanceOAuthProvider, + next: FetchLike, + serverUrl: string | URL +): Promise => { + const { resourceMetadataUrl, scope } = extractWWWAuthenticateParams(response); + let result = await auth(provider, { + serverUrl, + resourceMetadataUrl, + scope, + fetchFn: next + }); + + if (result === 'REDIRECT') { + // Ordinarily, we'd wait for the callback to be handled here, + // but in our conformance provider, we get the authorization code + // during the redirect handling, so we can go straight to + // retrying the auth step. + // await provider.waitForCallback(); + + const authorizationCode = await provider.getAuthCode(); + + // TODO: this retry logic should be incorporated into the typescript SDK + result = await auth(provider, { + serverUrl, + resourceMetadataUrl, + scope, + authorizationCode, + fetchFn: next + }); + if (result !== 'AUTHORIZED') { + throw new UnauthorizedError( + `Authentication failed with result: ${result}` + ); + } + } +}; +/** + * Creates a fetch wrapper that handles OAuth authentication with retry logic. + * + * Unlike the SDK's withOAuth, this version: + * - Automatically handles authorization redirects by retrying with fresh tokens + * - Does not throw UnauthorizedError on redirect, but instead retries + * - Calls next() instead of throwing for redirect-based auth + * + * @param provider - OAuth client provider for authentication + * @param baseUrl - Base URL for OAuth server discovery (defaults to request URL domain) + * @returns A fetch middleware function + */ +export const withOAuthRetry = ( + clientName: string, + baseUrl?: string | URL, + handle401Fn: typeof handle401 = handle401, + clientMetadataUrl?: string +): Middleware => { + const provider = new ConformanceOAuthProvider( + 'http://localhost:3000/callback', + { + client_name: clientName, + redirect_uris: ['http://localhost:3000/callback'] + }, + clientMetadataUrl + ); + return (next: FetchLike) => { + return async ( + input: string | URL, + init?: RequestInit + ): Promise => { + const makeRequest = async (): Promise => { + const headers = new Headers(init?.headers); + + // Add authorization header if tokens are available + const tokens = await provider.tokens(); + if (tokens) { + headers.set('Authorization', `Bearer ${tokens.access_token}`); + } + + return await next(input, { ...init, headers }); + }; + + let response = await makeRequest(); + + // Handle 401 responses by attempting re-authentication + if (response.status === 401 || response.status === 403) { + const serverUrl = + baseUrl || + (typeof input === 'string' ? new URL(input).origin : input.origin); + await handle401Fn(response, provider, next, serverUrl); + + response = await makeRequest(); + } + + // If we still have a 401 after re-auth attempt, throw an error + if (response.status === 401 || response.status === 403) { + const url = typeof input === 'string' ? input : input.toString(); + throw new UnauthorizedError(`Authentication failed for ${url}`); + } + + return response; + }; + }; +}; From efa15df6d80ffaa2ba52ef6ee3eb76db9de76aac Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Tue, 6 Jan 2026 10:16:48 +0000 Subject: [PATCH 02/10] fix: align everything-client.ts scenario names with conformance suite - Change tools-call to tools_call (underscore) - Change elicitation-defaults to elicitation-sep1034-client-defaults - Add sse-retry scenario handler - Update auth scenario names to match conformance suite: - auth/basic-dcr -> auth/basic-cimd - auth/basic-metadata-* -> auth/metadata-* - Add missing: auth/scope-retry-limit, auth/token-endpoint-auth-*, auth/client-credentials-* - Make tools_call handler actually call the add_numbers tool Test results: 188 passed, 9 failed, 1 warning (Known failures: elicitation defaults and client_credentials not yet supported) Claude-Generated-By: Claude Code (cli/claude-opus-4-5=100%) Claude-Steers: 0 Claude-Permission-Prompts: 0 Claude-Escapes: 0 --- src/conformance/everything-client.ts | 95 +++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 8 deletions(-) diff --git a/src/conformance/everything-client.ts b/src/conformance/everything-client.ts index f8a8ae581..3dcf19e84 100644 --- a/src/conformance/everything-client.ts +++ b/src/conformance/everything-client.ts @@ -39,7 +39,7 @@ function registerScenarios(names: string[], handler: ScenarioHandler): void { } // ============================================================================ -// Basic scenarios (initialize, tools-call) +// Basic scenarios (initialize, tools_call) // ============================================================================ async function runBasicClient(serverUrl: string): Promise { @@ -60,7 +60,37 @@ async function runBasicClient(serverUrl: string): Promise { logger.debug('Connection closed successfully'); } -registerScenarios(['initialize', 'tools-call'], runBasicClient); +// tools_call scenario needs to actually call a tool +async function runToolsCallClient(serverUrl: string): Promise { + const client = new Client( + { name: 'test-client', version: '1.0.0' }, + { capabilities: {} } + ); + + const transport = new StreamableHTTPClientTransport(new URL(serverUrl)); + + await client.connect(transport); + logger.debug('Successfully connected to MCP server'); + + const tools = await client.listTools(); + logger.debug('Successfully listed tools'); + + // Call the add_numbers tool + const addTool = tools.tools.find((t) => t.name === 'add_numbers'); + if (addTool) { + const result = await client.callTool({ + name: 'add_numbers', + arguments: { a: 5, b: 3 } + }); + logger.debug('Tool call result:', JSON.stringify(result, null, 2)); + } + + await transport.close(); + logger.debug('Connection closed successfully'); +} + +registerScenario('initialize', runBasicClient); +registerScenario('tools_call', runToolsCallClient); // ============================================================================ // Auth scenarios - well-behaved client @@ -97,16 +127,23 @@ async function runAuthClient(serverUrl: string): Promise { // Register all auth scenarios that should use the well-behaved auth client registerScenarios( [ - 'auth/basic-dcr', - 'auth/basic-metadata-var1', - 'auth/basic-metadata-var2', - 'auth/basic-metadata-var3', + 'auth/basic-cimd', + 'auth/metadata-default', + 'auth/metadata-var1', + 'auth/metadata-var2', + 'auth/metadata-var3', 'auth/2025-03-26-oauth-metadata-backcompat', 'auth/2025-03-26-oauth-endpoint-fallback', 'auth/scope-from-www-authenticate', 'auth/scope-from-scopes-supported', 'auth/scope-omitted-when-undefined', - 'auth/scope-step-up' + 'auth/scope-step-up', + 'auth/scope-retry-limit', + 'auth/token-endpoint-auth-basic', + 'auth/token-endpoint-auth-post', + 'auth/token-endpoint-auth-none', + 'auth/client-credentials-jwt', + 'auth/client-credentials-basic' ], runAuthClient ); @@ -175,7 +212,49 @@ async function runElicitationDefaultsClient(serverUrl: string): Promise { logger.debug('Connection closed successfully'); } -registerScenario('elicitation-defaults', runElicitationDefaultsClient); +registerScenario('elicitation-sep1034-client-defaults', runElicitationDefaultsClient); + +// ============================================================================ +// SSE retry scenario +// ============================================================================ + +async function runSSERetryClient(serverUrl: string): Promise { + const client = new Client( + { name: 'sse-retry-test-client', version: '1.0.0' }, + { capabilities: {} } + ); + + const transport = new StreamableHTTPClientTransport(new URL(serverUrl)); + + await client.connect(transport); + logger.debug('Successfully connected to MCP server'); + + // List tools to get the reconnection test tool + const tools = await client.listTools(); + logger.debug( + 'Available tools:', + tools.tools.map((t) => t.name) + ); + + // Call the test_reconnection tool which triggers stream closure + const testTool = tools.tools.find((t) => t.name === 'test_reconnection'); + if (!testTool) { + throw new Error('Test tool not found: test_reconnection'); + } + + logger.debug('Calling test_reconnection tool...'); + const result = await client.callTool({ + name: 'test_reconnection', + arguments: {} + }); + + logger.debug('Tool result:', JSON.stringify(result, null, 2)); + + await transport.close(); + logger.debug('Connection closed successfully'); +} + +registerScenario('sse-retry', runSSERetryClient); // ============================================================================ // Main entry point From 411a8201e3f4ff3924e9db377ff33f83ddf2494e Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Tue, 6 Jan 2026 10:38:14 +0000 Subject: [PATCH 03/10] feat: add client credentials auth scenario handlers to everything-client Add dedicated handlers for auth/client-credentials-jwt and auth/client-credentials-basic scenarios using the SDK's ClientCredentialsProvider and PrivateKeyJwtProvider. These scenarios require special handling because they use the client_credentials OAuth grant type (machine-to-machine auth) instead of the authorization code flow used by other auth scenarios. The conformance runner passes context via MCP_CONFORMANCE_CONTEXT env var containing client_id, private_key_pem/client_secret as appropriate. Claude-Generated-By: Claude Code (cli/claude-opus-4-5=100%) Claude-Steers: 0 Claude-Permission-Prompts: 0 Claude-Escapes: 0 --- src/conformance/everything-client.ts | 116 ++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 4 deletions(-) diff --git a/src/conformance/everything-client.ts b/src/conformance/everything-client.ts index 3dcf19e84..0042589d5 100644 --- a/src/conformance/everything-client.ts +++ b/src/conformance/everything-client.ts @@ -15,11 +15,45 @@ import { Client, StreamableHTTPClientTransport, - ElicitRequestSchema + ElicitRequestSchema, + ClientCredentialsProvider, + PrivateKeyJwtProvider } from '@modelcontextprotocol/client'; +import { z } from 'zod'; import { withOAuthRetry } from './helpers/withOAuthRetry.js'; import { logger } from './helpers/logger.js'; +/** + * Schema for client conformance test context passed via MCP_CONFORMANCE_CONTEXT. + * + * Each variant includes a `name` field matching the scenario name to enable + * discriminated union parsing and type-safe access to scenario-specific fields. + */ +const ClientConformanceContextSchema = z.discriminatedUnion('name', [ + z.object({ + name: z.literal('auth/client-credentials-jwt'), + client_id: z.string(), + private_key_pem: z.string(), + signing_algorithm: z.string().optional() + }), + z.object({ + name: z.literal('auth/client-credentials-basic'), + client_id: z.string(), + client_secret: z.string() + }) +]); + +/** + * Parse the conformance context from MCP_CONFORMANCE_CONTEXT env var. + */ +function parseContext() { + const raw = process.env.MCP_CONFORMANCE_CONTEXT; + if (!raw) { + throw new Error('MCP_CONFORMANCE_CONTEXT not set'); + } + return ClientConformanceContextSchema.parse(JSON.parse(raw)); +} + // Scenario handler type type ScenarioHandler = (serverUrl: string) => Promise; @@ -125,6 +159,7 @@ async function runAuthClient(serverUrl: string): Promise { } // Register all auth scenarios that should use the well-behaved auth client +// Note: client-credentials-jwt and client-credentials-basic have their own handlers below registerScenarios( [ 'auth/basic-cimd', @@ -141,13 +176,86 @@ registerScenarios( 'auth/scope-retry-limit', 'auth/token-endpoint-auth-basic', 'auth/token-endpoint-auth-post', - 'auth/token-endpoint-auth-none', - 'auth/client-credentials-jwt', - 'auth/client-credentials-basic' + 'auth/token-endpoint-auth-none' ], runAuthClient ); +// ============================================================================ +// Client Credentials scenarios +// ============================================================================ + +/** + * Client credentials with private_key_jwt authentication. + */ +async function runClientCredentialsJwt(serverUrl: string): Promise { + const ctx = parseContext(); + if (ctx.name !== 'auth/client-credentials-jwt') { + throw new Error(`Expected jwt context, got ${ctx.name}`); + } + + const provider = new PrivateKeyJwtProvider({ + clientId: ctx.client_id, + privateKey: ctx.private_key_pem, + algorithm: ctx.signing_algorithm || 'ES256' + }); + + const client = new Client( + { name: 'conformance-client-credentials-jwt', version: '1.0.0' }, + { capabilities: {} } + ); + + const transport = new StreamableHTTPClientTransport(new URL(serverUrl), { + authProvider: provider + }); + + await client.connect(transport); + logger.debug('Successfully connected with private_key_jwt auth'); + + await client.listTools(); + logger.debug('Successfully listed tools'); + + await transport.close(); + logger.debug('Connection closed successfully'); +} + +registerScenario('auth/client-credentials-jwt', runClientCredentialsJwt); + +/** + * Client credentials with client_secret_basic authentication. + */ +async function runClientCredentialsBasic(serverUrl: string): Promise { + const ctx = parseContext(); + if (ctx.name !== 'auth/client-credentials-basic') { + throw new Error(`Expected basic context, got ${ctx.name}`); + } + + const provider = new ClientCredentialsProvider({ + clientId: ctx.client_id, + clientSecret: ctx.client_secret + }); + + const client = new Client( + { name: 'conformance-client-credentials-basic', version: '1.0.0' }, + { capabilities: {} } + ); + + const transport = new StreamableHTTPClientTransport(new URL(serverUrl), { + authProvider: provider + }); + + await client.connect(transport); + logger.debug('Successfully connected with client_secret_basic auth'); + + await client.listTools(); + logger.debug('Successfully listed tools'); + + await transport.close(); + logger.debug('Connection closed successfully'); +} + +registerScenario('auth/client-credentials-basic', runClientCredentialsBasic); + // ============================================================================ // Elicitation defaults scenario // ============================================================================ From 261cbde6045e9870b0b0d59a70a32167d5168e16 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Tue, 6 Jan 2026 12:36:55 +0000 Subject: [PATCH 04/10] fix: add CIMD URL support and correct elicitation capability path --- src/conformance/everything-client.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/conformance/everything-client.ts b/src/conformance/everything-client.ts index 0042589d5..2c089bc10 100644 --- a/src/conformance/everything-client.ts +++ b/src/conformance/everything-client.ts @@ -20,9 +20,17 @@ import { PrivateKeyJwtProvider } from '@modelcontextprotocol/client'; import { z } from 'zod'; -import { withOAuthRetry } from './helpers/withOAuthRetry.js'; +import { withOAuthRetry, handle401 } from './helpers/withOAuthRetry.js'; import { logger } from './helpers/logger.js'; +/** + * Fixed client metadata URL for CIMD conformance tests. + * When server supports client_id_metadata_document_supported, this URL + * will be used as the client_id instead of doing dynamic registration. + */ +const CIMD_CLIENT_METADATA_URL = + 'https://conformance-test.local/client-metadata.json'; + /** * Schema for client conformance test context passed via MCP_CONFORMANCE_CONTEXT. * @@ -138,7 +146,9 @@ async function runAuthClient(serverUrl: string): Promise { const oauthFetch = withOAuthRetry( 'test-auth-client', - new URL(serverUrl) + new URL(serverUrl), + handle401, + CIMD_CLIENT_METADATA_URL )(fetch); const transport = new StreamableHTTPClientTransport(new URL(serverUrl), { @@ -266,7 +276,9 @@ async function runElicitationDefaultsClient(serverUrl: string): Promise { { capabilities: { elicitation: { - applyDefaults: true + form: { + applyDefaults: true + } } } } From 27ca0a6e64571d4672ce11886d0a0ff4ce5c3616 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Tue, 6 Jan 2026 12:37:31 +0000 Subject: [PATCH 05/10] feat: add npm scripts for client conformance testing --- package.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 63a832207..aceb635a9 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,9 @@ "lint:fix:all": "pnpm -r lint:fix", "check:all": "pnpm -r typecheck && pnpm -r lint", "test:all": "pnpm -r test", - "conformance:client": "npx @modelcontextprotocol/conformance client --command 'npx tsx src/conformance/everything-client.ts' --scenario initialize" + "conformance:client": "npx tsx src/conformance/everything-client.ts", + "test:conformance:client": "npx @modelcontextprotocol/conformance client --command 'npx tsx src/conformance/everything-client.ts'", + "test:conformance:client:all": "npx @modelcontextprotocol/conformance client --command 'npx tsx src/conformance/everything-client.ts' --suite all" }, "devDependencies": { "@modelcontextprotocol/client": "workspace:^", From c181f764accd11aba1df7c895c912efcc501a4d1 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Tue, 6 Jan 2026 12:49:04 +0000 Subject: [PATCH 06/10] fix: add permissions block and use correct conformance test command --- .github/workflows/conformance.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 4c7a477f1..f9c3f638d 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -6,6 +6,9 @@ on: pull_request: workflow_dispatch: +permissions: + contents: read + jobs: client-conformance: runs-on: ubuntu-latest @@ -21,4 +24,4 @@ jobs: version: 10.24.0 - run: pnpm install - run: pnpm run build:all - - run: pnpm run conformance:client + - run: pnpm run test:conformance:client:all From ce94b6466dfd059fa9dfcd5fe09a0703f6870267 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Tue, 6 Jan 2026 13:02:18 +0000 Subject: [PATCH 07/10] fix: rename conformance scripts and remove devDependency --- package.json | 9 ++++---- pnpm-lock.yaml | 56 +++----------------------------------------------- 2 files changed, 8 insertions(+), 57 deletions(-) diff --git a/package.json b/package.json index aceb635a9..76a96d7fe 100644 --- a/package.json +++ b/package.json @@ -30,16 +30,16 @@ "lint:fix:all": "pnpm -r lint:fix", "check:all": "pnpm -r typecheck && pnpm -r lint", "test:all": "pnpm -r test", - "conformance:client": "npx tsx src/conformance/everything-client.ts", "test:conformance:client": "npx @modelcontextprotocol/conformance client --command 'npx tsx src/conformance/everything-client.ts'", - "test:conformance:client:all": "npx @modelcontextprotocol/conformance client --command 'npx tsx src/conformance/everything-client.ts' --suite all" + "test:conformance:client:all": "npx @modelcontextprotocol/conformance client --command 'npx tsx src/conformance/everything-client.ts' --suite all", + "test:conformance:client:run": "npx tsx src/conformance/everything-client.ts" }, "devDependencies": { - "@modelcontextprotocol/client": "workspace:^", "@cfworker/json-schema": "catalog:runtimeShared", "@changesets/changelog-github": "^0.5.2", "@changesets/cli": "^2.29.8", "@eslint/js": "catalog:devTools", + "@modelcontextprotocol/client": "workspace:^", "@types/content-type": "catalog:devTools", "@types/cors": "catalog:devTools", "@types/cross-spawn": "catalog:devTools", @@ -60,7 +60,8 @@ "typescript": "catalog:devTools", "typescript-eslint": "catalog:devTools", "vitest": "catalog:devTools", - "ws": "catalog:devTools" + "ws": "catalog:devTools", + "zod": "catalog:runtimeShared" }, "resolutions": { "strip-ansi": "6.0.1" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d5148215f..c038fcdf3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -63,72 +63,19 @@ catalogs: typescript-eslint: specifier: ^8.48.1 version: 8.49.0 - vite-tsconfig-paths: - specifier: ^5.1.4 - version: 5.1.4 vitest: specifier: ^4.0.8 version: 4.0.9 ws: specifier: ^8.18.0 version: 8.18.3 - runtimeClientOnly: - cross-spawn: - specifier: ^7.0.5 - version: 7.0.6 - eventsource: - specifier: ^3.0.2 - version: 3.0.7 - eventsource-parser: - specifier: ^3.0.0 - version: 3.0.6 - jose: - specifier: ^6.1.1 - version: 6.1.3 - runtimeServerOnly: - '@hono/node-server': - specifier: ^1.19.7 - version: 1.19.7 - content-type: - specifier: ^1.0.5 - version: 1.0.5 - cors: - specifier: ^2.8.5 - version: 2.8.5 - express: - specifier: ^5.0.1 - version: 5.1.0 - express-rate-limit: - specifier: ^7.5.0 - version: 7.5.1 - hono: - specifier: ^4.11.1 - version: 4.11.1 - raw-body: - specifier: ^3.0.0 - version: 3.0.1 runtimeShared: '@cfworker/json-schema': specifier: ^4.1.1 version: 4.1.1 - ajv: - specifier: ^8.17.1 - version: 8.17.1 - ajv-formats: - specifier: ^3.0.1 - version: 3.0.1 - json-schema-typed: - specifier: ^8.0.2 - version: 8.0.2 - pkce-challenge: - specifier: ^5.0.0 - version: 5.0.0 zod: specifier: ^3.25 || ^4.0 version: 3.25.76 - zod-to-json-schema: - specifier: ^3.25.0 - version: 3.25.0 overrides: strip-ansi: 6.0.1 @@ -215,6 +162,9 @@ importers: ws: specifier: catalog:devTools version: 8.18.3 + zod: + specifier: catalog:runtimeShared + version: 3.25.76 common/eslint-config: dependencies: From 404374b28fed2f0a5fe850ffdfc7ca184615282e Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Thu, 8 Jan 2026 17:07:06 +0000 Subject: [PATCH 08/10] fix: address PR review comments - Change CI workflow cache from npm to pnpm - Use logger.error consistently instead of console.error - Regenerate pnpm-lock.yaml to fix spurious catalog deletions --- .github/workflows/conformance.yml | 2 +- pnpm-lock.yaml | 53 ++++++++++++++++++++++++++++ src/conformance/everything-client.ts | 18 +++++----- 3 files changed, 63 insertions(+), 10 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index f9c3f638d..5f9ad1dc6 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 24 - cache: npm + cache: pnpm - uses: pnpm/action-setup@v4 with: version: 10.24.0 diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c038fcdf3..ac398cbda 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -63,19 +63,72 @@ catalogs: typescript-eslint: specifier: ^8.48.1 version: 8.49.0 + vite-tsconfig-paths: + specifier: ^5.1.4 + version: 5.1.4 vitest: specifier: ^4.0.8 version: 4.0.9 ws: specifier: ^8.18.0 version: 8.18.3 + runtimeClientOnly: + cross-spawn: + specifier: ^7.0.5 + version: 7.0.6 + eventsource: + specifier: ^3.0.2 + version: 3.0.7 + eventsource-parser: + specifier: ^3.0.0 + version: 3.0.6 + jose: + specifier: ^6.1.1 + version: 6.1.3 + runtimeServerOnly: + '@hono/node-server': + specifier: ^1.19.7 + version: 1.19.7 + content-type: + specifier: ^1.0.5 + version: 1.0.5 + cors: + specifier: ^2.8.5 + version: 2.8.5 + express: + specifier: ^5.0.1 + version: 5.1.0 + express-rate-limit: + specifier: ^7.5.0 + version: 7.5.1 + hono: + specifier: ^4.11.1 + version: 4.11.1 + raw-body: + specifier: ^3.0.0 + version: 3.0.1 runtimeShared: '@cfworker/json-schema': specifier: ^4.1.1 version: 4.1.1 + ajv: + specifier: ^8.17.1 + version: 8.17.1 + ajv-formats: + specifier: ^3.0.1 + version: 3.0.1 + json-schema-typed: + specifier: ^8.0.2 + version: 8.0.2 + pkce-challenge: + specifier: ^5.0.0 + version: 5.0.0 zod: specifier: ^3.25 || ^4.0 version: 3.25.76 + zod-to-json-schema: + specifier: ^3.25.0 + version: 3.25.0 overrides: strip-ansi: 6.0.1 diff --git a/src/conformance/everything-client.ts b/src/conformance/everything-client.ts index 2c089bc10..4a0c588d9 100644 --- a/src/conformance/everything-client.ts +++ b/src/conformance/everything-client.ts @@ -385,25 +385,25 @@ async function main(): Promise { const serverUrl = process.argv[2]; if (!scenarioName || !serverUrl) { - console.error( + logger.error( 'Usage: MCP_CONFORMANCE_SCENARIO= everything-client ' ); - console.error( + logger.error( '\nThe MCP_CONFORMANCE_SCENARIO env var is set automatically by the conformance runner.' ); - console.error('\nAvailable scenarios:'); + logger.error('\nAvailable scenarios:'); for (const name of Object.keys(scenarioHandlers).sort()) { - console.error(` - ${name}`); + logger.error(` - ${name}`); } process.exit(1); } const handler = scenarioHandlers[scenarioName]; if (!handler) { - console.error(`Unknown scenario: ${scenarioName}`); - console.error('\nAvailable scenarios:'); + logger.error(`Unknown scenario: ${scenarioName}`); + logger.error('\nAvailable scenarios:'); for (const name of Object.keys(scenarioHandlers).sort()) { - console.error(` - ${name}`); + logger.error(` - ${name}`); } process.exit(1); } @@ -412,12 +412,12 @@ async function main(): Promise { await handler(serverUrl); process.exit(0); } catch (error) { - console.error('Error:', error); + logger.error('Error:', error); process.exit(1); } } main().catch((error) => { - console.error('Unhandled error:', error); + logger.error('Unhandled error:', error); process.exit(1); }); From e540a72254816cda04cbaf1b45324f718e708ac3 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Thu, 8 Jan 2026 18:08:56 +0000 Subject: [PATCH 09/10] fix: add conformance as pinned devDependency This avoids downloading on every run and allows manual version bumps when new conformance tests are released. --- package.json | 5 +++-- pnpm-lock.yaml | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 76a96d7fe..bb787f46a 100644 --- a/package.json +++ b/package.json @@ -30,8 +30,8 @@ "lint:fix:all": "pnpm -r lint:fix", "check:all": "pnpm -r typecheck && pnpm -r lint", "test:all": "pnpm -r test", - "test:conformance:client": "npx @modelcontextprotocol/conformance client --command 'npx tsx src/conformance/everything-client.ts'", - "test:conformance:client:all": "npx @modelcontextprotocol/conformance client --command 'npx tsx src/conformance/everything-client.ts' --suite all", + "test:conformance:client": "conformance client --command 'npx tsx src/conformance/everything-client.ts'", + "test:conformance:client:all": "conformance client --command 'npx tsx src/conformance/everything-client.ts' --suite all", "test:conformance:client:run": "npx tsx src/conformance/everything-client.ts" }, "devDependencies": { @@ -40,6 +40,7 @@ "@changesets/cli": "^2.29.8", "@eslint/js": "catalog:devTools", "@modelcontextprotocol/client": "workspace:^", + "@modelcontextprotocol/conformance": "0.1.9", "@types/content-type": "catalog:devTools", "@types/cors": "catalog:devTools", "@types/cross-spawn": "catalog:devTools", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ac398cbda..07aee5fda 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -152,6 +152,9 @@ importers: '@modelcontextprotocol/client': specifier: workspace:^ version: link:packages/client + '@modelcontextprotocol/conformance': + specifier: 0.1.9 + version: 0.1.9(@cfworker/json-schema@4.1.1)(hono@4.11.1) '@types/content-type': specifier: catalog:devTools version: 1.1.9 @@ -1072,6 +1075,20 @@ packages: '@manypkg/get-packages@1.1.3': resolution: {integrity: sha512-fo+QhuU3qE/2TQMQmbVMqaQ6EWbMhi4ABWP+O4AM1NqPBuy0OrApV5LO6BrrgnhtAHS2NH6RrVk9OL181tTi8A==} + '@modelcontextprotocol/conformance@0.1.9': + resolution: {integrity: sha512-hpR5PoW0feue3LHSi1kJNhQxbySEQNWR6McuB3QCoK0zsxIdoq+id4GxRwWVOnRnjOiTecDKMD1QMfXuurDZPQ==} + hasBin: true + + '@modelcontextprotocol/sdk@1.25.2': + resolution: {integrity: sha512-LZFeo4F9M5qOhC/Uc1aQSrBHxMrvxett+9KLHt7OhcExtoiRN9DKgbZffMP/nxjutWDQpfMDfP3nkHI4X9ijww==} + engines: {node: '>=18'} + peerDependencies: + '@cfworker/json-schema': ^4.1.1 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + '@cfworker/json-schema': + optional: true + '@napi-rs/wasm-runtime@0.2.12': resolution: {integrity: sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==} @@ -1779,6 +1796,10 @@ packages: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} + commander@14.0.2: + resolution: {integrity: sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ==} + engines: {node: '>=20'} + component-emitter@1.3.1: resolution: {integrity: sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ==} @@ -3736,6 +3757,43 @@ snapshots: globby: 11.1.0 read-yaml-file: 1.1.0 + '@modelcontextprotocol/conformance@0.1.9(@cfworker/json-schema@4.1.1)(hono@4.11.1)': + dependencies: + '@modelcontextprotocol/sdk': 1.25.2(@cfworker/json-schema@4.1.1)(hono@4.11.1)(zod@3.25.76) + commander: 14.0.2 + eventsource-parser: 3.0.6 + express: 5.1.0 + jose: 6.1.3 + zod: 3.25.76 + transitivePeerDependencies: + - '@cfworker/json-schema' + - hono + - supports-color + + '@modelcontextprotocol/sdk@1.25.2(@cfworker/json-schema@4.1.1)(hono@4.11.1)(zod@3.25.76)': + dependencies: + '@hono/node-server': 1.19.7(hono@4.11.1) + ajv: 8.17.1 + ajv-formats: 3.0.1(ajv@8.17.1) + content-type: 1.0.5 + cors: 2.8.5 + cross-spawn: 7.0.6 + eventsource: 3.0.7 + eventsource-parser: 3.0.6 + express: 5.1.0 + express-rate-limit: 7.5.1(express@5.1.0) + jose: 6.1.3 + json-schema-typed: 8.0.2 + pkce-challenge: 5.0.0 + raw-body: 3.0.1 + zod: 3.25.76 + zod-to-json-schema: 3.25.0(zod@3.25.76) + optionalDependencies: + '@cfworker/json-schema': 4.1.1 + transitivePeerDependencies: + - hono + - supports-color + '@napi-rs/wasm-runtime@0.2.12': dependencies: '@emnapi/core': 1.7.1 @@ -4401,6 +4459,8 @@ snapshots: dependencies: delayed-stream: 1.0.0 + commander@14.0.2: {} + component-emitter@1.3.1: {} concat-map@0.0.1: {} From 8ebf0b6436b2f80ac7b8e8f42b30dd949252c43d Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Thu, 8 Jan 2026 18:16:41 +0000 Subject: [PATCH 10/10] fix: install pnpm before setup-node with cache The setup-node action with cache: pnpm requires pnpm to be installed first to locate the lockfile and cache directory. --- .github/workflows/conformance.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 5f9ad1dc6..8caa40e50 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -15,13 +15,15 @@ jobs: continue-on-error: true # Non-blocking initially steps: - uses: actions/checkout@v4 + - name: Install pnpm + uses: pnpm/action-setup@v4 + with: + run_install: false - uses: actions/setup-node@v4 with: node-version: 24 cache: pnpm - - uses: pnpm/action-setup@v4 - with: - version: 10.24.0 + cache-dependency-path: pnpm-lock.yaml - run: pnpm install - run: pnpm run build:all - run: pnpm run test:conformance:client:all