Skip to content

Commit 69c08b6

Browse files
koickoic
authored
Merge pull request #652 from koic/run_every_referee_scenario_in_the_server_conformance_leg
Run every referee scenario in the server conformance leg
2 parents aeb871e + 1ab5242 commit 69c08b6

5 files changed

Lines changed: 126 additions & 10 deletions

File tree

‎.github/workflows/conformance.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ jobs:
2929
- run: bundle exec rake conformance:test
3030

3131
# Frozen requirement sets for SEP-1730 tier scoring: each spec revision's scenarios run at
32-
# that revision's own wire version, unlike the moving active suite above.
32+
# that revision's own wire version, unlike the moving full suite above.
3333
requirements-conformance:
3434
runs-on: ubuntu-latest
3535
continue-on-error: true

‎conformance/README.md‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,12 @@ Validates the Ruby SDK's conformance to the MCP specification using [`@modelcont
1515
bundle exec rake conformance:test
1616
```
1717

18-
Runs both server and client conformance tests in sequence. Server conformance starts the
19-
conformance server and tests it. Client conformance spawns test servers for each scenario
20-
and invokes `conformance/client.rb` against them. Scenarios listed in `expected_failures.yml`
21-
are allowed to fail without affecting the exit code.
18+
Runs both server and client conformance tests in sequence. Server conformance starts the conformance server and tests it.
19+
Client conformance spawns test servers for each scenario and invokes `conformance/client.rb` against them.
20+
By default both legs select every scenario the referee ships (`--suite all`), the extension-tagged ones included,
21+
so the Skills server and client implementations are checked; `SCENARIO` and `REQUIREMENTS` below replace that selection.
22+
Scenarios listed in `expected_failures.yml`, which cover the areas the SDK does not implement,
23+
are allowed to fail without affecting the exit code; the referee flags an entry as stale once its scenario passes.
2224

2325
### Environment variables
2426

@@ -55,7 +57,7 @@ bundle exec rake conformance:server PORT=3000
5557
bundle exec rake conformance:server
5658

5759
# Terminal 2: run all scenarios
58-
npx @modelcontextprotocol/conformance@alpha server --url http://localhost:9292/mcp
60+
npx @modelcontextprotocol/conformance@alpha server --url http://localhost:9292/mcp --suite all
5961

6062
# Terminal 2: run a single scenario
6163
npx @modelcontextprotocol/conformance@alpha server --url http://localhost:9292/mcp --scenario ping

‎conformance/expected_failures.yml‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,24 @@
1-
server: []
1+
server:
2+
# SEP-1699 resumability: the server sends no priming event or `retry` field on a POST SSE stream, and
3+
# the referee warns about each of the two; the baseline counts a warning as a failure. The resume on
4+
# `Last-Event-ID` is not exercised, since the server does not close the stream mid-call.
5+
- server-sse-polling
6+
# SEP-2243 custom headers: the server does not bind `Mcp-Param-*` headers to `x-mcp-header` tool parameters
7+
# (the client side sends them), so the conformance server exposes no such tool and the referee reports
8+
# the scenario's checks as untestable.
9+
- http-custom-header-server-validation
10+
# SEP-2663 (io.modelcontextprotocol/tasks): the server does not implement the tasks extension.
11+
# `tasks-status-notifications` is intentionally absent: the referee runs no check in it, and a baseline entry
12+
# for a scenario with no failing check is flagged stale.
13+
- tasks-lifecycle
14+
- tasks-capability-negotiation
15+
- tasks-wire-fields
16+
- tasks-request-state-removal
17+
- tasks-mrtr-input
18+
- tasks-request-headers
19+
- tasks-dispatch-and-envelope
20+
- tasks-required-task-error
21+
- tasks-mrtr-composition
222
client:
323
# Unimplemented authorization extensions.
424
- auth/dpop

‎conformance/server_runner.rb‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,17 @@ def build_command
3333
expected_failures_yml = File.expand_path("expected_failures.yml", __dir__)
3434

3535
npx_command = ["npx", "--yes", "@modelcontextprotocol/conformance@alpha", "server", "--url", "http://localhost:#{@port}/mcp"]
36-
npx_command += ["--scenario", @scenario] if @scenario
36+
# A requirement set already fixes which scenarios run, so it replaces the suite selection.
37+
# The referee's default `active` suite leaves out the extension-tagged scenarios (Skills, Tasks), which `all` selects,
38+
# as the client leg does; the areas the SDK does not implement are listed in expected_failures.yml.
39+
npx_command += if @requirements
40+
["--requirements", @requirements]
41+
elsif @scenario
42+
["--scenario", @scenario]
43+
else
44+
["--suite", "all"]
45+
end
3746
npx_command += ["--spec-version", @spec_version] if @spec_version
38-
npx_command += ["--requirements", @requirements] if @requirements
3947
npx_command += ["--verbose"] if @verbose
4048
npx_command += ["--expected-failures", expected_failures_yml]
4149
npx_command

‎test/conformance/server_runner_test.rb‎

Lines changed: 87 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ module Conformance
77
class ServerRunnerTest < ActiveSupport::TestCase
88
EXPECTED_FAILURES_YML = File.expand_path("../../conformance/expected_failures.yml", __dir__)
99

10-
test "build_command runs the active suite by default" do
10+
test "build_command runs the full suite by default" do
1111
command = ServerRunner.new.send(:build_command)
1212

1313
assert_equal(
@@ -18,6 +18,92 @@ class ServerRunnerTest < ActiveSupport::TestCase
1818
"server",
1919
"--url",
2020
"http://localhost:#{Server::DEFAULT_PORT}/mcp",
21+
"--suite",
22+
"all",
23+
"--expected-failures",
24+
EXPECTED_FAILURES_YML,
25+
],
26+
command,
27+
)
28+
end
29+
30+
test "build_command keeps the single scenario selection" do
31+
command = ServerRunner.new(scenario: "ping").send(:build_command)
32+
33+
assert_equal(
34+
[
35+
"npx",
36+
"--yes",
37+
"@modelcontextprotocol/conformance@alpha",
38+
"server",
39+
"--url",
40+
"http://localhost:#{Server::DEFAULT_PORT}/mcp",
41+
"--scenario",
42+
"ping",
43+
"--expected-failures",
44+
EXPECTED_FAILURES_YML,
45+
],
46+
command,
47+
)
48+
end
49+
50+
test "build_command pins the full suite to a spec version" do
51+
command = ServerRunner.new(spec_version: "2025-11-25").send(:build_command)
52+
53+
assert_equal(
54+
[
55+
"npx",
56+
"--yes",
57+
"@modelcontextprotocol/conformance@alpha",
58+
"server",
59+
"--url",
60+
"http://localhost:#{Server::DEFAULT_PORT}/mcp",
61+
"--suite",
62+
"all",
63+
"--spec-version",
64+
"2025-11-25",
65+
"--expected-failures",
66+
EXPECTED_FAILURES_YML,
67+
],
68+
command,
69+
)
70+
end
71+
72+
test "build_command pins a single scenario to a spec version" do
73+
command = ServerRunner.new(scenario: "ping", spec_version: "2025-11-25").send(:build_command)
74+
75+
assert_equal(
76+
[
77+
"npx",
78+
"--yes",
79+
"@modelcontextprotocol/conformance@alpha",
80+
"server",
81+
"--url",
82+
"http://localhost:#{Server::DEFAULT_PORT}/mcp",
83+
"--scenario",
84+
"ping",
85+
"--spec-version",
86+
"2025-11-25",
87+
"--expected-failures",
88+
EXPECTED_FAILURES_YML,
89+
],
90+
command,
91+
)
92+
end
93+
94+
test "build_command lets a requirement set replace a scenario selection" do
95+
command = ServerRunner.new(requirements: "2026-07-28", scenario: "ping").send(:build_command)
96+
97+
assert_equal(
98+
[
99+
"npx",
100+
"--yes",
101+
"@modelcontextprotocol/conformance@alpha",
102+
"server",
103+
"--url",
104+
"http://localhost:#{Server::DEFAULT_PORT}/mcp",
105+
"--requirements",
106+
"2026-07-28",
21107
"--expected-failures",
22108
EXPECTED_FAILURES_YML,
23109
],

0 commit comments

Comments
 (0)