Initial import
This commit is contained in:
Vincent Bourdon
@@ -0,0 +1,222 @@
|
||||
# Task 17 — Integration tests + golden files (sandboxed)
|
||||
|
||||
> **Index:** [README](README.md). **Spec:** [design](../../specs/2026-04-29-quantum-tutor-design.md).
|
||||
|
||||
## Goal
|
||||
|
||||
End-to-end MCP roundtrip for the 4 tutor tools, using `QB_PROGRESS_PATH` to point each test process at a private temp file. Add golden files in `tests/reference/tutor/` (spec §6) that exercise pass/fail per module.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Task 16 merged (curriculum complete).
|
||||
|
||||
## Files
|
||||
|
||||
- Create: `tests/tutor_integration.rs`
|
||||
- Create: `tests/reference/tutor/exercises_pass.jsonl`
|
||||
- Create: `tests/reference/tutor/exercises_fail.jsonl`
|
||||
|
||||
## Steps
|
||||
|
||||
- [ ] **Step 1: Create `tests/tutor_integration.rs`**
|
||||
|
||||
```rust
|
||||
//! MCP JSON-RPC roundtrip tests for the tutor tools. Sandboxed via QB_PROGRESS_PATH.
|
||||
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::process::{Child, Command, Stdio};
|
||||
|
||||
use tempfile::TempDir;
|
||||
|
||||
struct McpProcess {
|
||||
child: Child,
|
||||
_tmp: TempDir,
|
||||
}
|
||||
|
||||
impl McpProcess {
|
||||
fn spawn() -> Self {
|
||||
let tmp = TempDir::new().expect("tempdir");
|
||||
let progress_path = tmp.path().join("progress.json");
|
||||
let child = Command::new(env!("CARGO_BIN_EXE_quantum-bridge-mcp"))
|
||||
.env("QB_PROGRESS_PATH", &progress_path)
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.expect("spawn");
|
||||
Self { child, _tmp: tmp }
|
||||
}
|
||||
|
||||
fn send(&mut self, msg: &str) {
|
||||
let stdin = self.child.stdin.as_mut().unwrap();
|
||||
writeln!(stdin, "{msg}").unwrap();
|
||||
stdin.flush().unwrap();
|
||||
}
|
||||
|
||||
fn recv(&mut self) -> serde_json::Value {
|
||||
let stdout = self.child.stdout.as_mut().unwrap();
|
||||
let mut reader = BufReader::new(stdout);
|
||||
let mut line = String::new();
|
||||
reader.read_line(&mut line).unwrap();
|
||||
serde_json::from_str(line.trim()).expect("response is JSON")
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for McpProcess {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.child.kill();
|
||||
}
|
||||
}
|
||||
|
||||
fn initialize(p: &mut McpProcess) {
|
||||
p.send(r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"test","version":"0.0.1"}}}"#);
|
||||
p.recv();
|
||||
}
|
||||
|
||||
fn extract(resp: &serde_json::Value) -> serde_json::Value {
|
||||
let text = resp["result"]["content"][0]["text"].as_str().unwrap();
|
||||
serde_json::from_str(text).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tools_list_exposes_seven_tools() {
|
||||
let mut p = McpProcess::spawn();
|
||||
initialize(&mut p);
|
||||
p.send(r#"{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}"#);
|
||||
let resp = p.recv();
|
||||
let tools = resp["result"]["tools"].as_array().unwrap();
|
||||
assert_eq!(tools.len(), 7);
|
||||
let names: Vec<&str> = tools.iter().map(|t| t["name"].as_str().unwrap()).collect();
|
||||
for required in ["get_lesson", "check_exercise", "explain_result", "get_progress"] {
|
||||
assert!(names.contains(&required), "missing {required}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn check_exercise_persists_progress_in_sandbox() {
|
||||
let mut p = McpProcess::spawn();
|
||||
initialize(&mut p);
|
||||
let circuit = r#"OPENQASM 3.0;\ninclude \"stdgates.inc\";\nqubit[1] q;\nbit[1] c;\nx q[0];\nc = measure q;"#;
|
||||
let call = format!(
|
||||
r#"{{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{{"name":"check_exercise","arguments":{{"exercise_id":"1-1-a","circuit":"{circuit}"}}}}}}"#
|
||||
);
|
||||
p.send(&call);
|
||||
let payload = extract(&p.recv());
|
||||
assert_eq!(payload["passed"], true);
|
||||
assert_eq!(payload["progress_updated"], true);
|
||||
|
||||
p.send(r#"{"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"name":"get_progress","arguments":{}}}"#);
|
||||
let progress = extract(&p.recv());
|
||||
assert!(progress["total_exercises_solved"].as_u64().unwrap() >= 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn check_exercise_invalid_circuit_returns_diagnostics_in_payload_not_protocol_error() {
|
||||
let mut p = McpProcess::spawn();
|
||||
initialize(&mut p);
|
||||
p.send(r#"{"jsonrpc":"2.0","id":5,"method":"tools/call","params":{"name":"check_exercise","arguments":{"exercise_id":"1-1-a","circuit":"not valid qasm"}}}"#);
|
||||
let resp = p.recv();
|
||||
assert!(resp.get("error").is_none(), "expected payload not error: {resp}");
|
||||
let payload = extract(&resp);
|
||||
assert_eq!(payload["passed"], false);
|
||||
assert!(payload["diagnostics"].as_array().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explain_result_returns_ast_based_breakdown_with_descriptions() {
|
||||
let mut p = McpProcess::spawn();
|
||||
initialize(&mut p);
|
||||
let call = r#"{"jsonrpc":"2.0","id":6,"method":"tools/call","params":{"name":"explain_result","arguments":{"circuit":"OPENQASM 3.0;\ninclude \"stdgates.inc\";\nqubit[2] q;\nbit[2] c;\nh q[0];\ncx q[0], q[1];\nc = measure q;","counts":{"00":512,"11":512}}}}"#;
|
||||
p.send(call);
|
||||
let payload = extract(&p.recv());
|
||||
assert_eq!(payload["key_concept"], "entanglement");
|
||||
assert_eq!(payload["num_qubits"], 2);
|
||||
let breakdown = payload["gate_breakdown"].as_array().unwrap();
|
||||
assert_eq!(breakdown.len(), 2);
|
||||
assert_eq!(breakdown[0]["name"], "h");
|
||||
assert_eq!(breakdown[1]["name"], "cx");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn golden_pass_cases_all_pass() {
|
||||
let golden = std::fs::read_to_string("tests/reference/tutor/exercises_pass.jsonl").unwrap();
|
||||
let mut p = McpProcess::spawn();
|
||||
initialize(&mut p);
|
||||
for line in golden.lines().filter(|l| !l.trim().is_empty()) {
|
||||
let case: serde_json::Value = serde_json::from_str(line).unwrap();
|
||||
let id = case["exercise_id"].as_str().unwrap();
|
||||
let circuit = case["circuit"].as_str().unwrap();
|
||||
let call = format!(
|
||||
r#"{{"jsonrpc":"2.0","id":99,"method":"tools/call","params":{{"name":"check_exercise","arguments":{{"exercise_id":"{id}","circuit":{circuit:?}}}}}}}"#
|
||||
);
|
||||
p.send(&call);
|
||||
let payload = extract(&p.recv());
|
||||
assert!(payload["passed"].as_bool().unwrap_or(false), "id={id} payload={payload}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn golden_fail_cases_all_fail() {
|
||||
let golden = std::fs::read_to_string("tests/reference/tutor/exercises_fail.jsonl").unwrap();
|
||||
let mut p = McpProcess::spawn();
|
||||
initialize(&mut p);
|
||||
for line in golden.lines().filter(|l| !l.trim().is_empty()) {
|
||||
let case: serde_json::Value = serde_json::from_str(line).unwrap();
|
||||
let id = case["exercise_id"].as_str().unwrap();
|
||||
let circuit = case["circuit"].as_str().unwrap();
|
||||
let call = format!(
|
||||
r#"{{"jsonrpc":"2.0","id":98,"method":"tools/call","params":{{"name":"check_exercise","arguments":{{"exercise_id":"{id}","circuit":{circuit:?}}}}}}}"#
|
||||
);
|
||||
p.send(&call);
|
||||
let payload = extract(&p.recv());
|
||||
assert_eq!(payload["passed"].as_bool().unwrap_or(true), false, "id={id} should fail");
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Create `tests/reference/tutor/exercises_pass.jsonl`**
|
||||
|
||||
One line per test case (≥ 1 per module). Format: `{"exercise_id": "...", "circuit": "..."}`.
|
||||
|
||||
```jsonl
|
||||
{"exercise_id": "1-1-a", "circuit": "OPENQASM 3.0;\ninclude \"stdgates.inc\";\nqubit[1] q;\nbit[1] c;\nx q[0];\nc = measure q;"}
|
||||
{"exercise_id": "1-1-b", "circuit": "OPENQASM 3.0;\ninclude \"stdgates.inc\";\nqubit[1] q;\nbit[1] c;\nc = measure q;"}
|
||||
{"exercise_id": "2-1-a", "circuit": "OPENQASM 3.0;\ninclude \"stdgates.inc\";\nqubit[1] q;\nbit[1] c;\nh q[0];\nc = measure q;"}
|
||||
{"exercise_id": "2-1-b", "circuit": "OPENQASM 3.0;\ninclude \"stdgates.inc\";\nqubit[1] q;\nbit[1] c;\nh q[0];\nh q[0];\nc = measure q;"}
|
||||
{"exercise_id": "3-1-a", "circuit": "OPENQASM 3.0;\ninclude \"stdgates.inc\";\nqubit[1] q;\nbit[1] c;\nh q[0];\nz q[0];\nh q[0];\nc = measure q;"}
|
||||
{"exercise_id": "4-1-a", "circuit": "OPENQASM 3.0;\ninclude \"stdgates.inc\";\nqubit[2] q;\nbit[2] c;\nh q[0];\ncx q[0], q[1];\nc = measure q;"}
|
||||
{"exercise_id": "5-1-a", "circuit": "OPENQASM 3.0;\ninclude \"stdgates.inc\";\nqubit[3] q;\nbit[3] c;\nh q[0];\ncx q[0], q[1];\ncx q[0], q[2];\nc = measure q;"}
|
||||
{"exercise_id": "6-1-b", "circuit": "OPENQASM 3.0;\ninclude \"stdgates.inc\";\nqubit[2] q;\nbit[1] c;\nx q[1];\nh q[0];\ncz q[0], q[1];\nh q[0];\nc[0] = measure q[0];"}
|
||||
{"exercise_id": "7-1-a", "circuit": "OPENQASM 3.0;\ninclude \"stdgates.inc\";\nqubit[2] q;\nbit[2] c;\nh q[0];\nh q[1];\ncz q[0], q[1];\nh q[0];\nh q[1];\nx q[0];\nx q[1];\ncz q[0], q[1];\nx q[0];\nx q[1];\nh q[0];\nh q[1];\nc = measure q;"}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Create `tests/reference/tutor/exercises_fail.jsonl`**
|
||||
|
||||
```jsonl
|
||||
{"exercise_id": "1-1-a", "circuit": "OPENQASM 3.0;\ninclude \"stdgates.inc\";\nqubit[1] q;\nbit[1] c;\nc = measure q;"}
|
||||
{"exercise_id": "2-1-a", "circuit": "OPENQASM 3.0;\ninclude \"stdgates.inc\";\nqubit[1] q;\nbit[1] c;\nx q[0];\nc = measure q;"}
|
||||
{"exercise_id": "4-1-a", "circuit": "OPENQASM 3.0;\ninclude \"stdgates.inc\";\nqubit[2] q;\nbit[2] c;\nh q[0];\nh q[1];\nc = measure q;"}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run the integration suite**
|
||||
|
||||
```bash
|
||||
cargo test --test tutor_integration 2>&1 | grep -E "test result|FAILED"
|
||||
```
|
||||
|
||||
Expected: every test green.
|
||||
|
||||
- [ ] **Step 5: Run the full test suite (regression check)**
|
||||
|
||||
```bash
|
||||
cargo test 2>&1 | grep -E "test result|FAILED"
|
||||
```
|
||||
|
||||
Expected: no regression in v1 suites; every tutor suite green.
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add tests/tutor_integration.rs tests/reference/tutor/
|
||||
git commit -m "test: add MCP tutor integration tests + golden files (sandboxed)"
|
||||
```
|
||||
Reference in New Issue
Block a user