diff --git a/Cargo.lock b/Cargo.lock index c1ab0890..311d23cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -212,6 +212,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +[[package]] +name = "comma" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55b672471b4e9f9e95499ea597ff64941a309b2cdbffcc46f2cc5e2d971fd335" + [[package]] name = "config" version = "0.14.0" @@ -766,6 +772,17 @@ dependencies = [ "smallvec", ] +[[package]] +name = "nix" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + [[package]] name = "nix" version = "0.28.0" @@ -1059,6 +1076,18 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +[[package]] +name = "rexpect" +version = "0.5.0" +source = "git+https://github.com/rust-cli/rexpect#9eb61dd444f25307639367ac7e1a49d5a1915d55" +dependencies = [ + "comma", + "nix 0.27.1", + "regex", + "tempfile", + "thiserror", +] + [[package]] name = "ron" version = "0.8.1" @@ -1108,7 +1137,7 @@ dependencies = [ "libc", "log", "memchr", - "nix", + "nix 0.28.0", "radix_trie", "unicode-segmentation", "unicode-width", @@ -1485,6 +1514,7 @@ dependencies = [ "petname", "rand", "regex", + "rexpect", "rustyline", "rustyline-derive", "serde", diff --git a/Cargo.toml b/Cargo.toml index 48011af6..8a63b405 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ goldenfile = "1.7.1" goldenscript = "0.7.0" hex = "0.4.3" paste = "1.0.14" +rexpect = { git = "https://github.com/rust-cli/rexpect" } # needs https://github.com/rust-cli/rexpect/pull/103 serde_json = "1.0.117" serial_test = "3.1.1" tempfile = "3.10.1" diff --git a/tests/cluster.rs b/tests/cluster.rs new file mode 100644 index 00000000..f15a7277 --- /dev/null +++ b/tests/cluster.rs @@ -0,0 +1,207 @@ +use toydb::raft::NodeID; +use toydb::Client; + +use rand::Rng; +use std::collections::BTreeMap; +use std::error::Error; +use std::fmt::Write as _; +use std::path::Path; +use std::time::Duration; + +/// Timeout for command responses and node readiness. +const TIMEOUT: Duration = Duration::from_secs(5); + +/// The base SQL port (+id). +const SQL_BASE_PORT: u16 = 19600; + +/// The base Raft port (+id). +const RAFT_BASE_PORT: u16 = 19700; + +/// Runs a toyDB cluster using the built binary in a temporary directory. The +/// cluster will be killed and removed when dropped. +/// +/// This runs the cluster as child processes using the built binary instead of +/// spawning in-memory threads for a couple of reasons: it avoids having to +/// gracefully shut down the server (which is complicated by e.g. +/// TcpListener::accept() not being interruptable), and it tests the entire +/// server (and eventually the toySQL client) end-to-end. +pub struct TestCluster { + servers: BTreeMap, + #[allow(dead_code)] + dir: tempfile::TempDir, // deleted when dropped +} + +type NodePorts = BTreeMap; // raft,sql on localhost + +impl TestCluster { + /// Runs and returns a test cluster. It keeps running until dropped. + pub fn run(nodes: u8) -> Result> { + // Create temporary directory. + let dir = tempfile::TempDir::with_prefix("toydb")?; + + // Allocate port numbers for nodes. + let ports: NodePorts = (1..=nodes) + .map(|id| (id, (RAFT_BASE_PORT + id as u16, SQL_BASE_PORT + id as u16))) + .collect(); + + // Start nodes. + let mut servers = BTreeMap::new(); + for id in 1..=nodes { + let dir = dir.path().join(format!("toydb{id}")); + servers.insert(id, TestServer::run(id, &dir, &ports)?); + } + + // Wait for the nodes to be ready, by fetching the server status. + let started = std::time::Instant::now(); + for server in servers.values_mut() { + while let Err(error) = server.connect().and_then(|mut c| Ok(c.status()?)) { + server.assert_alive(); + if started.elapsed() >= TIMEOUT { + return Err(error); + } + std::thread::sleep(Duration::from_millis(200)); + } + } + + Ok(Self { servers, dir }) + } + + /// Connects to a random cluster node using the regular client. + #[allow(dead_code)] + pub fn connect(&self) -> Result> { + let id = rand::thread_rng().gen_range(1..=self.servers.len()) as NodeID; + self.servers.get(&id).unwrap().connect() + } + + /// Connects to a random cluster node using the toysql binary. + pub fn connect_toysql(&self) -> Result> { + let id = rand::thread_rng().gen_range(1..=self.servers.len()) as NodeID; + self.servers.get(&id).unwrap().connect_toysql() + } +} + +/// A toyDB server. +pub struct TestServer { + id: NodeID, + child: std::process::Child, + sql_port: u16, +} + +impl TestServer { + /// Runs a toyDB server. + fn run(id: NodeID, dir: &Path, ports: &NodePorts) -> Result> { + // Build and write the configuration file. + let configfile = dir.join("toydb.yaml"); + std::fs::create_dir_all(dir)?; + std::fs::write(&configfile, Self::build_config(id, dir, ports)?)?; + + // Build the binary. + // TODO: this may contribute to slow start times, consider building once + // and passing it in. + let build = escargot::CargoBuild::new().bin("toydb").run()?; + + // Spawn process. Discard output. + let child = build + .command() + .args(["-c", &configfile.to_string_lossy()]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn()?; + + let (_, sql_port) = ports.get(&id).copied().expect("node not in ports"); + Ok(Self { id, child, sql_port }) + } + + /// Generates a config file for the given node. + fn build_config(id: NodeID, dir: &Path, ports: &NodePorts) -> Result> { + let (raft_port, sql_port) = ports.get(&id).expect("node not in ports"); + let mut cfg = String::new(); + writeln!(cfg, "id: {id}")?; + writeln!(cfg, "data_dir: {}", dir.to_string_lossy())?; + writeln!(cfg, "listen_raft: localhost:{raft_port}")?; + writeln!(cfg, "listen_sql: localhost:{sql_port}")?; + writeln!(cfg, "peers: {{")?; + for (peer_id, (peer_raft_port, _)) in ports.iter().filter(|(peer, _)| **peer != id) { + writeln!(cfg, " '{peer_id}': localhost:{peer_raft_port},")?; + } + writeln!(cfg, "}}")?; + Ok(cfg) + } + + /// Asserts that the server is still running. + fn assert_alive(&mut self) { + if let Some(status) = self.child.try_wait().expect("failed to check exit status") { + panic!("node {id} exited with status {status}", id = self.id) + } + } + + /// Connects to the server using a regular client. + fn connect(&self) -> Result> { + Ok(Client::connect(("localhost", self.sql_port))?) + } + + /// Connects to the server using the toysql binary. + pub fn connect_toysql(&self) -> Result> { + TestClient::connect(self.sql_port) + } +} + +impl Drop for TestServer { + // Kills the child process when dropped. + fn drop(&mut self) { + self.child.kill().expect("failed to kill node"); + self.child.wait().expect("failed to wait for node to terminate"); + } +} + +/// A toySQL client using the toysql binary. +pub struct TestClient { + session: rexpect::session::PtySession, +} + +impl TestClient { + /// Connects to a toyDB server at the given SQL port number, using + /// the toysql binary. + fn connect(port: u16) -> Result> { + // Build the binary. + let build = escargot::CargoBuild::new().bin("toysql").run()?; + + // Run it, using rexpect to manage stdin/stdout. + let mut command = build.command(); + command.args(["-p", &port.to_string()]); + let session = rexpect::spawn_with_options( + command, + rexpect::reader::Options { + timeout_ms: Some(TIMEOUT.as_millis() as u64), + strip_ansi_escape_codes: true, + }, + )?; + + // Wait for the initial prompt. + let mut client = Self { session }; + client.read_until_prompt()?; + Ok(client) + } + + /// Executes a command, returning it and the resulting toysql prompt. + pub fn execute(&mut self, command: &str) -> Result<(String, String), Box> { + let mut command = command.to_string(); + if !command.ends_with(';') && !command.starts_with('!') { + command = format!("{command};"); + } + self.session.send_line(&command)?; + self.session.exp_string(&command)?; // wait for echo + self.read_until_prompt() + } + + /// Reads output until the next prompt, returning both. + fn read_until_prompt(&mut self) -> Result<(String, String), Box> { + static UNTIL: std::sync::OnceLock = std::sync::OnceLock::new(); + let until = UNTIL.get_or_init(|| { + let re = regex::Regex::new(r"toydb(:\d+|@\d+)?>\s+").expect("invalid regex"); + rexpect::ReadUntil::Regex(re) + }); + let (output, prompt) = self.session.reader.read_until(until)?; + Ok((output.trim().to_string(), prompt.trim().to_string())) + } +} diff --git a/tests/scripts/status b/tests/scripts/status new file mode 100644 index 00000000..c828cc8a --- /dev/null +++ b/tests/scripts/status @@ -0,0 +1,17 @@ +# Tests toysql status. + +cluster nodes=1 +--- +ok + +c1:> SELECT 1 + 2 +--- +c1: 3 + +c1:> !status +--- +c1: Server: n1 with Raft leader n1 in term 1 for 1 nodes +c1: Raft log: 1 committed, 1 applied, 0.000 MB, 0% garbage (bitcask engine) +c1: Replication: n1:1 +c1: SQL storage: 1 keys, 0.000 MB logical, 1x 0.000 MB disk, 0% garbage (bitcask engine) +c1: Transactions: 0 active, 0 total diff --git a/tests/tests.rs b/tests/tests.rs index 8c005caa..006fe24e 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -1,3 +1,96 @@ #![warn(clippy::all)] -mod e2e; +mod cluster; + +use cluster::{TestClient, TestCluster}; + +use std::fmt::Write as _; +use std::{collections::HashMap, error::Error}; +use test_each_file::test_each_path; + +// Run goldenscript tests in src/raft/testscripts/node. +test_each_path! { in "tests/scripts" => test_goldenscript } + +fn test_goldenscript(path: &std::path::Path) { + goldenscript::run(&mut Runner::new(), path).expect("goldenscript failed") +} + +/// Runs Raft goldenscript tests. See run() for available commands. +struct Runner { + cluster: Option, + clients: HashMap, +} + +impl Runner { + fn new() -> Self { + Self { cluster: None, clients: HashMap::new() } + } + + /// Ensures a client exists with the given name. Does not return it, since + /// it would require taking a mutable borrow to the entire Runner instead of + /// just the clients map for the lifetime of the client borrow. + fn ensure_client(&mut self, name: &str) -> Result<(), Box> { + if self.clients.contains_key(name) { + return Ok(()); + } + let Some(cluster) = self.cluster.as_mut() else { + return Err("no cluster".into()); + }; + let client = cluster.connect_toysql()?; + self.clients.insert(name.to_string(), client); + Ok(()) + } +} + +impl goldenscript::Runner for Runner { + /// Runs a goldenscript command. + fn run(&mut self, command: &goldenscript::Command) -> Result> { + let mut output = String::new(); + let mut tags = command.tags.clone(); + + match command.name.as_str() { + // cluster nodes=N + "cluster" => { + let mut args = command.consume_args(); + let nodes = args.lookup_parse("nodes")?.unwrap_or(0); + args.reject_rest()?; + if self.cluster.is_some() { + return Err("cluster already exists".into()); + } + self.cluster = Some(TestCluster::run(nodes)?); + return Ok(output); + } + + c if command.prefix.is_none() => return Err(format!("unknown command {c}").into()), + _ => {} + } + + // Take the entire command as a toysql command and run it, using the + // prefix as a client identifier (if any). + if !command.args.is_empty() { + return Err("statements should be given as a command with no args".into()); + } + let prefix = command.prefix.as_deref().unwrap_or_default(); + self.ensure_client(prefix)?; + let client = self.clients.get_mut(prefix).expect("no client"); + let input = &command.name; + + // Execute the command and display the output. + let (stdout, prompt) = client.execute(input)?; + write!(output, "{stdout}")?; + + // If requested, also display the resulting prompt. + if tags.remove("prompt") { + // TODO: goldenscript emits a spurious prefix line if the output + // ends with \n. Fix it. + writeln!(output)?; + write!(output, "{prompt}")?; + } + + if let Some(tag) = tags.iter().next() { + return Err(format!("invalid tag {tag}").into()); + } + + Ok(output) + } +}