Merge pull request #18 from AaronKutch/dev13

Version 0.13.0

.github/workflows/ci.yml

@@ -17,11 +17,13 @@ jobs:
         run: |
           rustup set profile minimal
           rustup default stable
+          rustup target add x86_64-unknown-linux-musl
       - name: Run test suite
         run: |
           cargo test --all-features
           cargo test --release --all-features
           cargo r --example paths
+          cargo r --example file_options
           cargo r --example basic_commands
           cargo r --example commands
           cargo r --example basic_containers

CHANGELOG.md

@@ -1,5 +1,38 @@
 # Changelog
 
+## [0.13.0] - 2024-06-06
+### Fixes
+- Large container networks with common build definitions are dramatically faster to start
+- Fixed a long standing issue where stdout and stderr were combined from container runners
+- unsuccesful `CommandResult`s from `Container::run` are returned now 
+- Used `*_locationless` in many more places so that errors would not be cluttered with in-library
+  locations (but all string messages now clearly state the function origin)
+- Ctrl+C on `ContainerNetwork::wait_with_timeout` now consistently returns the correct error
+- Many, many small issues were fixed
+
+### Changes
+- Total refactor of the docker module. `ContainerNetwork::new` no longer has the vector of
+  containers or internal boolean argument, instead `add_container` should be used and the
+  `--internal` should be passed through network arguments
+- `Command::get_command_result` now returns `Option<&CommandResult>`, use
+  `Command::take_command_result` for the original behavior
+- Removed `FileOptions::create` and `FileOptions::append` in favor of new functions
+- Improved some debug and display outputs
+- `CommandRunner::child_process` now holds the `ChildStdout` and `ChildStderr` if the streams have
+  no recording
+- Running containers forward with their corresponding name as the line prefix instead
+- Container building and creation messages are no longer `debug`
+- Container debug and log settings are per-container now, with only debug being on by default
+- The `ContainerNetwork` is silent by default now
+- auto_exec implementations should use `-it` by default
+- *.tmp.dockerfile should be .gitignored now in the dockerfiles directory
+
+### Additions
+- Added several `FileOptions` functions and functions for `ReadOrWrite`
+- Added some helper functions to `Command`
+- Added the ability to customize the `Command` debug line prefix
+- Added missing functions to `CommandResultNoDebug`
+
 ## [0.12.1] - 2024-05-20
 ### Fixes
 - Fixed several minor issues with the `Command` recorder forwarding to stdout

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "super_orchestrator"
-version = "0.12.1"
+version = "0.13.0"
 edition = "2021"
 authors = ["Aaron Kutch <[email protected]>"]
 license = "MIT OR Apache-2.0"
@@ -9,12 +9,13 @@ repository = "https://github.com/AaronKutch/super_orchestrator"
 documentation = "https://docs.rs/super_orchestrator"
 keywords = ["container", "docker"]
 description = "programmable container orchestration tools"
+# TODO turn into workspace with testcrate and examples crate
 
 [dependencies]
 bstr = "1"
 ctrlc = { version = "3", default-features = false }
 dunce = "1.0"
-nix = { version = "0.28", optional = true, default-features = false, features = ["signal"] }
+nix = { version = "0.29", optional = true, default-features = false, features = ["signal"] }
 owo-colors = { version = "4.0", default-features = false }
 postcard = { version = "1", features = ["use-std"] }
 serde = { version = "1.0", features = ["derive"] }

README.md

@@ -6,8 +6,8 @@ convenient tools for file management, command running, and Docker container mana
 
 First, see the documentation of `stacked_errors`
 (https://docs.rs/stacked_errors/latest/stacked_errors/) to understand the error strategy. Then, look
-over the documentation. Finally, check the examples in order of: paths, basic_commands,
-basic_containers, commands, dockerfile_entrypoint_pattern, postgres, and clean.
+over the documentation. Finally, check the examples in order of: paths, file_options,
+basic_commands, basic_containers, commands, dockerfile_entrypoint_pattern, postgres, and clean.
 
 Note that Windows has several intrinsic issues such as cross compilation being a pain (the
 dockerfile entrypoint pattern will not work without a lot of setup). Any of the examples with

dockerfiles/.gitignore

@@ -1 +1,2 @@
 __tmp.dockerfile
+*.tmp.dockerfile

dockerfiles/example.dockerfile

@@ -1 +1 @@
-FROM fedora:38
+FROM alpine:3.20

examples/auto_exec.rs

@@ -2,27 +2,21 @@ use clap::Parser;
 use stacked_errors::Result;
 use super_orchestrator::{ctrlc_init, docker_helpers::auto_exec};
 
-/// Runs `super_orchestrator::docker_helpers::auto_exec`
+/// Runs `super_orchestrator::docker_helpers::auto_exec`, `-it` is passed by
+/// default
 #[derive(Parser, Debug)]
 #[command(about)]
 struct Args {
     /// Prefix of the name of the container
     #[arg(short, long)]
     prefix: String,
-    /// Adds the `-t` argument to use a TTY, may be needed on Windows to get
-    /// around issues with carriage returns being passed.
-    #[arg(short, long, default_value_t = false)]
-    tty: bool,
 }
 
 #[tokio::main]
 async fn main() -> Result<()> {
     tracing_subscriber::fmt().init();
     ctrlc_init().unwrap();
     let args = Args::parse();
-    auto_exec(if args.tty { ["-it"] } else { ["-i"] }, &args.prefix, [
-        "sh",
-    ])
-    .await?;
+    auto_exec(["-it"], &args.prefix, ["sh"]).await?;
     Ok(())
 }

examples/basic_containers.rs

@@ -4,98 +4,109 @@ use stacked_errors::{ensure, ensure_eq, Result, StackableErr};
 use super_orchestrator::{
     docker::{Container, ContainerNetwork, Dockerfile},
     net_message::wait_for_ok_lookup_host,
-    FileOptions,
 };
+use tracing::info;
 
+const BASE_CONTAINER: &str = "fedora:40";
 const TIMEOUT: Duration = Duration::from_secs(300);
 
 #[tokio::main]
 async fn main() -> Result<()> {
-    tracing_subscriber::fmt().init();
+    // note that you need the `DEBUG` level to see some of the debug output when it
+    // is enabled
+    tracing_subscriber::fmt()
+        .with_max_level(tracing::Level::DEBUG)
+        .init();
     let logs_dir = "./logs";
 
-    println!("\n\nexample 0\n");
+    info!("\n\nexample 0\n");
 
-    // a default container configuration with the fedora:38 image
-    let container = Container::new("container0", Dockerfile::name_tag("fedora:38"));
+    // a default container configuration with the `BASE_CONTAINER` image
+    let container = Container::new("example0", Dockerfile::name_tag(BASE_CONTAINER));
 
-    // solo run `/usr/bin/ls -a /` inside the container
-    let comres = container
-        .entrypoint("/usr/bin/ls", ["-a", "/"])
-        .run(None, TIMEOUT, logs_dir, true)
-        .await
-        .stack()?;
+    // set log files to be generated in `logs_dir` and for the entrypoint and
+    // entrypoint args of the container to be equivalent to running "ls -a /" from a
+    // shell inside the container
+    let container = container.log(true).entrypoint("/usr/bin/ls", ["-a", "/"]);
+
+    // run the container in debug mode
+    let comres = container.run(None, TIMEOUT, logs_dir, true).await.stack()?;
     // and get the output as if it were run locally
     comres.assert_success().stack()?;
-    dbg!(&comres.stdout_as_utf8().stack()?);
+    dbg!(comres.stdout_as_utf8().stack()?);
+    ensure!(!comres.stdout.is_empty());
+    ensure!(comres.stderr.is_empty());
 
-    println!("\n\nexample 1\n");
+    info!("\n\nexample 1\n");
+
+    // test that stderr and errors are handled correctly
+    let comres = Container::new("example1", Dockerfile::name_tag(BASE_CONTAINER))
+        .log(true)
+        .entrypoint("/usr/bin/ls", ["-a", "/nonexistent"])
+        .run(None, TIMEOUT, logs_dir, false)
+        .await
+        .stack()?;
+    ensure!(!comres.successful());
+    dbg!(comres.stderr_as_utf8().stack()?);
+    ensure!(comres.stdout.is_empty());
+    ensure!(!comres.stderr.is_empty());
+
+    info!("\n\nexample 2\n");
 
     // sleep for 1 second
-    Container::new("container0", Dockerfile::name_tag("fedora:38"))
+    Container::new("example2", Dockerfile::name_tag(BASE_CONTAINER))
         .entrypoint("/usr/bin/sleep", ["1"])
-        .run(None, TIMEOUT, logs_dir, true)
+        .run(None, TIMEOUT, logs_dir, false)
         .await
         .stack()?
         .assert_success()
         .stack()?;
 
-    println!("\n\nexample 2\n");
+    info!("\n\nexample 3\n");
 
     // purposely timeout
-    let comres = Container::new("container0", Dockerfile::name_tag("fedora:38"))
+    let comres = Container::new("example3", Dockerfile::name_tag(BASE_CONTAINER))
         .entrypoint("/usr/bin/sleep", ["infinity"])
-        .run(None, Duration::from_secs(1), logs_dir, true)
+        .run(None, Duration::from_secs(1), logs_dir, false)
         .await;
     dbg!(&comres);
     ensure!(comres.unwrap_err().is_timeout());
 
-    println!("\n\nexample 3\n");
+    info!("\n\nexample 4\n");
 
     // read from a local folder that is mapped to the container's filesystem with a
     // volume
-    let comres = Container::new("container0", Dockerfile::name_tag("fedora:38"))
+    let comres = Container::new("example4", Dockerfile::name_tag(BASE_CONTAINER))
         .entrypoint("/usr/bin/cat", ["/dockerfile_resources/example.txt"])
         .volume(
             "./dockerfiles/dockerfile_resources/",
             "/dockerfile_resources/",
         )
-        .run(None, TIMEOUT, logs_dir, true)
+        .run(None, TIMEOUT, logs_dir, false)
         .await
         .stack()?;
     comres.assert_success().stack()?;
-    ensure_eq!(
-        comres.stdout_as_utf8().stack()?,
-        FileOptions::read_to_string("./dockerfiles/dockerfile_resources/example.txt")
-            .await
-            .stack()?
-    );
+    ensure_eq!(comres.stdout_as_utf8().stack()?, "hello from example.txt");
 
-    println!("\n\nexample 4\n");
+    info!("\n\nexample 5\n");
 
     // for more complicated things we need `ContainerNetwork`s
-    let mut cn = ContainerNetwork::new(
-        "test",
-        vec![
-            Container::new("container0", Dockerfile::name_tag("fedora:38"))
-                .entrypoint("/usr/bin/sleep", ["3"]),
-        ],
-        None,
-        true,
-        logs_dir,
+    let mut cn = ContainerNetwork::new("test", None, logs_dir);
+    cn.add_container(
+        Container::new("example5", Dockerfile::name_tag(BASE_CONTAINER))
+            .entrypoint("/usr/bin/sleep", ["3"]),
     )
     .stack()?;
     // run all containers
-    cn.run_all(true).await.stack()?;
-
-    let uuid = cn.uuid_as_string();
-    // when communicating inside a container to another container in the network,
-    // you would use a hostname like this
-    let host = format!("container0_{uuid}");
-    dbg!(&host);
-    // but outside we need the IP address
+    cn.run_all().await.stack()?;
+
+    // when communicating inside a container to another container in the same
+    // network, you can use the `container_name` of the container as the
+    // hostname, in this case "example5"
+
+    // but outside the network we need the IP address
     let host_ip = cn
-        .wait_get_ip_addr(20, Duration::from_millis(300), "container0")
+        .wait_get_ip_addr(20, Duration::from_millis(300), "example5")
         .await
         .stack()?;
     dbg!(&host_ip);

examples/clean.rs

@@ -1,8 +1,7 @@
-use stacked_errors::{ensure, StackableErr};
+use stacked_errors::ensure;
 use super_orchestrator::{
-    acquire_dir_path, acquire_file_path, remove_files_in_dir, stacked_errors::Result, FileOptions,
+    acquire_file_path, remove_files_in_dir, stacked_errors::Result, FileOptions,
 };
-use tokio::fs;
 
 #[tokio::main]
 async fn main() -> Result<()> {
@@ -11,8 +10,8 @@ async fn main() -> Result<()> {
     // note: in regular use you would use `.await.stack()?` on the ends
     // to tell what lines are failing
 
-    // remove special temporary
-    remove_files_in_dir("./dockerfiles", &["__tmp.dockerfile"]).await?;
+    // remove special temporaries
+    remove_files_in_dir("./dockerfiles", &["__tmp.dockerfile", ".tmp.dockerfile"]).await?;
     // remove log files only
     remove_files_in_dir("./logs", &[".log"]).await?;
 
@@ -72,9 +71,9 @@ async fn main() -> Result<()> {
     ensure!(acquire_file_path("./logs/tar.gz").await.is_err());
 
     // may need to sudo remove because of docker
-    if let Ok(pg_data_dir) = acquire_dir_path("./logs/pg_data").await {
+    /*if let Ok(pg_data_dir) = acquire_dir_path("./logs/pg_data").await {
         fs::remove_dir_all(pg_data_dir).await.stack()?;
-    }
+    }*/
 
     Ok(())
 }

examples/commands.rs

@@ -25,6 +25,8 @@ async fn main() -> Result<()> {
     let args = Args::parse();
 
     if args.nonutf8 {
+        // testing that the recorder handles non-UTF8 correctly by converting to the
+        // replacement symbol
         let mut bytes = vec![];
         bytes.extend("\u{1f60a}".as_bytes());
         for i in 0..=u8::MAX {
@@ -40,6 +42,14 @@ async fn main() -> Result<()> {
             }
         }
         bytes.push(b'\n');
+        std::io::stdout()
+            .write_all("(stdout)".as_bytes())
+            .stack()
+            .unwrap();
+        std::io::stderr()
+            .write_all("(stderr)".as_bytes())
+            .stack()
+            .unwrap();
         // check that starting with a cutoff multibyte char is continued
         let mut i = 5;
         for chunk in bytes.chunk_by(|_, _| {
@@ -145,6 +155,34 @@ async fn main() -> Result<()> {
     ensure!(comres.stderr_as_utf8().is_err());
     dbg!(comres.stdout_as_utf8_lossy());
     dbg!(comres.stderr_as_utf8_lossy());
+    dbg!(comres).assert_success().stack()?;
+
+    // check command debug
+    let command = Command::new("ls")
+        .arg("-la")
+        .env_clear(true)
+        .envs([("TEST0", "test0"), ("TEST1", "test1")])
+        .cwd("./")
+        .recording(false)
+        .stderr_debug(true)
+        .stderr_log(Some(FileOptions::write("./hello.txt")))
+        .record_limit(Some(9))
+        .log_limit(Some(8))
+        .forget_on_drop(true);
+    dbg!(command);
+
+    // check custom prefixes
+    let command = Command::new("cargo r --example commands -- --print --to-stdout hello")
+        .debug(true)
+        .stdout_debug_line_prefix(Some("stdout |".to_owned()))
+        .stderr_debug_line_prefix(Some("stderr |".to_owned()));
+    dbg!(&command);
+    command
+        .run_to_completion()
+        .await
+        .stack()?
+        .assert_success()
+        .stack()?;
 
     Ok(())
 }