Merge pull request #70 from anton-rs/merklefruit/l2-geth

feat: l2-geth and op-node containers 👨‍🚀

crates/composer/src/build_context.rs

@@ -0,0 +1,99 @@
+use std::{collections::HashMap, fs::File, io::Write, path::Path};
+
+use eyre::Result;
+use flate2::{write::GzEncoder, Compression};
+
+/// A Docker build context containing all necessary info to build a Docker image
+/// from scratch. Files and directories are copied into the build context
+/// archive from the local filesystem.
+#[derive(Debug)]
+pub struct BuildContext<P> {
+    /// The Dockerfile contents
+    pub(crate) dockerfile: String,
+    /// Files to be included in the build context
+    pub(crate) files: Vec<BuildContextObject<P>>,
+    /// Directories to be included in the build context
+    /// (recursively copied with all their contents)
+    pub(crate) dirs: Vec<BuildContextObject<P>>,
+    /// Build args to be passed to the Docker build command
+    pub(crate) buildargs: HashMap<String, String>,
+}
+
+/// A single file or directory to be included in the build context.
+#[derive(Debug)]
+pub(crate) struct BuildContextObject<P> {
+    /// The source path on the local filesystem
+    pub(crate) src: P,
+    /// The destination path in the docker image context
+    pub(crate) dest: P,
+}
+
+impl<P: AsRef<Path>> BuildContext<P> {
+    /// Create a new build context from a Dockerfile string.
+    pub fn from_dockerfile(dockerfile: &str) -> Self {
+        Self {
+            dockerfile: dockerfile.to_string(),
+            files: Vec::new(),
+            dirs: Vec::new(),
+            buildargs: HashMap::new(),
+        }
+    }
+
+    /// Add a file to the build context.
+    pub fn add_file(mut self, src: P, dest: impl Into<P>) -> Self {
+        let dest = dest.into();
+        self.files.push(BuildContextObject { src, dest });
+        self
+    }
+
+    /// Add a directory to the build context (recursively with all its contents).
+    pub fn add_dir(mut self, src: P, dest: impl Into<P>) -> Self {
+        let dest = dest.into();
+        self.dirs.push(BuildContextObject { src, dest });
+        self
+    }
+
+    /// Add a build arg to the build context.
+    pub fn add_build_arg<S>(mut self, key: S, value: S) -> Self
+    where
+        S: Into<String>,
+    {
+        self.buildargs.insert(key.into(), value.into());
+        self
+    }
+
+    /// Create a tarball and gzip the tarball. Returns the compressed output bytes.
+    /// Consumes the build context.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if the tarball cannot be created or compressed.
+    pub fn create_archive(self) -> Result<Vec<u8>> {
+        // First create a Dockerfile tarball
+        let mut header = tar::Header::new_gnu();
+        header.set_path("Dockerfile")?;
+        header.set_size(self.dockerfile.len() as u64);
+        header.set_mode(0o755);
+        header.set_cksum();
+        let mut tar = tar::Builder::new(Vec::new());
+        tar.append(&header, self.dockerfile.as_bytes())?;
+
+        // Append any additional files
+        for file in self.files {
+            let mut f = File::open(file.src)?;
+            tar.append_file(file.dest, &mut f)?;
+        }
+
+        // Append any additional directories
+        for dir in self.dirs {
+            tar.append_dir_all(dir.dest, dir.src)?;
+        }
+
+        let uncompressed = tar.into_inner()?;
+
+        // Finally, gzip the tarball
+        let mut c = GzEncoder::new(Vec::new(), Compression::default());
+        c.write_all(&uncompressed)?;
+        c.finish().map_err(Into::into)
+    }
+}

crates/composer/src/lib.rs

@@ -8,7 +8,7 @@
 #![deny(unused_must_use, rust_2018_idioms)]
 #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))]
 
-use std::{collections::HashMap, fmt::Debug};
+use std::{collections::HashMap, fmt::Debug, path::Path};
 
 use bollard::{
     container::{
@@ -28,10 +28,10 @@ pub use bollard::container::Config;
 pub use bollard::image::CreateImageOptions;
 pub use bollard::service::HostConfig;
 pub use bollard::volume::CreateVolumeOptions;
-pub use utils::bind_host_port;
+pub use build_context::BuildContext;
 
-/// Utilities for Docker operations
-mod utils;
+/// Utilities for building Docker images
+mod build_context;
 
 /// The Composer is responsible for managing the OP-UP docker containers.
 #[derive(Debug)]
@@ -108,24 +108,27 @@ impl Composer {
     /// Build a Docker image from the specified Dockerfile and build context files.
     pub async fn build_image(
         &self,
-        name: &str,
-        dockerfile: &str,
-        build_context_files: &[(&str, &[u8])],
+        name: impl Into<String>,
+        build_context: BuildContext<impl AsRef<Path>>,
     ) -> Result<()> {
         let build_options = BuildImageOptions {
-            t: name,
-            dockerfile: "Dockerfile",
+            t: name.into(),
+            dockerfile: "Dockerfile".to_string(),
+            buildargs: build_context.buildargs.clone(),
             pull: true,
             ..Default::default()
         };
 
-        let files = utils::create_dockerfile_build_context(dockerfile, build_context_files)?;
+        let build_context = build_context.create_archive()?;
         let mut image_build_stream =
             self.daemon
-                .build_image(build_options, None, Some(files.into()));
+                .build_image(build_options, None, Some(build_context.into()));
 
         while let Some(build_info) = image_build_stream.next().await {
-            let res = build_info?;
+            let res = match build_info {
+                Ok(build_info) => build_info,
+                Err(e) => eyre::bail!("Error building docker image: {:?}", e),
+            };
             tracing::debug!(target: "composer", "Build info: {:?}", res);
         }
 
@@ -181,7 +184,7 @@ impl Composer {
 
             if overwrite {
                 self.daemon
-                    .remove_container(&id, None::<RemoveContainerOptions>)
+                    .remove_container(name, None::<RemoveContainerOptions>)
                     .await?;
                 tracing::debug!(target: "composer", "Removed existing docker container {}", name);
             } else {
@@ -307,3 +310,11 @@ impl Composer {
         }
     }
 }
+
+/// Given a host port, bind it to the container.
+pub fn bind_host_port(host_port: u16) -> Option<Vec<bollard::service::PortBinding>> {
+    Some(vec![bollard::service::PortBinding {
+        host_ip: None,
+        host_port: Some(host_port.to_string()),
+    }])
+}

crates/primitives/src/artifacts.rs

@@ -44,11 +44,25 @@ impl Artifacts {
         self.path().join("genesis-l1.json")
     }
 
+    /// Returns the l2 genesis fle path.
+    pub fn l2_genesis(&self) -> PathBuf {
+        self.path().join("genesis-l2.json")
+    }
+
+    /// Returns the genesis rollup file path.
+    pub fn rollup_genesis(&self) -> PathBuf {
+        self.path().join("genesis-rollup.json")
+    }
+
     /// Returns the jwt secret file path.
     pub fn jwt_secret(&self) -> PathBuf {
         self.path().join("jwt-secret.txt")
     }
 
+    pub fn p2p_node_key(&self) -> PathBuf {
+        self.path().join("p2p-node-key.txt")
+    }
+
     /// Create the artifacts directory if it does not exist.
     pub fn create(&self) -> Result<()> {
         if !self.pwd.exists() {

crates/primitives/src/monorepo.rs

@@ -92,11 +92,6 @@ impl Monorepo {
         self.devnet().join("genesis-l1.json")
     }
 
-    /// Returns the L2 genesis file.
-    pub fn l2_genesis(&self) -> PathBuf {
-        self.devnet().join("genesis-l2.json")
-    }
-
     /// Contracts directory.
     pub fn contracts(&self) -> PathBuf {
         self.path().join("packages/contracts-bedrock")
@@ -131,11 +126,6 @@ impl Monorepo {
     pub fn op_node_dir(&self) -> PathBuf {
         self.path().join("op-node")
     }
-
-    /// Returns the genesis rollup file.
-    pub fn genesis_rollup(&self) -> PathBuf {
-        self.devnet().join("rollup.json")
-    }
 }
 
 impl Monorepo {

crates/stages/src/stages.rs

@@ -65,7 +65,10 @@ impl Stages<'_> {
                 Arc::clone(&artifacts),
                 Arc::clone(&monorepo),
             )),
-            Box::new(prestate::Prestate::new(Arc::clone(&monorepo))),
+            Box::new(prestate::Prestate::new(
+                Arc::clone(&monorepo),
+                Arc::clone(&artifacts),
+            )),
             Box::new(allocs::Allocs::new(
                 Arc::clone(&artifacts),
                 Arc::clone(&monorepo),
@@ -82,21 +85,27 @@ impl Stages<'_> {
             Box::new(l1_exec::Executor::new(
                 self.config.l1_client_port,
                 self.config.l1_client,
-                composer,
+                Arc::clone(&composer),
                 Arc::clone(&artifacts),
             )),
             Box::new(l2_genesis::L2Genesis::new(
                 self.config.l1_client_url.clone(),
                 Arc::clone(&monorepo),
+                Arc::clone(&artifacts),
             )),
             Box::new(contracts::Contracts::new()),
             Box::new(l2_exec::Executor::new(
                 self.config.l2_client_port,
                 self.config.l2_client,
+                Arc::clone(&composer),
+                Arc::clone(&artifacts),
             )),
             Box::new(rollup::Rollup::new(
                 self.config.rollup_client_port,
                 self.config.rollup_client,
+                Arc::clone(&composer),
+                Arc::clone(&monorepo),
+                Arc::clone(&artifacts),
             )),
             Box::new(proposer::Proposer::new(Arc::clone(&artifacts))),
             Box::new(batcher::Batcher::new(

crates/stages/src/stages/allocs.rs

@@ -17,7 +17,7 @@ impl crate::Stage for Allocs {
     async fn execute(&self) -> Result<()> {
         tracing::info!(target: "stages", "Executing allocs stage");
 
-        let l2_genesis_file = self.monorepo.l2_genesis();
+        let l2_genesis_file = self.artifacts.l2_genesis();
         if l2_genesis_file.exists() {
             tracing::info!(target: "stages", "l2 genesis file already found");
             return Ok(());

crates/stages/src/stages/batcher.rs

@@ -25,7 +25,7 @@ impl crate::Stage for Batcher {
         let addresses_json = self.artifacts.l1_deployments();
         let addresses = crate::json::read_json(&addresses_json)?;
 
-        let genesis_rollup_file = self.monorepo.genesis_rollup();
+        let genesis_rollup_file = self.artifacts.rollup_genesis();
         let rollup_config = crate::json::read_json(&genesis_rollup_file)?;
         let start_batcher = Command::new("docker-compose")
             .args(["up", "-d", "--no-deps", "--build", "batcher"])