Reuse Template Service (#345)

* base template service in progress

* reuse code from base, and simplify errors in template service

Cargo.toml

@@ -8,6 +8,7 @@ members = [
     "golem-common",
     "golem-service-base",
     "golem-template-compilation-service",
+    "golem-template-service-base",
     "golem-template-service",
     "golem-shard-manager",
     "golem-worker-executor-base",
@@ -67,6 +68,7 @@ futures-util = "0.3.29"
 golem-wasm-ast = "0.2.1"
 golem-wasm-rpc = { version = "0.0.16", default-features = false, features = ["host"] }
 http = "1.0.0" # keep in sync with wasmtime
+http_02 = { package = "http", version = "0.2.11" }
 humantime-serde = "1.1.1"
 hyper = { version = "1.0.1", features = ["full"] } # keep in sync with wasmtime
 iso8601-timestamp = "0.2.16"

golem-service-base/Cargo.toml

@@ -16,7 +16,7 @@ aws-config = { workspace = true }
 aws-sdk-s3 = { workspace = true }
 bigdecimal = "0.4.2"
 http = { workspace = true }
-http_02 = { package = "http", version = "0.2.11" }
+http_02 = { workspace = true }
 hyper = { workspace = true }
 num-traits = "0.2"
 poem-openapi = { workspace = true }

golem-shard-manager/Cargo.toml

@@ -23,7 +23,7 @@ figment = { workspace = true }
 fred = { workspace = true }
 futures = { workspace = true }
 http = { workspace = true }
-http_02 = { package = "http", version = "0.2.11" }
+http_02 = { workspace = true }
 humantime-serde = { workspace = true }
 kube = { version = "0.88.1", features = ["client"], optional = true }
 k8s-openapi = { version = "0.21.1", features = ["earliest"], optional = true }

golem-template-compilation-service/Cargo.toml

@@ -17,7 +17,7 @@ golem-worker-executor-base = { version = "0.0.0", path = "../golem-worker-execut
 async-trait = { workspace = true }
 futures = { workspace = true }
 http = { workspace = true }
-http_02 = { package = "http", version = "0.2.11" }
+http_02 = { workspace = true }
 tokio = { workspace = true }
 
 tracing = { workspace = true }

golem-template-compilation-service/src/config.rs

@@ -51,7 +51,7 @@ impl TemplateServiceConfig {
             .authority(format!("{}:{}", self.host, self.port).as_str())
             .path_and_query("/")
             .build()
-            .expect("Failed to build TemplateServiceTemplateService  URI")
+            .expect("Failed to build TemplateService URI")
     }
 }
 

golem-template-service-base/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "golem-template-service-base"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+async-trait = { workspace = true } 
+golem-api-grpc = { path = "../golem-api-grpc", version = "0.0.0" }
+golem-common = { path = "../golem-common", version = "0.0.0" }
+golem-service-base = { path = "../golem-service-base" }
+golem-wasm-ast = { workspace = true }
+http_02 = { workspace = true }
+serde = { workspace = true }
+thiserror = { workspace = true }
+tracing = { workspace = true }

golem-template-service-base/src/config.rs

@@ -0,0 +1,27 @@
+use serde::Deserialize;
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(tag = "type", content = "config")]
+#[derive(Default)]
+pub enum TemplateCompilationConfig {
+    Enabled(TemplateCompilationEnabledConfig),
+    #[default]
+    Disabled,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+pub struct TemplateCompilationEnabledConfig {
+    pub host: String,
+    pub port: u16,
+}
+
+impl TemplateCompilationEnabledConfig {
+    pub fn uri(&self) -> http_02::Uri {
+        http_02::Uri::builder()
+            .scheme("http")
+            .authority(format!("{}:{}", self.host, self.port).as_str())
+            .path_and_query("/")
+            .build()
+            .expect("Failed to build TemplateCompilationService URI")
+    }
+}

golem-template-service-base/src/lib.rs

@@ -0,0 +1,2 @@
+pub mod config;
+pub mod service;

golem-template-service-base/src/service/mod.rs

@@ -0,0 +1,2 @@
+pub mod template_compilation;
+pub mod template_processor;

golem-template-service/src/service/template_compilation.rs → golem-template-service-base/src/service/template_compilation.rs

@@ -15,10 +15,7 @@ pub struct TemplateCompilationServiceDefault {
 }
 
 impl TemplateCompilationServiceDefault {
-    pub fn new(host: String, port: u16) -> Self {
-        let uri = format!("http://{}:{}", host, port)
-            .parse()
-            .expect("Failed to parse TemplateCompilationService URI");
+    pub fn new(uri: http_02::Uri) -> Self {
         Self { uri }
     }
 }

golem-template-service-base/src/service/template_processor.rs

@@ -0,0 +1,89 @@
+use std::fmt::{self, Display, Formatter};
+
+use golem_service_base::model::TemplateMetadata;
+use golem_wasm_ast::{
+    analysis::{AnalysedExport, AnalysedFunction, AnalysisContext, AnalysisFailure},
+    component::Component,
+    IgnoreAllButMetadata,
+};
+
+pub fn process_template(data: &[u8]) -> Result<TemplateMetadata, TemplateProcessingError> {
+    let component = Component::<IgnoreAllButMetadata>::from_bytes(data)
+        .map_err(TemplateProcessingError::Parsing)?;
+
+    let producers = component
+        .get_all_producers()
+        .into_iter()
+        .map(|producers| producers.into())
+        .collect::<Vec<_>>();
+
+    let state = AnalysisContext::new(component);
+
+    let mut exports = state
+        .get_top_level_exports()
+        .map_err(TemplateProcessingError::Analysis)?;
+
+    add_resource_drops(&mut exports);
+
+    let exports = exports
+        .into_iter()
+        .map(|export| export.into())
+        .collect::<Vec<_>>();
+
+    Ok(TemplateMetadata { exports, producers })
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum TemplateProcessingError {
+    Parsing(String),
+    Analysis(AnalysisFailure),
+}
+
+impl Display for TemplateProcessingError {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        match self {
+            TemplateProcessingError::Parsing(e) => write!(f, "Parsing error: {}", e),
+            TemplateProcessingError::Analysis(source) => {
+                let AnalysisFailure::Failed(error) = source;
+                write!(f, "Analysis error: {}", error)
+            }
+        }
+    }
+}
+
+fn add_resource_drops(exports: &mut Vec<AnalysedExport>) {
+    // Components are not exporting explicit drop functions for exported resources, but
+    // worker executor does. So we keep golem-wasm-ast as an universal library and extend
+    // its result with the explicit drops here, for each resource, identified by an exported
+    // constructor.
+
+    let mut to_add = Vec::new();
+    for export in exports.iter_mut() {
+        match export {
+            AnalysedExport::Function(fun) => {
+                if fun.is_constructor() {
+                    let drop_name = fun.name.replace("[constructor]", "[drop]");
+                    to_add.push(AnalysedExport::Function(AnalysedFunction {
+                        name: drop_name,
+                        ..fun.clone()
+                    }));
+                }
+            }
+            AnalysedExport::Instance(instance) => {
+                let mut to_add = Vec::new();
+                for fun in &instance.funcs {
+                    if fun.is_constructor() {
+                        let drop_name = fun.name.replace("[constructor]", "[drop]");
+                        to_add.push(AnalysedFunction {
+                            name: drop_name,
+                            ..fun.clone()
+                        });
+                    }
+                }
+                instance.funcs.extend(to_add.into_iter());
+            }
+        }
+    }
+
+    exports.extend(to_add);
+}

golem-template-service/Cargo.toml

@@ -49,4 +49,7 @@ tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
 uuid = { workspace = true }
 humantime-serde = { workspace = true }
-http_02 = { package = "http", version = "0.2.11" }
+http_02 = { workspace = true }
+golem-template-service-base = { version = "0.1.0", path = "../golem-template-service-base" }
+thiserror.workspace = true
+anyhow.workspace = true

golem-template-service/src/api/template.rs

@@ -51,44 +51,33 @@ pub struct UploadPayload {
 type Result<T> = std::result::Result<T, TemplateError>;
 
 impl From<TemplateServiceError> for TemplateError {
-    fn from(value: TemplateServiceError) -> Self {
-        match value {
-            TemplateServiceError::Internal(error) => {
-                TemplateError::InternalError(Json(ErrorBody { error }))
-            }
-            TemplateServiceError::TemplateProcessingError(error) => {
-                TemplateError::BadRequest(Json(ErrorsBody {
-                    errors: vec![error],
-                }))
-            }
-            TemplateServiceError::UnknownTemplateId(_) => {
+    fn from(error: TemplateServiceError) -> Self {
+        match error {
+            TemplateServiceError::UnknownTemplateId(_)
+            | TemplateServiceError::UnknownVersionedTemplateId(_) => {
                 TemplateError::NotFound(Json(ErrorBody {
-                    error: "Template not found".to_string(),
+                    error: error.to_string(),
                 }))
             }
-            TemplateServiceError::UnknownVersionedTemplateId(_) => {
-                TemplateError::NotFound(Json(ErrorBody {
-                    error: "Template not found".to_string(),
+            TemplateServiceError::AlreadyExists(_) => {
+                TemplateError::AlreadyExists(Json(ErrorBody {
+                    error: error.to_string(),
                 }))
             }
-            TemplateServiceError::IOError(error) => {
-                TemplateError::InternalError(Json(ErrorBody { error }))
+            TemplateServiceError::Internal(error) => {
+                TemplateError::InternalError(Json(ErrorBody {
+                    error: error.to_string(),
+                }))
             }
-            TemplateServiceError::AlreadyExists(_) => {
-                TemplateError::AlreadyExists(Json(ErrorBody {
-                    error: "Template already exists".to_string(),
+            TemplateServiceError::TemplateProcessingError(error) => {
+                TemplateError::BadRequest(Json(ErrorsBody {
+                    errors: vec![error.to_string()],
                 }))
             }
         }
     }
 }
 
-impl From<String> for TemplateError {
-    fn from(value: String) -> Self {
-        TemplateError::InternalError(Json(ErrorBody { error: value }))
-    }
-}
-
 impl From<ReadBodyError> for TemplateError {
     fn from(value: ReadBodyError) -> Self {
         TemplateError::InternalError(Json(ErrorBody {

golem-template-service/src/config.rs

@@ -15,6 +15,7 @@
 use figment::providers::{Env, Format, Toml};
 use figment::Figment;
 use golem_service_base::config::TemplateStoreConfig;
+use golem_template_service_base::config::TemplateCompilationConfig;
 use serde::Deserialize;
 use std::time::Duration;
 
@@ -49,7 +50,6 @@ pub struct TemplateServiceConfig {
     pub db: DbConfig,
     pub template_store: TemplateStoreConfig,
     pub compilation: TemplateCompilationConfig,
-    pub worker_executor_client_cache: WorkerExecutorClientCacheConfig,
 }
 
 #[derive(Clone, Debug, Deserialize)]
@@ -79,21 +79,6 @@ pub struct DbPostgresConfig {
     pub schema: Option<String>,
 }
 
-#[derive(Clone, Debug, Deserialize)]
-#[serde(tag = "type", content = "config")]
-#[derive(Default)]
-pub enum TemplateCompilationConfig {
-    Enabled(TemplateCompilationEnabledConfig),
-    #[default]
-    Disabled,
-}
-
-#[derive(Clone, Debug, Deserialize)]
-pub struct TemplateCompilationEnabledConfig {
-    pub host: String,
-    pub port: u16,
-}
-
 impl TemplateServiceConfig {
     pub fn new() -> Self {
         Figment::new()
@@ -114,7 +99,6 @@ impl Default for TemplateServiceConfig {
             db: DbConfig::default(),
             template_store: TemplateStoreConfig::default(),
             compilation: TemplateCompilationConfig::default(),
-            worker_executor_client_cache: WorkerExecutorClientCacheConfig::default(),
         }
     }
 }