From 5cd696a23fda172eb13af101e8b3bd40d289d49a Mon Sep 17 00:00:00 2001 From: Jiajie Chen Date: Thu, 21 Mar 2024 15:40:42 +0800 Subject: [PATCH] feat: initial support for reading ENVREQ --- buildit-utils/src/github.rs | 78 ++++++++++++++++++++++++++++++++++++- server/src/api.rs | 25 +++++++++--- server/src/bot.rs | 5 --- 3 files changed, 97 insertions(+), 11 deletions(-) diff --git a/buildit-utils/src/github.rs b/buildit-utils/src/github.rs index c62a7b7..fcae57f 100644 --- a/buildit-utils/src/github.rs +++ b/buildit-utils/src/github.rs @@ -7,7 +7,7 @@ use jsonwebtoken::EncodingKey; use octocrab::{models::pulls::PullRequest, params}; use std::{ borrow::Cow, - collections::{HashMap, HashSet}, + collections::{BTreeMap, HashMap, HashSet}, fs, io::{BufRead, BufReader}, path::{Path, PathBuf}, @@ -857,6 +857,82 @@ pub fn resolve_packages(pkgs: &[String], p: &Path) -> anyhow::Result Ok(req_pkgs) } +#[derive(Debug, Clone, Copy, Default)] +pub struct EnvironmentRequirement { + pub min_core: Option, + pub min_total_mem: Option, + pub min_total_mem_per_core: Option, + pub min_disk: Option, +} + +/// `packages` should have no groups nor modifiers +/// Return one ENVREQ for each arch +#[tracing::instrument(skip(p))] +pub fn get_environment_requirement( + p: &Path, + packages: &[String], +) -> BTreeMap<&'static str, EnvironmentRequirement> { + let mut res = BTreeMap::new(); + + for_each_abbs(p, |pkg, path| { + if !packages.contains(&pkg.to_string()) { + return; + } + + let spec = path.join("spec"); + let spec = std::fs::read_to_string(spec); + + if let Ok(spec) = spec { + let spec = read_ab_with_apml(&spec); + for arch in ALL_ARCH { + let res_arch: &mut EnvironmentRequirement = res.entry(*arch).or_default(); + if let Some(env_req) = spec + .get(&format!("ENV_REQ__{arch}")) + .or_else(|| spec.get("ENV_REQ")) + { + for req in env_req.split(" ") { + if let Some((key, value)) = req.split_once("=") { + let val = value.parse::(); + match (key, val) { + ("core", Ok(val)) => { + *res_arch.min_core.get_or_insert(0) = + std::cmp::max(res_arch.min_core.unwrap_or(0), val as i32); + } + ("total_mem", Ok(val)) => { + // unit: GiB -> B + *res_arch.min_total_mem.get_or_insert(0) = std::cmp::max( + res_arch.min_total_mem.unwrap_or(0), + (val as i64) * 1024 * 1024 * 1024, + ); + } + ("total_mem_per_core", Ok(val)) => { + // unit: GiB + *res_arch.min_total_mem_per_core.get_or_insert(0.0) = f32::max( + res_arch.min_total_mem_per_core.unwrap_or(0.0), + val * 1024.0 * 1024.0 * 1024.0, + ); + } + ("disk", Ok(val)) => { + // unit: GB -> B + *res_arch.min_disk.get_or_insert(0) = std::cmp::max( + res_arch.min_disk.unwrap_or(0), + (val as i64) * 1000 * 1000 * 1000, + ); + } + _ => { + warn!("Unsupported environment requirement: {}", req); + } + } + } + } + } + } + } + }); + + res +} + #[test] fn test_get_archs() { let binding = ["autobuild3".to_owned(), "autobuild4".to_owned()]; diff --git a/server/src/api.rs b/server/src/api.rs index 93f40ec..7cedead 100644 --- a/server/src/api.rs +++ b/server/src/api.rs @@ -5,7 +5,9 @@ use crate::{ }; use anyhow::anyhow; use anyhow::Context; -use buildit_utils::github::{get_archs, resolve_packages, update_abbs}; +use buildit_utils::github::{ + get_archs, get_environment_requirement, resolve_packages, update_abbs, +}; use diesel::{ dsl::count, ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl, SelectableHelper, }; @@ -98,6 +100,18 @@ pub async fn pipeline_new( String::from_utf8_lossy(&output.stdout).trim().to_string() } }; + + // find environment requirements + let resolved_pkgs = resolve_packages( + &packages + .split(",") + .map(|s| s.to_string()) + .collect::>(), + &ARGS.abbs_path, + ) + .context("Resolve packages")?; + let env_req = get_environment_requirement(&ARGS.abbs_path, &resolved_pkgs); + // create a new pipeline let mut conn = pool .get() @@ -169,6 +183,7 @@ pub async fn pipeline_new( // create a new job use crate::schema::jobs; + let env_req_current = env_req.get(*arch).cloned().unwrap_or_default(); let new_job = NewJob { pipeline_id: pipeline.id, packages: packages.to_string(), @@ -176,10 +191,10 @@ pub async fn pipeline_new( creation_time: chrono::Utc::now(), status: "created".to_string(), github_check_run_id: github_check_run_id.map(|id| id as i64), - require_min_core: None, - require_min_total_mem: None, - require_min_total_mem_per_core: None, - require_min_disk: None, + require_min_core: env_req_current.min_core, + require_min_total_mem: env_req_current.min_total_mem, + require_min_total_mem_per_core: env_req_current.min_total_mem_per_core, + require_min_disk: env_req_current.min_disk, }; diesel::insert_into(jobs::table) .values(&new_job) diff --git a/server/src/bot.rs b/server/src/bot.rs index de411a2..60459fa 100644 --- a/server/src/bot.rs +++ b/server/src/bot.rs @@ -381,11 +381,6 @@ pub async fn answer(bot: Bot, msg: Message, cmd: Command, pool: DbPool) -> Respo None }; - let pkgs = parts[2] - .split(',') - .map(|x| x.to_string()) - .collect::>(); - let archs = if parts.len() == 5 { let archs = parts[4].split(',').collect::>(); Some(handle_archs_args(archs))