From b09ab4b687a35391d3fe710fae719fae7c13c1de Mon Sep 17 00:00:00 2001 From: Cappy Ishihara Date: Sat, 23 Sep 2023 23:54:33 +0700 Subject: [PATCH] (wip) refactor code, initial disk support --- .github/workflows/arm-build.yml | 45 ++++ src/cfg.rs | 50 ++++- src/creator.rs | 361 +++++++++++++++++++++++++++----- src/main.rs | 29 ++- tests/.gitignore | 2 + tests/katsudon-arm.yaml | 45 ++++ 6 files changed, 464 insertions(+), 68 deletions(-) create mode 100644 .github/workflows/arm-build.yml create mode 100644 tests/katsudon-arm.yaml diff --git a/.github/workflows/arm-build.yml b/.github/workflows/arm-build.yml new file mode 100644 index 0000000..0a32460 --- /dev/null +++ b/.github/workflows/arm-build.yml @@ -0,0 +1,45 @@ +name: ARM cross-build test + +on: + push: + +jobs: + unit-test: + runs-on: ubuntu-latest + container: + image: ghcr.io/terrapkg/builder:f38 + # priv + options: --cap-add=SYS_ADMIN --privileged + + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: | + dnf install -y \ + xorriso \ + rpm \ + limine \ + systemd \ + btrfs-progs \ + e2fsprogs \ + xfsprogs \ + dosfstools \ + grub2 \ + grub2-efi \ + uboot-images-armv8 \ + uboot-tools \ + rustc \ + cargo + + - uses: Swatinem/rust-cache@v2 + + - name: Build and install katsu + run: | + cargo install --path . + - name: Run test + run: | + pushd tests + katsu katsudon-arm.yaml + popd + diff --git a/src/cfg.rs b/src/cfg.rs index 6a60cd1..2341a13 100644 --- a/src/cfg.rs +++ b/src/cfg.rs @@ -3,7 +3,33 @@ use std::path::PathBuf; use serde_derive::Deserialize; use smartstring::alias::String as SStr; -#[derive(Deserialize, Debug)] +#[derive(Deserialize, Debug, Default, Clone)] +#[serde(rename_all = "lowercase")] +pub enum OutputFormat { + /// The ISO file will be created. + /// This is the default. + #[default] + Iso, + /// Generates a disk image + /// This is not implemented yet. + Disk, +} + +// from string to enum +impl From<&str> for OutputFormat { + fn from(value: &str) -> Self { + match value.to_lowercase().as_str() { + "iso" => Self::Iso, + "disk" => Self::Disk, + _ => { + tracing::warn!("Unknown format: {}, setting ISO mode", value); + Self::Iso + } + } + } +} + +#[derive(Deserialize, Debug, Clone)] pub struct Config { /// The name of the distro / edition. /// This is used to name the directory with the ISO content. @@ -26,9 +52,27 @@ pub struct Config { pub dnf: Option, /// The volume id of the ISO file. pub volid: String, + /// The system architecture. + /// - `x86` (default) + pub arch: Option, + /// Output format. + pub format: OutputFormat, + + /// The disk layout of the new system. + pub disk: Option, +} +#[derive(Deserialize, Debug, Clone)] +pub struct DiskLayout { + /// Create bootloader partition? + pub bootloader: bool, + /// Filesystem of the bootloader partition. + pub root_format: String, + /// Total size of the disk image. + pub disk_size: String, } -#[derive(Deserialize, Debug)] + +#[derive(Deserialize, Debug, Clone)] pub struct System { /// The release version of the new system. pub releasever: u8, @@ -46,7 +90,7 @@ pub struct System { pub kernel_params: Option, } -#[derive(Deserialize, Debug)] +#[derive(Deserialize, Debug, Clone)] pub struct Script { /// The path to the init script. /// The init script is run after the mountpoint is created but before the packages are installed. diff --git a/src/creator.rs b/src/creator.rs index 2a40b52..b048b94 100644 --- a/src/creator.rs +++ b/src/creator.rs @@ -1,16 +1,25 @@ use color_eyre::{eyre::eyre, Help, Result}; -use std::{io::Write, path::Path}; +use std::{ + fs, + io::Write, + path::{Path, PathBuf}, +}; use tracing::{debug, error, info, instrument, trace, warn}; -use crate::{cfg::Config, run}; +use crate::{ + cfg::{Config, OutputFormat}, + run, + util::Arch, +}; const DEFAULT_DNF: &str = "dnf5"; const DEFAULT_BOOTLOADER: &str = "limine"; +const UBOOT_DATA: &str = "/usr/share/uboot"; -pub trait LiveImageCreator { +pub trait ImageCreator { /// src, dest, required const EFI_FILES: &'static [(&'static str, &'static str, bool)]; - const ARCH: crate::util::Arch; + // const ARCH: crate::util::Arch; fn get_cfg(&self) -> &Config; @@ -58,13 +67,40 @@ pub trait LiveImageCreator { Ok(()) } + fn copy_uboot_files(&self, bootpath: &Path) -> Result<()> { + info!("Copying U-Boot files"); + // copy u-boot files to bootpath + + // dictionary of files to copy and destination + + let files = vec![ + ("rpi_4/u-boot.bin", "rpi4-u-boot.bin"), + ("rpi_3/u-boot.bin", "rpi3-u-boot.bin"), + ("rpi_arm64/u-boot.bin", "rpi-u-boot.bin"), + ]; + + // fs::copy(UBOOT_DATA, bootpath)?; + + Ok(()) + } + + fn get_arch(&self) -> Result { + let cfg = self.get_cfg(); + Ok(match cfg.arch.as_ref().is_some() { + true => Arch::from(cfg.arch.as_ref().unwrap().as_str()), + false => Arch::get()?, + }) + } + fn copy_efi_files(&self, instroot: &Path) -> Result { info!("Copying EFI files"); + // get config arch + let arch_str: &str = self.get_arch()?.into(); let mut fail = false; std::fs::create_dir_all(Path::new(instroot).join("EFI/BOOT/fonts"))?; for (srcs, dest, req) in Self::EFI_FILES { - let srcs = srcs.replace("%arch%", Self::ARCH.into()); - let dest = dest.replace("%arch%", Self::ARCH.into()); + let srcs = srcs.replace("%arch%", arch_str); + let dest = dest.replace("%arch%", arch_str); let root = &self.get_cfg().instroot.canonicalize().expect("Cannot canonicalize instroot."); let root = root.to_str().unwrap(); @@ -99,7 +135,18 @@ pub trait LiveImageCreator { Ok(fail) } + /// Redirects to output-specific functions fn exec(&self) -> Result<()> { + let cfg = self.get_cfg(); + let out_fmt = &cfg.format; + + match out_fmt { + OutputFormat::Iso => self.exec_iso(), + OutputFormat::Disk => self.exec_disk(), + } + } + + fn exec_iso(&self) -> Result<()> { self.mkmountpt()?; self.init_script()?; self.instpkgs()?; @@ -115,8 +162,32 @@ pub trait LiveImageCreator { Ok(()) } + fn exec_disk(&self) -> Result<()> { + self.mkmountpt()?; + self.init_script()?; + self.instpkgs()?; + // self.dracut()?; + // self.rootpw()?; + // self.postinst_script()?; + + // self.squashfs()?; + // self.liveos()?; + // self.xorriso()?; + // self.bootloader()?; + let cfg = self.get_cfg(); + info!("Done: {}.raw", cfg.out); + Ok(()) + } + fn bootloader(&self) -> Result<()> { - match self.get_cfg().sys.bootloader.as_ref().map(|x| x.as_str()).unwrap_or(DEFAULT_BOOTLOADER) { + match self + .get_cfg() + .sys + .bootloader + .as_ref() + .map(|x| x.as_str()) + .unwrap_or(DEFAULT_BOOTLOADER) + { "limine" => self.limine(), "grub" => self.grub(), x => Err(eyre!("Unknown bootloader: {x}")), @@ -169,7 +240,9 @@ pub trait LiveImageCreator { f.write_fmt(format_args!("TIMEOUT=5\n\n:{distro}\n\tPROTOCOL=linux\n\t"))?; f.write_fmt(format_args!("KERNEL_PATH=boot:///boot/vmlinuz-{kver}\n\t"))?; f.write_fmt(format_args!("MODULE_PATH=boot:///boot/initramfs-{kver}.img\n\t"))?; - f.write_fmt(format_args!("CMDLINE=root=live:LABEL={volid} rd.live.image selinux=0 {cmdline}"))?; // maybe enforcing=0 + f.write_fmt(format_args!( + "CMDLINE=root=live:LABEL={volid} rd.live.image selinux=0 {cmdline}" + ))?; // maybe enforcing=0 Ok(()) } @@ -303,6 +376,137 @@ pub trait LiveImageCreator { self.get_cfg().sys.releasever.to_string() } + fn prep_disk(&self) -> Result<()> { + let cfg = self.get_cfg(); + + if let Some(layout) = &cfg.disk { + // Now let's create a disk file called {out}.raw + let out_file = format!("{}.raw", cfg.out); + + // Create the disk file + + let disk_size = &layout.disk_size; + info!(out_file, "Creating disk file"); + + cmd_lib::run_cmd!( + fallocate -l $disk_size $out_file; + )?; + + // Mount disk image to loop device, and return the loop device name + + info!("Mounting disk image to loop device"); + + let loop_dev = cmd_lib::run_fun!(losetup -f)?; + + debug!("Found loop device: {loop_dev:?}"); + + cmd_lib::run_cmd!( + losetup $loop_dev $out_file -v; + )?; + + // Partition disk + + info!("Partitioning disk"); + + // Create partition table, GPT + + cmd_lib::run_cmd!( + parted -s $loop_dev mklabel gpt; + )?; + + // number to track partition number + + let mut part_num = 1; + let mut efi_num: Option = None; + let boot_num: i32; + let root_num: i32; + + if layout.bootloader { + // create EFI partition with ESP flag for the first 250MiB + // label it as EFI + + cmd_lib::run_cmd!( + parted -s $loop_dev mkpart primary fat32 1MiB 250MiB; + parted -s $loop_dev set $part_num esp on; + parted -s $loop_dev name $part_num EFI; + )?; + + // format EFI partition + + cmd_lib::run_cmd!( + mkfs.fat -F32 ${loop_dev}p$part_num -n EFI; + )?; + efi_num = Some(part_num); + + // increment partition number + part_num += 1; + } + + // create boot partition for installing kernels with the next 1GiB + // label as BOOT + // ext4 + cmd_lib::run_cmd!( + parted -s $loop_dev mkpart primary ext4 250MiB 1.25GiB; + parted -s $loop_dev name $part_num BOOT; + )?; + + cmd_lib::run_cmd!( + mkfs.ext4 -F ${loop_dev}p$part_num -L BOOT; + )?; + + boot_num = part_num; + + part_num += 1; + + // Create blank partition with the rest of the free space + + let volid = &cfg.volid; + cmd_lib::run_cmd!( + parted -s $loop_dev mkpart primary ext4 1.25GiB 100%; + parted -s $loop_dev name $part_num $volid; + )?; + + root_num = part_num; + + // now format the partition + + let root_format = &layout.root_format; + + cmd_lib::run_cmd!( + mkfs.${root_format} ${loop_dev}p$part_num -L $volid; + )?; + + // Now, mount them all + + info!("Mounting partitions"); + + let instroot = &cfg + .instroot + .to_str() + .unwrap_or_default(); + + cmd_lib::run_cmd!( + mkdir -p $instroot; + mount ${loop_dev}p$root_num $instroot; + mkdir -p $instroot/boot; + mount ${loop_dev}p$boot_num $instroot/boot; + )?; + + if layout.bootloader { + let efi_num = efi_num.unwrap(); + cmd_lib::run_cmd!( + mkdir -p $instroot/boot/efi; + mount ${loop_dev}p$efi_num $instroot/boot/efi; + )?; + } + + Ok(()) + } else { + // error out + return Err(eyre!("No disk layout specified")); + } + } + #[instrument(skip(self))] fn mkmountpt(&self) -> Result<()> { debug!("Checking for mount point"); @@ -320,8 +524,17 @@ pub trait LiveImageCreator { } std::fs::create_dir(instroot)?; } - trace!("Checking for ISO directory"); - std::fs::create_dir_all(format!("./{}/LiveOS", cfg.distro))?; + match cfg.format { + OutputFormat::Iso => { + trace!("Checking for ISO directory"); + std::fs::create_dir_all(format!("./{}/LiveOS", cfg.distro))?; + }, + OutputFormat::Disk => { + std::fs::create_dir_all(format!("{}/boot/efi", instroot.display()))?; + self.prep_disk()?; + }, + } + Ok(()) } @@ -337,17 +550,95 @@ pub trait LiveImageCreator { // if dnf == "dnf5" { // pkgs.push("--use-host-config"); // } + + let mut extra_args = vec![]; + if cfg.arch.is_some() { + extra_args.push("--forcearch"); + extra_args.push(cfg.arch.as_ref().unwrap()); + } cmd_lib::run_cmd!( - $dnf in -y --releasever=$rel --installroot $root $[pkgs]; + $dnf in -y --releasever=$rel $[extra_args] --installroot $root $[pkgs]; $dnf clean all; )?; Ok(()) } } -pub struct LiveImageCreatorX86 { +pub struct KatsuCreator { cfg: Config, } +impl From for KatsuCreator { + fn from(cfg: Config) -> Self { + Self { cfg } + } +} +impl ImageCreator for KatsuCreator { + // const ARCH: crate::util::Arch = crate::util::Arch::X86; + const EFI_FILES: &'static [(&'static str, &'static str, bool)] = &[ + ("/boot/efi/EFI/*/shim%arch%.efi", "/EFI/BOOT/BOOT%arch%.EFI", true), + ("/boot/efi/EFI/*/gcd%arch%.efi", "/EFI/BOOT/grub%arch%.efi", true), + ("/boot/efi/EFI/*/shimia32.efi", "/EFI/BOOT/BOOTIA32.EFI", false), + ("/boot/efi/EFI/*/gcdia32.efi", "/EFI/BOOT/grubia32.efi", false), + ("/usr/share/grub/unicode.pf2", "/EFI/BOOT/fonts/", true), + ]; + + #[inline] + fn get_cfg(&self) -> &Config { + &self.cfg + } +} + +// @madonuko: why? Why did you hardcode everything per architecture? I... My sanity hurts. -@korewaChino +// pub struct LiveImageCreatorX86 { +// cfg: Config, +// } + +// impl From for LiveImageCreatorX86 { +// fn from(cfg: Config) -> Self { +// Self { cfg } +// } +// } + +// impl ImageCreator for LiveImageCreatorX86 { +// // const ARCH: crate::util::Arch = crate::util::Arch::X86; +// const EFI_FILES: &'static [(&'static str, &'static str, bool)] = &[ +// ("/boot/efi/EFI/*/shim%arch%.efi", "/EFI/BOOT/BOOT%arch%.EFI", true), +// ("/boot/efi/EFI/*/gcd%arch%.efi", "/EFI/BOOT/grub%arch%.efi", true), +// ("/boot/efi/EFI/*/shimia32.efi", "/EFI/BOOT/BOOTIA32.EFI", false), +// ("/boot/efi/EFI/*/gcdia32.efi", "/EFI/BOOT/grubia32.efi", false), +// ("/usr/share/grub/unicode.pf2", "/EFI/BOOT/fonts/", true), +// ]; + +// #[inline] +// fn get_cfg(&self) -> &Config { +// &self.cfg +// } +// } +// pub struct LiveImageCreatorX86_64 { +// cfg: Config, +// } + +// impl From for LiveImageCreatorX86_64 { +// fn from(cfg: Config) -> Self { +// Self { cfg } +// } +// } + +// impl ImageCreator for LiveImageCreatorX86_64 { +// // const ARCH: crate::util::Arch = crate::util::Arch::X86_64; +// const EFI_FILES: &'static [(&'static str, &'static str, bool)] = &[ +// ("/boot/efi/EFI/*/shim%arch%.efi", "/EFI/BOOT/BOOT%arch%.EFI", true), +// ("/boot/efi/EFI/*/gcd%arch%.efi", "/EFI/BOOT/grub%arch%.efi", true), +// ("/boot/efi/EFI/*/shimia32.efi", "/EFI/BOOT/BOOTIA32.EFI", false), +// ("/boot/efi/EFI/*/gcdia32.efi", "/EFI/BOOT/grubia32.efi", false), +// ("/usr/share/grub/unicode.pf2", "/EFI/BOOT/fonts/", true), +// ]; + +// fn get_cfg(&self) -> &Config { +// &self.cfg +// } +// } + /// Prepare chroot by mounting /dev, /proc, /sys fn prepare_chroot(root: &str) -> Result<()> { cmd_lib::run_cmd! ( @@ -376,49 +667,3 @@ fn unmount_chroot(root: &str) -> Result<()> { )?; Ok(()) } - -impl From for LiveImageCreatorX86 { - fn from(cfg: Config) -> Self { - Self { cfg } - } -} - -impl LiveImageCreator for LiveImageCreatorX86 { - const ARCH: crate::util::Arch = crate::util::Arch::X86; - const EFI_FILES: &'static [(&'static str, &'static str, bool)] = &[ - ("/boot/efi/EFI/*/shim%arch%.efi", "/EFI/BOOT/BOOT%arch%.EFI", true), - ("/boot/efi/EFI/*/gcd%arch%.efi", "/EFI/BOOT/grub%arch%.efi", true), - ("/boot/efi/EFI/*/shimia32.efi", "/EFI/BOOT/BOOTIA32.EFI", false), - ("/boot/efi/EFI/*/gcdia32.efi", "/EFI/BOOT/grubia32.efi", false), - ("/usr/share/grub/unicode.pf2", "/EFI/BOOT/fonts/", true), - ]; - - #[inline] - fn get_cfg(&self) -> &Config { - &self.cfg - } -} -pub struct LiveImageCreatorX86_64 { - cfg: Config, -} - -impl From for LiveImageCreatorX86_64 { - fn from(cfg: Config) -> Self { - Self { cfg } - } -} - -impl LiveImageCreator for LiveImageCreatorX86_64 { - const ARCH: crate::util::Arch = crate::util::Arch::X86_64; - const EFI_FILES: &'static [(&'static str, &'static str, bool)] = &[ - ("/boot/efi/EFI/*/shim%arch%.efi", "/EFI/BOOT/BOOT%arch%.EFI", true), - ("/boot/efi/EFI/*/gcd%arch%.efi", "/EFI/BOOT/grub%arch%.efi", true), - ("/boot/efi/EFI/*/shimia32.efi", "/EFI/BOOT/BOOTIA32.EFI", false), - ("/boot/efi/EFI/*/gcdia32.efi", "/EFI/BOOT/grubia32.efi", false), - ("/usr/share/grub/unicode.pf2", "/EFI/BOOT/fonts/", true), - ]; - - fn get_cfg(&self) -> &Config { - &self.cfg - } -} diff --git a/src/main.rs b/src/main.rs index d49894c..bd8faf5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,12 +2,11 @@ mod cfg; mod creator; mod util; -use crate::creator::{LiveImageCreator, LiveImageCreatorX86, LiveImageCreatorX86_64}; +use crate::creator::{ImageCreator, KatsuCreator}; use cfg::Config; use color_eyre::Result; use tracing::trace; use tracing_subscriber::{prelude::__tracing_subscriber_SubscriberExt, Layer}; -use util::Arch; fn main() -> Result<()> { dotenv::dotenv()?; @@ -25,11 +24,27 @@ fn main() -> Result<()> { trace!(cfg_file, "Reading/Parsing config"); let config: Config = serde_yaml::from_str(&std::fs::read_to_string(cfg_file)?)?; trace!("Config read done: {config:#?}"); - match Arch::get()? { - Arch::X86 => LiveImageCreatorX86::from(config).exec()?, - Arch::X86_64 => LiveImageCreatorX86_64::from(config).exec()?, - _ => panic!("Unknown architecture"), - } + // let arch = { + // let cfg_arch = config.clone().arch; + + // if cfg_arch.is_none() { + // Arch::get()? + // } else { + // Arch::from(cfg_arch.as_ref().unwrap().as_str()) + // } + // }; + // match arch { + // Arch::X86 => LiveImageCreatorX86::from(config).exec_iso()?, + + // Arch::X86_64 => LiveImageCreatorX86_64::from(config).exec_iso()?, + + // // todo: please clean this up + + // Arch::AArch64 => todo!(), + + // _ => panic!("Unknown architecture"), + // } + KatsuCreator::from(config).exec()?; } Ok(()) } diff --git a/tests/.gitignore b/tests/.gitignore index d48c39b..f495424 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,3 +1,5 @@ instroot/ +instroot-arm/ Ultramarine-Linux out.iso +*.raw \ No newline at end of file diff --git a/tests/katsudon-arm.yaml b/tests/katsudon-arm.yaml new file mode 100644 index 0000000..ec1e521 --- /dev/null +++ b/tests/katsudon-arm.yaml @@ -0,0 +1,45 @@ +dnf: dnf +distro: Ultramarine-ARM +instroot: instroot-arm/ +# this is the name of the iso +out: out +volid: ULTRAMARINE +arch: aarch64 +format: disk + +packages: + # dracut stuff below + - filesystem + - setup + - lvm2 + - btrfs-progs + - dmraid + - nvme-cli + - dbus-daemon + # necessary stuff + - "@core" + - fedora-repos + - kernel + - glibc + - glibc-common + - "@standard" + - ultramarine-release + - dnf + - terra-release + - dracut + - NetworkManager + - mkpasswd + - polkit + +disk: + bootloader: true + root_format: btrfs + disk_size: 8G + +sys: + releasever: 38 + rootpw: ultramarine + +script: + init: init.sh + postinst: postinst.sh \ No newline at end of file