tools/trace-parser: Add Rust trace.dat parser

FEATURE

tools/recipes/trace-tools.recipe

@@ -0,0 +1,35 @@
+#! /bin/bash
+
+ALPINE_VERSION=v3.18
+ALPINE_BUILD_DEPENDENCIES=(bash git curl musl-dev gcc)
+# We technically could cross compile, but would require to auto detect a cross linker, by setting e.g.:
+# export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-ld
+CROSS_COMPILATION_BROKEN=1
+
+run_rustup() {
+    export CARGO_HOME="$(readlink -f .)/.cargo"
+    export RUSTUP_HOME="$(readlink -f .)/.rustup"
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain none -y --no-modify-path
+    source $CARGO_HOME/env
+    rustup toolchain install stable --allow-downgrade --profile minimal
+}
+
+build_tracedump() {
+    cd trace-tools &&
+    time RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target=x86_64-unknown-linux-musl
+}
+
+download() {
+    cp -a "$LISA_HOME/tools/trace-tools" .
+    cp "$LISA_HOME/LICENSE.txt" trace-tools/
+}
+
+build() {
+    run_rustup && (build_tracedump)
+}
+
+install() {
+    source "$LISA_HOME/tools/recipes/utils.sh"
+    cp -v trace-tools/trace-dump/target/release/trace-dump "$LISA_ARCH_ASSETS/"
+    install_readme trace-dump trace-tools/ LICENSE.txt
+}

tools/trace-parser/rustfmt.toml

@@ -0,0 +1,3 @@
+imports_granularity="Crate"
+group_imports="StdExternalCrate"
+skip_macro_invocations=["make_closure_coerce", "make_closure_coerce_type", "closure"]

tools/trace-parser/trace-tools/.gitignore

@@ -0,0 +1,4 @@
+debug/
+target/
+Cargo.lock
+**/*.rs.bk

tools/trace-parser/trace-tools/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "trace-tools"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+name = "lib"
+path = "src/lib/lib.rs"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+traceevent = { path = "../traceevent" }
+
+thiserror = "1.0"
+smartstring = {version = "1.0", features = ["serde"]}
+arrow2 = { version = "0.18.0", features = ["io_parquet", "io_parquet_compression"] }
+crossbeam = "0.8"
+serde_json = "1.0"
+
+nom = "7.1"
+bytemuck = "1.13"
+clap = { version = "4.4", features = ["derive"] }
+
+[target.'cfg(target_arch = "x86_64")'.dependencies]
+mimalloc = {version = "0.1", default-features = false }
+
+[profile.release]
+debug = true
+
+# Static build:
+# rustup target add x86_64-unknown-linux-musl
+# RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target x86_64-unknown-linux-musl

tools/trace-parser/trace-tools/fuzz/.gitignore

@@ -0,0 +1,4 @@
+target
+corpus
+artifacts
+coverage

tools/trace-parser/trace-tools/fuzz/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "trace-tools-fuzz"
+version = "0.0.0"
+publish = false
+edition = "2021"
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+libfuzzer-sys = "0.4"
+
+[target.'cfg(target_arch = "x86_64")'.dependencies]
+mimalloc = {version = "0.1", default-features = false }
+
+[dependencies.trace-tools]
+path = ".."
+
+[dependencies.traceevent]
+path = "../../traceevent/"
+
+# Prevent this from interfering with workspaces
+[workspace]
+members = ["."]
+
+[profile.release]
+debug = 1
+
+[[bin]]
+name = "print"
+path = "fuzz_targets/print.rs"
+test = false
+doc = false

tools/trace-parser/trace-tools/fuzz/fuzz_targets/print.rs

@@ -0,0 +1,27 @@
+#![no_main]
+#[cfg(target_arch = "x86_64")]
+use mimalloc::MiMalloc;
+use traceevent::header;
+#[global_allocator]
+#[cfg(target_arch = "x86_64")]
+static GLOBAL: MiMalloc = MiMalloc;
+
+use std::io::Write;
+
+use lib::{check_header, parquet::dump_events, print::print_events};
+use libfuzzer_sys::fuzz_target;
+use traceevent;
+
+fuzz_target!(|data: &[u8]| {
+    // Speedup the test by not writing anything anywhere
+    let mut out = std::io::sink();
+
+    let mut run = move || {
+        let mut reader: traceevent::io::BorrowingCursor<_> = data.into();
+        let header = header::header(&mut reader)?;
+        let res = print_events(&header, reader, &mut out);
+        res
+    };
+
+    let _ = run();
+});

tools/trace-parser/trace-tools/src/bin/trace-dump.rs

@@ -0,0 +1,120 @@
+use std::{error::Error, fs::File, io::Write, path::PathBuf, process::ExitCode};
+
+#[cfg(target_arch = "x86_64")]
+use mimalloc::MiMalloc;
+use traceevent::{header, header::Timestamp};
+#[global_allocator]
+#[cfg(target_arch = "x86_64")]
+static GLOBAL: MiMalloc = MiMalloc;
+
+use clap::{Parser, Subcommand};
+use lib::{
+    check_header,
+    parquet::{dump_events, dump_header_metadata},
+    print::print_events,
+};
+
+#[derive(Parser)]
+#[command(author, version, about, long_about = None)]
+struct Cli {
+    #[arg(long, value_name = "TRACE")]
+    trace: PathBuf,
+
+    #[arg(long, value_name = "ERRORS_JSON")]
+    errors_json: Option<PathBuf>,
+
+    #[command(subcommand)]
+    command: Command,
+}
+
+#[derive(Subcommand)]
+enum Command {
+    HumanReadable,
+    Parquet {
+        #[arg(long, value_name = "EVENTS")]
+        events: Option<Vec<String>>,
+        #[arg(long)]
+        unique_timestamps: bool,
+    },
+    CheckHeader,
+    Metadata,
+}
+
+fn _main() -> Result<(), Box<dyn Error>> {
+    let cli = Cli::parse();
+
+    let path = cli.trace;
+    let file = std::fs::File::open(path)?;
+    let mut reader = unsafe { traceevent::io::MmapFile::new(file) }?;
+    let header = header::header(&mut reader)?;
+
+    // We make the timestamp unique assuming it will be manipulated as an f64 number of seconds by
+    // consumers
+    let conv_ts = |ts| (ts as f64) / 1e9;
+    let make_unique_timestamps = {
+        let mut prev = (0, conv_ts(0));
+        move |mut ts: Timestamp| {
+            ts = std::cmp::max(prev.0, ts);
+            let mut _ts = conv_ts(ts);
+            while prev.1 == _ts {
+                ts += 1;
+                _ts = conv_ts(ts);
+            }
+            prev = (ts, _ts);
+            ts
+        }
+    };
+
+    let stdout = std::io::stdout().lock();
+    let mut out = std::io::BufWriter::with_capacity(1024 * 1024, stdout);
+
+    let res = match cli.command {
+        Command::HumanReadable => print_events(&header, reader, &mut out),
+        Command::Parquet {
+            events,
+            unique_timestamps,
+        } => {
+            let make_ts: Box<dyn FnMut(_) -> _> = if unique_timestamps {
+                Box::new(make_unique_timestamps)
+            } else {
+                Box::new(|ts| ts)
+            };
+
+            dump_events(&header, reader, make_ts, events)
+        }
+        Command::CheckHeader => check_header(&header, &mut out),
+        Command::Metadata => dump_header_metadata(&header, &mut out),
+    };
+    out.flush()?;
+
+    if let Err(err) = &res {
+        eprintln!("Errors happened while processing the trace:{err}");
+    }
+
+    if let Some(path) = &cli.errors_json {
+        let errors = match &res {
+            Err(err) => err
+                .errors()
+                .into_iter()
+                .map(|err| err.to_string())
+                .collect(),
+            Ok(_) => Vec::new(),
+        };
+        let mut file = File::create(&path)?;
+        let json_value = serde_json::json!({
+            "errors": errors,
+        });
+        file.write_all(json_value.to_string().as_bytes())?;
+    }
+    match res {
+        Ok(_) => Ok(()),
+        Err(_) => Err("Errors happened".into()),
+    }
+}
+
+fn main() -> ExitCode {
+    match _main() {
+        Err(_) => ExitCode::from(1),
+        Ok(_) => ExitCode::from(0),
+    }
+}

tools/trace-parser/trace-tools/src/lib/error.rs

@@ -0,0 +1,57 @@
+use std::{error::Error, fmt};
+
+#[derive(Debug)]
+pub struct MultiError<E> {
+    errors: Vec<E>,
+}
+
+impl<E> MultiError<E> {
+    fn new<I: IntoIterator<Item = E>>(errors: I) -> Self {
+        MultiError {
+            errors: errors.into_iter().collect(),
+        }
+    }
+
+    pub fn errors(&self) -> impl IntoIterator<Item = &E> {
+        &self.errors
+    }
+}
+
+impl<E: fmt::Display> fmt::Display for MultiError<E> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
+        for err in &self.errors {
+            err.fmt(f)?;
+            write!(f, "\n")?;
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+pub struct DynMultiError(MultiError<Box<dyn Error>>);
+
+impl DynMultiError {
+    pub fn new<E: Error + 'static, I: IntoIterator<Item = E>>(errors: I) -> Self {
+        DynMultiError(MultiError::new(
+            errors
+                .into_iter()
+                .map(|err| Box::new(err) as Box<dyn Error>),
+        ))
+    }
+    pub fn errors(&self) -> impl IntoIterator<Item = &dyn Error> {
+        self.0.errors().into_iter().map(AsRef::as_ref)
+    }
+}
+
+impl fmt::Display for DynMultiError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
+        self.0.fmt(f)
+    }
+}
+
+impl<E: Error + 'static> From<E> for DynMultiError {
+    #[inline]
+    fn from(error: E) -> Self {
+        error.into()
+    }
+}

tools/trace-parser/trace-tools/src/lib/lib.rs

@@ -0,0 +1,42 @@
+pub mod error;
+pub mod parquet;
+pub mod print;
+
+use std::io::Write;
+
+use traceevent::header::Header;
+
+use crate::error::DynMultiError;
+
+pub fn check_header<W: Write>(header: &Header, mut out: W) -> Result<(), DynMultiError> {
+    for desc in header.event_descs() {
+        writeln!(&mut out, "Checking event \"{}\" format", desc.name)?;
+
+        let raw_fmt = std::str::from_utf8(desc.raw_fmt()?)?;
+        match desc.event_fmt() {
+            Err(err) => {
+                writeln!(
+                    &mut out,
+                    "    Error while parsing event format: {err}:\n{raw_fmt}"
+                )
+            }
+            Ok(fmt) => {
+                match fmt.print_args() {
+                    Ok(print_args) => {
+                        print_args.iter().enumerate().try_for_each(|(i, res)| match res {
+                            Err(err) => {
+                                writeln!(&mut out, "    Error while compiling printk argument #{i}: {err}:\n{raw_fmt}")
+                            }
+                            Ok(_) => Ok(()),
+                        })?;
+                        Ok(())
+                    }
+                    Err(err) => {
+                        writeln!(&mut out, "    Error while parsing event print format arguments: {err}:\n{raw_fmt}")
+                    }
+                }
+            }
+        }?;
+    }
+    Ok(())
+}