Merge branch 'main' into ensure_windsock_path_exists

readme.md

@@ -19,14 +19,13 @@ Windsock is suitable for:
 
 * Iteratively testing performance during development of a database or service (for microbenchmarks you will need a different tool though)
 * Investigating performance of different workloads on a database you intend to use.
-* Generating a webpage of graphs to show off the performance of your released database. (not yet implemented)
 
 ## Define benches
 
 To use windsock create a rust crate that imports windsock:
 
 ```toml
-windsock = { git = "https://github.com/shotover/windsock" }
+windsock = "0.1"
 ```
 
 And then implement the crate like this (simplified):
@@ -136,8 +135,6 @@ Measurements ══════════════════════
 
 TODO: make this into a comparison to make it more flashy and use an image to include the coloring
 
-and graphs: TODO
-
 ## How to perform various tasks in windsock
 
 ### Just run every bench
@@ -197,7 +194,7 @@ and graphs: TODO
 
 ### Generate graph webpage
 
-TODO: not yet implemented
+TODO: planned, but not implemented
 
 ```shell
 > cargo windsock local-run # run all benches

windsock/src/bench.rs

@@ -112,6 +112,7 @@ impl<ResourcesRequired, Resources> BenchState<ResourcesRequired, Resources> {
 }
 
 /// Implement this to define your benchmarks
+///
 /// A single implementation of `Bench` can represent multiple benchmarks by initializing it multiple times with different state that returns unique tags.
 #[async_trait]
 pub trait Bench {
@@ -213,6 +214,7 @@ fn run_args_vec(name_and_resources: String, bench_parameters: &BenchParameters)
     args
 }
 
+/// Instructs the benches on how the bench should be run according to the users request.
 pub struct BenchParameters {
     pub runtime_seconds: u32,
     pub operations_per_second: Option<u64>,
@@ -227,6 +229,7 @@ impl BenchParameters {
     }
 }
 
+/// Instructs the benches to activate different profilers on the users request.
 pub struct Profiling {
     pub results_path: PathBuf,
     pub profilers_to_use: Vec<String>,
@@ -283,6 +286,7 @@ impl Tags {
 }
 
 /// An optional helper trait for defining benchmarks.
+///
 /// Usually you have an async rust DB driver that you need to call across multiple tokio tasks
 /// This helper will spawn these tasks and send the required `Report::QueryCompletedIn`.
 ///

windsock/src/cloud.rs

@@ -1,8 +1,11 @@
+//! create, manage and destroy cloud resources
+
 use std::path::Path;
 
 use async_trait::async_trait;
 
 /// Implement this to give windsock some control over your cloud.
+///
 /// Currently the only thing windsock needs is the ability to cleanup resources since resource creation should happen within your own benches.
 #[async_trait(?Send)]
 pub trait Cloud {
@@ -70,9 +73,10 @@ pub trait Cloud {
     }
 }
 
-pub struct BenchInfo<CloudResourceRequest> {
+/// Defines the cloud resources required by a specific bench
+pub struct BenchInfo<CloudResourceRequired> {
     pub name: String,
-    pub resources: CloudResourceRequest,
+    pub resources: CloudResourceRequired,
 }
 
 /// A dummy cloud instance for when the user isnt using windsock cloud functionality

windsock/src/lib.rs

@@ -24,6 +24,7 @@ use filter::Filter;
 use std::process::exit;
 use tokio::runtime::Runtime;
 
+/// Takes control of your application, providing a CLI into your benchmarks.
 pub struct Windsock<ResourcesRequired, Resources> {
     benches: Vec<BenchState<ResourcesRequired, Resources>>,
     cloud: Box<dyn Cloud<CloudResourcesRequired = ResourcesRequired, CloudResources = Resources>>,

windsock/src/report.rs

@@ -6,6 +6,9 @@ use strum::{EnumCount, EnumIter, IntoEnumIterator};
 use time::OffsetDateTime;
 use tokio::sync::mpsc::UnboundedReceiver;
 
+/// An individual measurement reported to windsock.
+///
+/// These will be collected, analyzed and then turned into a ReportArchive at the conclusion of the bench run.
 #[derive(Debug, Serialize, Deserialize)]
 pub enum Report {
     /// Indicates the warmup is over and the benchmark has begun.
@@ -54,6 +57,7 @@ pub enum Report {
     ExternalBenchmark(Box<ExternalReport>),
 }
 
+/// Defines bench results that occured entirely outside of windsock.
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct ExternalReport {
     pub bench_started_at: OffsetDateTime,
@@ -125,6 +129,7 @@ impl Percentile {
 
 pub type Percentiles = [Duration; Percentile::COUNT];
 
+/// The entire results of a single benchmark run.
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct ReportArchive {
     pub(crate) running_in_release: bool,
@@ -137,6 +142,7 @@ pub struct ReportArchive {
     pub info_messages: Vec<String>,
 }
 
+/// The operation results of a benchmark run.
 #[derive(Clone, Debug, Serialize, Deserialize, Default)]
 pub struct OperationsReport {
     pub total: u64,
@@ -149,6 +155,7 @@ pub struct OperationsReport {
     pub total_each_second: Vec<u64>,
 }
 
+/// The pubsub results of a benchmark run.
 #[derive(Clone, Debug, Serialize, Deserialize, Default)]
 pub struct PubSubReport {
     pub total_produce: u64,
@@ -168,6 +175,7 @@ pub struct PubSubReport {
     pub backlog_each_second: Vec<i64>,
 }
 
+/// Extra metrics that can be inserted into bench results.
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub enum Metric {
     Total {
@@ -186,6 +194,7 @@ pub enum Metric {
     },
 }
 
+/// Latency metrics that can be included in [`Metric::LatencyPercentiles`]
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct LatencyPercentile {
     pub quantile: String,

windsock/src/tables.rs

@@ -844,6 +844,9 @@ struct Measurement {
     color: Color,
 }
 
+/// Defines wether a change in results is good or bad.
+///
+/// When comparing bench results in a table this is used to highlight good and bad results.
 #[derive(Clone, Copy, Debug, Serialize, Deserialize)]
 pub enum Goal {
     BiggerIsBetter,