From 68560d8e49505911555257d33c88f23d10f414d1 Mon Sep 17 00:00:00 2001 From: Henry Denny Date: Mon, 4 Dec 2023 15:58:00 +0000 Subject: [PATCH] Inherit Dependencies (#91) * Inherit dependencies from main `Cargo.toml` file. * Update file paths. * Update Cargo.toml Co-authored-by: Dan Nixon * Update Cargo.toml Co-authored-by: Dan Nixon * Update Cargo.toml Co-authored-by: Dan Nixon * Undo autoformat. * Inherit all singleton dependencies. * Prefix `supermusr-` and sort alphabetically. * Update package names. * Update all references to `common`. * Fix hyphens. * Update names. * Sort imports alphabetically. * Revert alphabetical order for anti-example. --------- Co-authored-by: Dan Nixon --- Cargo.lock | 66 +++++++++---------- Cargo.toml | 16 ++++- common/Cargo.toml | 2 +- docs/style.md | 4 +- events-to-histogram/Cargo.toml | 5 +- events-to-histogram/src/main.rs | 11 ++-- events-to-histogram/src/metrics.rs | 8 +-- events-to-histogram/src/processing.rs | 6 +- kafka-daq-report/Cargo.toml | 9 ++- kafka-daq-report/src/main.rs | 19 +++--- simulator/Cargo.toml | 7 +- simulator/src/main.rs | 11 ++-- stream-to-file/Cargo.toml | 5 +- stream-to-file/src/file/base.rs | 2 +- stream-to-file/src/file/event.rs | 4 +- stream-to-file/src/file/trace/mod.rs | 4 +- stream-to-file/src/file/trace/tests/basic.rs | 2 +- stream-to-file/src/file/trace/tests/mod.rs | 2 +- .../file/trace/tests/multiple_digitizers.rs | 2 +- .../tests/multiple_digitizers_missing_data.rs | 2 +- stream-to-file/src/main.rs | 19 +++--- stream-to-file/src/metrics.rs | 8 +-- streaming-types/Cargo.toml | 4 +- trace-archiver-tdengine/Cargo.toml | 12 ++-- trace-archiver-tdengine/src/main.rs | 4 +- .../src/tdengine/error_reporter.rs | 2 +- .../src/tdengine/framedata.rs | 4 +- trace-archiver-tdengine/src/tdengine/mod.rs | 2 +- trace-archiver-tdengine/src/tdengine/views.rs | 4 +- .../src/tdengine/wrapper.rs | 2 +- trace-archiver/Cargo.toml | 5 +- trace-archiver/src/file.rs | 8 +-- trace-archiver/src/main.rs | 19 +++--- trace-archiver/src/metrics.rs | 8 +-- trace-reader/Cargo.toml | 13 ++-- trace-reader/src/main.rs | 9 ++- trace-reader/src/processing.rs | 4 +- trace-to-events/Cargo.toml | 11 ++-- trace-to-events/src/main.rs | 9 ++- trace-to-events/src/metrics.rs | 8 +-- trace-to-events/src/processing.rs | 6 +- .../src/pulse_detection/datatype/mod.rs | 2 +- .../window/finite_differences.rs | 2 +- 43 files changed, 189 insertions(+), 163 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 94d3da30..4bf00dc0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -359,14 +359,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" -[[package]] -name = "common" -version = "0.1.0" -dependencies = [ - "kagiyama", - "rdkafka", -] - [[package]] name = "convert_case" version = "0.4.0" @@ -738,7 +730,6 @@ dependencies = [ "anyhow", "chrono", "clap", - "common", "env_logger", "kagiyama", "lazy_static", @@ -746,7 +737,8 @@ dependencies = [ "ndarray", "ndarray-stats", "rdkafka", - "streaming-types", + "supermusr-common", + "supermusr-streaming-types", "tokio", ] @@ -1186,13 +1178,13 @@ dependencies = [ "anyhow", "chrono", "clap", - "common", "crossterm", "env_logger", "log", "ratatui", "rdkafka", - "streaming-types", + "supermusr-common", + "supermusr-streaming-types", "tokio", ] @@ -2248,11 +2240,11 @@ version = "0.1.0" dependencies = [ "chrono", "clap", - "common", "env_logger", "log", "rdkafka", - "streaming-types", + "supermusr-common", + "supermusr-streaming-types", "tokio", ] @@ -2298,7 +2290,6 @@ dependencies = [ "anyhow", "chrono", "clap", - "common", "env_logger", "hdf5", "kagiyama", @@ -2307,19 +2298,11 @@ dependencies = [ "ndarray", "ndarray-stats", "rdkafka", - "streaming-types", + "supermusr-common", + "supermusr-streaming-types", "tokio", ] -[[package]] -name = "streaming-types" -version = "0.1.0" -dependencies = [ - "chrono", - "flatbuffers", - "flatc-rust", -] - [[package]] name = "strsim" version = "0.10.0" @@ -2345,6 +2328,23 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "supermusr-common" +version = "0.1.0" +dependencies = [ + "kagiyama", + "rdkafka", +] + +[[package]] +name = "supermusr-streaming-types" +version = "0.1.0" +dependencies = [ + "chrono", + "flatbuffers", + "flatc-rust", +] + [[package]] name = "syn" version = "1.0.109" @@ -2662,7 +2662,6 @@ dependencies = [ "anyhow", "chrono", "clap", - "common", "env_logger", "hdf5", "kagiyama", @@ -2670,7 +2669,8 @@ dependencies = [ "log", "ndarray", "rdkafka", - "streaming-types", + "supermusr-common", + "supermusr-streaming-types", "tokio", ] @@ -2682,12 +2682,12 @@ dependencies = [ "async-trait", "chrono", "clap", - "common", "env_logger", "itertools 0.11.0", "log", "rdkafka", - "streaming-types", + "supermusr-common", + "supermusr-streaming-types", "taos", "tokio", ] @@ -2699,12 +2699,12 @@ dependencies = [ "anyhow", "chrono", "clap", - "common", "env_logger", "log", "rand", "rdkafka", - "streaming-types", + "supermusr-common", + "supermusr-streaming-types", "tokio", ] @@ -2716,7 +2716,6 @@ dependencies = [ "assert_approx_eq", "chrono", "clap", - "common", "env_logger", "kagiyama", "lazy_static", @@ -2724,7 +2723,8 @@ dependencies = [ "num 0.4.1", "rand", "rdkafka", - "streaming-types", + "supermusr-common", + "supermusr-streaming-types", "tokio", ] diff --git a/Cargo.toml b/Cargo.toml index c172cbe9..a185213e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,15 +20,27 @@ edition = "2021" [workspace.dependencies] anyhow = "1.0" -chrono = "0.4.31" +assert_approx_eq = "1.1.0" +async-trait = "0.1.68" +chrono = "0.4.22" clap = { version = "4.4", features = ["derive", "env"] } +crossterm = "0.26.1" env_logger = "0.10" +flatbuffers = "22.12.6" hdf5 = "0.8.1" +itertools = "0.11.0" kagiyama = "0.3.0" lazy_static = "1.4.0" log = "0.4" ndarray = "0.15.6" ndarray-stats = "0.5.1" -rdkafka = { version = "0.31.0", features = ["cmake-build", "ssl", "gssapi", "sasl"] } +num = "0.4.1" +rand = "0.8.5" +ratatui = "0.22.0" +rayon = "1.8.0" +rdkafka = { version = "0.31.0", features = [ "cmake-build", "ssl", "gssapi", "sasl", ] } serde = { version = "1", features = ["derive"] } +supermusr-common = { path = "./common" } +supermusr-streaming-types = { path = "./streaming-types" } +taos = { version = "0.10.16", default_features = false, features = ["ws"] } tokio = { version = "1.33", features = ["macros", "rt-multi-thread"] } diff --git a/common/Cargo.toml b/common/Cargo.toml index ed29c06a..0cb234fc 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "common" +name = "supermusr-common" version.workspace = true license.workspace = true edition.workspace = true diff --git a/docs/style.md b/docs/style.md index 7a16dd9d..89e1c220 100644 --- a/docs/style.md +++ b/docs/style.md @@ -16,8 +16,8 @@ In cases where the formatter does not care, the following rules apply: ```rust use crate::Something; use super::SomethingElse -use common::Time use std::time::Duration +use supermusr_common::Time use tokio::task::JoinHandle; ``` @@ -27,7 +27,7 @@ instead of use crate::Something; use tokio::task::JoinHandle; -use common::Time +use supermusr_common::Time use std::time::Duration use super::SomethingElse diff --git a/events-to-histogram/Cargo.toml b/events-to-histogram/Cargo.toml index d6c3c586..8ec7360b 100644 --- a/events-to-histogram/Cargo.toml +++ b/events-to-histogram/Cargo.toml @@ -5,9 +5,6 @@ license.workspace = true edition.workspace = true [dependencies] -common = { path = "../common" } -streaming-types = { path = "../streaming-types" } - anyhow.workspace = true clap.workspace = true env_logger.workspace = true @@ -17,6 +14,8 @@ log.workspace = true ndarray.workspace = true ndarray-stats.workspace = true rdkafka.workspace = true +supermusr-common.workspace = true +supermusr-streaming-types.workspace = true tokio.workspace = true [dev-dependencies] diff --git a/events-to-histogram/src/main.rs b/events-to-histogram/src/main.rs index 859742ed..52670cc0 100644 --- a/events-to-histogram/src/main.rs +++ b/events-to-histogram/src/main.rs @@ -3,7 +3,6 @@ mod processing; use anyhow::Result; use clap::Parser; -use common::Time; use kagiyama::{AlwaysReady, Watcher}; use rdkafka::{ consumer::{stream_consumer::StreamConsumer, CommitMode, Consumer}, @@ -11,7 +10,8 @@ use rdkafka::{ producer::{FutureProducer, FutureRecord}, }; use std::{net::SocketAddr, time::Duration}; -use streaming_types::dev1_digitizer_event_v1_generated::{ +use supermusr_common::Time; +use supermusr_streaming_types::dev1_digitizer_event_v1_generated::{ digitizer_event_list_message_buffer_has_identifier, root_as_digitizer_event_list_message, }; @@ -59,8 +59,11 @@ async fn main() -> Result<()> { metrics::register(&watcher); watcher.start_server(args.observability_address).await; - let mut client_config = - common::generate_kafka_client_config(&args.broker, &args.username, &args.password); + let mut client_config = supermusr_common::generate_kafka_client_config( + &args.broker, + &args.username, + &args.password, + ); let consumer: StreamConsumer = client_config .set("group.id", &args.consumer_group) diff --git a/events-to-histogram/src/metrics.rs b/events-to-histogram/src/metrics.rs index 77889a6e..b2820138 100644 --- a/events-to-histogram/src/metrics.rs +++ b/events-to-histogram/src/metrics.rs @@ -1,12 +1,12 @@ -pub(crate) use common::metrics::{ - failures::{FailureKind, FailureLabels}, - messages_received::{MessageKind, MessagesReceivedLabels}, -}; use kagiyama::{ prometheus::metrics::{counter::Counter, family::Family}, AlwaysReady, Watcher, }; use lazy_static::lazy_static; +pub(crate) use supermusr_common::metrics::{ + failures::{FailureKind, FailureLabels}, + messages_received::{MessageKind, MessagesReceivedLabels}, +}; pub(crate) fn register(watcher: &Watcher) { let mut registry = watcher.metrics_registry(); diff --git a/events-to-histogram/src/processing.rs b/events-to-histogram/src/processing.rs index 32b342e8..db1699b5 100644 --- a/events-to-histogram/src/processing.rs +++ b/events-to-histogram/src/processing.rs @@ -1,8 +1,8 @@ -use common::{Channel, Time}; use ndarray::array; use ndarray_stats::histogram::{self, Bins, Edges, Grid}; use std::collections::HashMap; -use streaming_types::{ +use supermusr_common::{Channel, Time}; +use supermusr_streaming_types::{ dev1_digitizer_event_v1_generated::DigitizerEventListMessage, flatbuffers::FlatBufferBuilder, frame_metadata_v1_generated::{FrameMetadataV1, FrameMetadataV1Args}, @@ -107,7 +107,7 @@ pub(crate) fn process( mod tests { use super::*; use chrono::Utc; - use streaming_types::{ + use supermusr_streaming_types::{ dev1_digitizer_event_v1_generated::{ finish_digitizer_event_list_message_buffer, root_as_digitizer_event_list_message, DigitizerEventListMessage, DigitizerEventListMessageArgs, diff --git a/kafka-daq-report/Cargo.toml b/kafka-daq-report/Cargo.toml index b524a32e..b8aa9e10 100644 --- a/kafka-daq-report/Cargo.toml +++ b/kafka-daq-report/Cargo.toml @@ -5,15 +5,14 @@ license.workspace = true edition.workspace = true [dependencies] -common = { path = "../common" } -streaming-types = { path = "../streaming-types" } - anyhow.workspace = true chrono.workspace = true clap.workspace = true -crossterm = "0.26.1" +crossterm.workspace = true env_logger.workspace = true log.workspace = true -ratatui = "0.22.0" +ratatui.workspace = true rdkafka.workspace = true +supermusr-common.workspace = true +supermusr-streaming-types.workspace = true tokio.workspace = true diff --git a/kafka-daq-report/src/main.rs b/kafka-daq-report/src/main.rs index 180fa9dc..60597f4e 100644 --- a/kafka-daq-report/src/main.rs +++ b/kafka-daq-report/src/main.rs @@ -22,7 +22,7 @@ use std::{ thread, time::{Duration, Instant}, }; -use streaming_types::dat1_digitizer_analog_trace_v1_generated::{ +use supermusr_streaming_types::dat1_digitizer_analog_trace_v1_generated::{ digitizer_analog_trace_message_buffer_has_identifier, root_as_digitizer_analog_trace_message, }; use tokio::task; @@ -97,13 +97,16 @@ async fn main() -> Result<()> { let args = Cli::parse(); log::debug!("Args: {:?}", args); - let consumer: StreamConsumer = - common::generate_kafka_client_config(&args.broker, &args.username, &args.password) - .set("group.id", &args.consumer_group) - .set("enable.partition.eof", "false") - .set("session.timeout.ms", "6000") - .set("enable.auto.commit", "false") - .create()?; + let consumer: StreamConsumer = supermusr_common::generate_kafka_client_config( + &args.broker, + &args.username, + &args.password, + ) + .set("group.id", &args.consumer_group) + .set("enable.partition.eof", "false") + .set("session.timeout.ms", "6000") + .set("enable.auto.commit", "false") + .create()?; consumer.subscribe(&[&args.trace_topic])?; diff --git a/simulator/Cargo.toml b/simulator/Cargo.toml index b3ce8422..151a30a5 100644 --- a/simulator/Cargo.toml +++ b/simulator/Cargo.toml @@ -5,12 +5,11 @@ license.workspace = true edition.workspace = true [dependencies] -common = { path = "../common" } -streaming-types = { path = "../streaming-types" } - -clap.workspace = true chrono.workspace = true +clap.workspace = true env_logger.workspace = true log.workspace = true rdkafka.workspace = true +supermusr-common.workspace = true +supermusr-streaming-types.workspace = true tokio.workspace = true diff --git a/simulator/src/main.rs b/simulator/src/main.rs index e8759ec7..8c9d74f5 100644 --- a/simulator/src/main.rs +++ b/simulator/src/main.rs @@ -1,12 +1,12 @@ use chrono::Utc; use clap::{Parser, Subcommand}; -use common::{Channel, Intensity, Time}; use rdkafka::{ producer::{FutureProducer, FutureRecord}, util::Timeout, }; use std::time::{Duration, SystemTime}; -use streaming_types::{ +use supermusr_common::{Channel, Intensity, Time}; +use supermusr_streaming_types::{ dat1_digitizer_analog_trace_v1_generated::{ finish_digitizer_analog_trace_message_buffer, ChannelTrace, ChannelTraceArgs, DigitizerAnalogTraceMessage, DigitizerAnalogTraceMessageArgs, @@ -92,8 +92,11 @@ async fn main() { let cli = Cli::parse(); - let client_config = - common::generate_kafka_client_config(&cli.broker_address, &cli.username, &cli.password); + let client_config = supermusr_common::generate_kafka_client_config( + &cli.broker_address, + &cli.username, + &cli.password, + ); let producer = client_config.create().unwrap(); let mut fbb = FlatBufferBuilder::new(); diff --git a/stream-to-file/Cargo.toml b/stream-to-file/Cargo.toml index ab8ae8a9..601e69ff 100644 --- a/stream-to-file/Cargo.toml +++ b/stream-to-file/Cargo.toml @@ -5,9 +5,6 @@ license.workspace = true edition.workspace = true [dependencies] -common = { path = "../common" } -streaming-types = { path = "../streaming-types" } - anyhow.workspace = true chrono.workspace = true clap.workspace = true @@ -19,4 +16,6 @@ log.workspace = true ndarray.workspace = true ndarray-stats.workspace = true rdkafka.workspace = true +supermusr-common.workspace = true +supermusr-streaming-types.workspace = true tokio.workspace = true diff --git a/stream-to-file/src/file/base.rs b/stream-to-file/src/file/base.rs index 0d7c4214..2a27d9e3 100644 --- a/stream-to-file/src/file/base.rs +++ b/stream-to-file/src/file/base.rs @@ -1,9 +1,9 @@ use anyhow::Result; use chrono::{DateTime, Utc}; -use common::FrameNumber; use hdf5::{Dataset, File}; use ndarray::{s, Array}; use std::path::Path; +use supermusr_common::FrameNumber; pub(super) struct BaseFile { pub(super) file: File, diff --git a/stream-to-file/src/file/event.rs b/stream-to-file/src/file/event.rs index 570b80c6..1c0cb375 100644 --- a/stream-to-file/src/file/event.rs +++ b/stream-to-file/src/file/event.rs @@ -3,7 +3,7 @@ use anyhow::{anyhow, Result}; use hdf5::Dataset; use ndarray::{s, Array}; use std::path::Path; -use streaming_types::aev1_frame_assembled_event_v1_generated::FrameAssembledEventListMessage; +use supermusr_streaming_types::aev1_frame_assembled_event_v1_generated::FrameAssembledEventListMessage; pub(crate) struct EventFile { base: BaseFile, @@ -98,7 +98,7 @@ impl EventFile { mod tests { use super::*; use std::{env, fs, path::PathBuf}; - use streaming_types::{ + use supermusr_streaming_types::{ aev1_frame_assembled_event_v1_generated::{ finish_frame_assembled_event_list_message_buffer, root_as_frame_assembled_event_list_message, FrameAssembledEventListMessage, diff --git a/stream-to-file/src/file/trace/mod.rs b/stream-to-file/src/file/trace/mod.rs index f1bbc270..21a0377d 100644 --- a/stream-to-file/src/file/trace/mod.rs +++ b/stream-to-file/src/file/trace/mod.rs @@ -1,11 +1,11 @@ use super::base::BaseFile; use anyhow::{anyhow, Result}; -use common::{channel_index, Intensity, SampleRate, CHANNELS_PER_DIGITIZER}; use hdf5::Dataset; use ndarray::{s, Array, Array0, Array1}; use ndarray_stats::QuantileExt; use std::path::Path; -use streaming_types::dat1_digitizer_analog_trace_v1_generated::DigitizerAnalogTraceMessage; +use supermusr_common::{channel_index, Intensity, SampleRate, CHANNELS_PER_DIGITIZER}; +use supermusr_streaming_types::dat1_digitizer_analog_trace_v1_generated::DigitizerAnalogTraceMessage; pub(crate) struct TraceFile { base: BaseFile, diff --git a/stream-to-file/src/file/trace/tests/basic.rs b/stream-to-file/src/file/trace/tests/basic.rs index 9e7d0c5c..409daca7 100644 --- a/stream-to-file/src/file/trace/tests/basic.rs +++ b/stream-to-file/src/file/trace/tests/basic.rs @@ -1,7 +1,7 @@ use super::*; use ndarray::arr2; use std::fs; -use streaming_types::frame_metadata_v1_generated::GpsTime; +use supermusr_streaming_types::frame_metadata_v1_generated::GpsTime; #[test] fn test_basic() { diff --git a/stream-to-file/src/file/trace/tests/mod.rs b/stream-to-file/src/file/trace/tests/mod.rs index e6f2984e..12642174 100644 --- a/stream-to-file/src/file/trace/tests/mod.rs +++ b/stream-to-file/src/file/trace/tests/mod.rs @@ -1,6 +1,6 @@ use super::*; use std::{env, path::PathBuf}; -use streaming_types::{ +use supermusr_streaming_types::{ dat1_digitizer_analog_trace_v1_generated::{ finish_digitizer_analog_trace_message_buffer, root_as_digitizer_analog_trace_message, ChannelTrace, ChannelTraceArgs, DigitizerAnalogTraceMessage, diff --git a/stream-to-file/src/file/trace/tests/multiple_digitizers.rs b/stream-to-file/src/file/trace/tests/multiple_digitizers.rs index 6aaefb2b..c5eba6c2 100644 --- a/stream-to-file/src/file/trace/tests/multiple_digitizers.rs +++ b/stream-to-file/src/file/trace/tests/multiple_digitizers.rs @@ -1,7 +1,7 @@ use super::*; use ndarray::{arr1, arr2}; use std::fs; -use streaming_types::frame_metadata_v1_generated::GpsTime; +use supermusr_streaming_types::frame_metadata_v1_generated::GpsTime; #[test] fn test_multiple_digitizers() { diff --git a/stream-to-file/src/file/trace/tests/multiple_digitizers_missing_data.rs b/stream-to-file/src/file/trace/tests/multiple_digitizers_missing_data.rs index 96f22c09..253684a0 100644 --- a/stream-to-file/src/file/trace/tests/multiple_digitizers_missing_data.rs +++ b/stream-to-file/src/file/trace/tests/multiple_digitizers_missing_data.rs @@ -1,7 +1,7 @@ use super::*; use ndarray::{arr1, arr2}; use std::fs; -use streaming_types::frame_metadata_v1_generated::GpsTime; +use supermusr_streaming_types::frame_metadata_v1_generated::GpsTime; #[test] fn test_multiple_digitizers_missing_data() { diff --git a/stream-to-file/src/main.rs b/stream-to-file/src/main.rs index 082bc37f..fba913c3 100644 --- a/stream-to-file/src/main.rs +++ b/stream-to-file/src/main.rs @@ -10,7 +10,7 @@ use rdkafka::{ message::Message, }; use std::{net::SocketAddr, path::PathBuf}; -use streaming_types::{ +use supermusr_streaming_types::{ aev1_frame_assembled_event_v1_generated::{ frame_assembled_event_list_message_buffer_has_identifier, root_as_frame_assembled_event_list_message, @@ -87,13 +87,16 @@ async fn main() -> Result<()> { } watcher.start_server(args.observability_address).await; - let consumer: StreamConsumer = - common::generate_kafka_client_config(&args.broker, &args.username, &args.password) - .set("group.id", &args.consumer_group) - .set("enable.partition.eof", "false") - .set("session.timeout.ms", "6000") - .set("enable.auto.commit", "false") - .create()?; + let consumer: StreamConsumer = supermusr_common::generate_kafka_client_config( + &args.broker, + &args.username, + &args.password, + ) + .set("group.id", &args.consumer_group) + .set("enable.partition.eof", "false") + .set("session.timeout.ms", "6000") + .set("enable.auto.commit", "false") + .create()?; let topics_to_subscribe: Vec = vec![args.event_topic, args.trace_topic] .into_iter() diff --git a/stream-to-file/src/metrics.rs b/stream-to-file/src/metrics.rs index d5fb0806..7daf1a23 100644 --- a/stream-to-file/src/metrics.rs +++ b/stream-to-file/src/metrics.rs @@ -1,12 +1,12 @@ -pub(crate) use common::metrics::{ - failures::{FailureKind, FailureLabels}, - messages_received::{MessageKind, MessagesReceivedLabels}, -}; use kagiyama::{ prometheus::metrics::{counter::Counter, family::Family}, AlwaysReady, Watcher, }; use lazy_static::lazy_static; +pub(crate) use supermusr_common::metrics::{ + failures::{FailureKind, FailureLabels}, + messages_received::{MessageKind, MessagesReceivedLabels}, +}; pub(crate) fn register(watcher: &mut Watcher) { let mut registry = watcher.metrics_registry(); diff --git a/streaming-types/Cargo.toml b/streaming-types/Cargo.toml index 777810b7..ae63c41c 100644 --- a/streaming-types/Cargo.toml +++ b/streaming-types/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "streaming-types" +name = "supermusr-streaming-types" version.workspace = true license.workspace = true edition.workspace = true @@ -9,4 +9,4 @@ flatc-rust = "*" [dependencies] chrono.workspace = true -flatbuffers = "22.12.6" +flatbuffers.workspace = true diff --git a/trace-archiver-tdengine/Cargo.toml b/trace-archiver-tdengine/Cargo.toml index af3f25c8..0cf0a7a8 100644 --- a/trace-archiver-tdengine/Cargo.toml +++ b/trace-archiver-tdengine/Cargo.toml @@ -5,15 +5,15 @@ license.workspace = true edition.workspace = true [dependencies] -common = { path = "../common" } -streaming-types = { path = "../streaming-types" } anyhow.workspace = true +async-trait.workspace = true chrono.workspace = true clap.workspace = true env_logger.workspace = true +itertools.workspace = true log.workspace = true -tokio.workspace = true rdkafka.workspace = true -async-trait = "0.1.68" -itertools = "0.11.0" -taos = { version = "0.10.16", default_features = false, features = ["ws"] } \ No newline at end of file +supermusr-common.workspace = true +supermusr-streaming-types.workspace = true +taos.workspace = true +tokio.workspace = true diff --git a/trace-archiver-tdengine/src/main.rs b/trace-archiver-tdengine/src/main.rs index 0a87c7f4..dbf20a9b 100644 --- a/trace-archiver-tdengine/src/main.rs +++ b/trace-archiver-tdengine/src/main.rs @@ -9,7 +9,7 @@ use rdkafka::{ consumer::{stream_consumer::StreamConsumer, CommitMode, Consumer}, message::Message, }; -use streaming_types::dat1_digitizer_analog_trace_v1_generated::{ +use supermusr_streaming_types::dat1_digitizer_analog_trace_v1_generated::{ digitizer_analog_trace_message_buffer_has_identifier, root_as_digitizer_analog_trace_message, }; use tdengine::{wrapper::TDEngine, TimeSeriesEngine}; @@ -86,7 +86,7 @@ async fn main() { // All other modes require a kafka builder, a topic, and redpanda consumer debug!("Creating Kafka instance"); - let mut client_config = common::generate_kafka_client_config( + let mut client_config = supermusr_common::generate_kafka_client_config( &cli.kafka_broker, &cli.kafka_username, &cli.kafka_password, diff --git a/trace-archiver-tdengine/src/tdengine/error_reporter.rs b/trace-archiver-tdengine/src/tdengine/error_reporter.rs index 2c15783f..517be14b 100644 --- a/trace-archiver-tdengine/src/tdengine/error_reporter.rs +++ b/trace-archiver-tdengine/src/tdengine/error_reporter.rs @@ -1,5 +1,5 @@ use std::cmp::Ordering; -use streaming_types::{ +use supermusr_streaming_types::{ dat1_digitizer_analog_trace_v1_generated::{ChannelTrace, DigitizerAnalogTraceMessage}, flatbuffers::{ForwardsUOffset, Vector}, }; diff --git a/trace-archiver-tdengine/src/tdengine/framedata.rs b/trace-archiver-tdengine/src/tdengine/framedata.rs index 29d92ef6..98291a58 100644 --- a/trace-archiver-tdengine/src/tdengine/framedata.rs +++ b/trace-archiver-tdengine/src/tdengine/framedata.rs @@ -1,9 +1,9 @@ use super::{TDEngineError, TraceMessageErrorCode}; use anyhow::Result; use chrono::{DateTime, Duration, Utc}; -use common::{DigitizerId, FrameNumber}; use std::ops::Div; -use streaming_types::dat1_digitizer_analog_trace_v1_generated::DigitizerAnalogTraceMessage; +use supermusr_common::{DigitizerId, FrameNumber}; +use supermusr_streaming_types::dat1_digitizer_analog_trace_v1_generated::DigitizerAnalogTraceMessage; /// Stores and handles some of the data obtained from a DigitizerAnalogTraceMessage message. /// # Fields diff --git a/trace-archiver-tdengine/src/tdengine/mod.rs b/trace-archiver-tdengine/src/tdengine/mod.rs index 05587801..1d40e310 100644 --- a/trace-archiver-tdengine/src/tdengine/mod.rs +++ b/trace-archiver-tdengine/src/tdengine/mod.rs @@ -7,7 +7,7 @@ pub mod wrapper; use anyhow::Result; use async_trait::async_trait; use error::{StatementErrorCode, TDEngineError, TraceMessageErrorCode}; -use streaming_types::dat1_digitizer_analog_trace_v1_generated::DigitizerAnalogTraceMessage; +use supermusr_streaming_types::dat1_digitizer_analog_trace_v1_generated::DigitizerAnalogTraceMessage; #[async_trait] pub(crate) trait TimeSeriesEngine { diff --git a/trace-archiver-tdengine/src/tdengine/views.rs b/trace-archiver-tdengine/src/tdengine/views.rs index da2d3ccd..780f255f 100644 --- a/trace-archiver-tdengine/src/tdengine/views.rs +++ b/trace-archiver-tdengine/src/tdengine/views.rs @@ -3,10 +3,10 @@ use super::{ TraceMessageErrorCode, }; use anyhow::Result; -use common::Intensity; use itertools::Itertools; use std::iter::repeat; -use streaming_types::{ +use supermusr_common::Intensity; +use supermusr_streaming_types::{ dat1_digitizer_analog_trace_v1_generated::ChannelTrace, flatbuffers::{ForwardsUOffset, Vector}, }; diff --git a/trace-archiver-tdengine/src/tdengine/wrapper.rs b/trace-archiver-tdengine/src/tdengine/wrapper.rs index cc63b9af..fbca02ac 100644 --- a/trace-archiver-tdengine/src/tdengine/wrapper.rs +++ b/trace-archiver-tdengine/src/tdengine/wrapper.rs @@ -7,7 +7,7 @@ use super::{ use anyhow::{Error, Result}; use async_trait::async_trait; use log::debug; -use streaming_types::dat1_digitizer_analog_trace_v1_generated::DigitizerAnalogTraceMessage; +use supermusr_streaming_types::dat1_digitizer_analog_trace_v1_generated::DigitizerAnalogTraceMessage; use taos::{AsyncBindable, AsyncQueryable, AsyncTBuilder, Stmt, Taos, TaosBuilder, Value}; pub(crate) struct TDEngine { diff --git a/trace-archiver/Cargo.toml b/trace-archiver/Cargo.toml index 54e0deec..86e7ed95 100644 --- a/trace-archiver/Cargo.toml +++ b/trace-archiver/Cargo.toml @@ -5,9 +5,6 @@ license.workspace = true edition.workspace = true [dependencies] -common = { path = "../common" } -streaming-types = { path = "../streaming-types" } - anyhow.workspace = true chrono.workspace = true clap.workspace = true @@ -18,4 +15,6 @@ lazy_static.workspace = true log.workspace = true ndarray.workspace = true rdkafka.workspace = true +supermusr-common.workspace = true +supermusr-streaming-types.workspace = true tokio.workspace = true diff --git a/trace-archiver/src/file.rs b/trace-archiver/src/file.rs index ff4238a4..e98997f0 100644 --- a/trace-archiver/src/file.rs +++ b/trace-archiver/src/file.rs @@ -1,11 +1,11 @@ use anyhow::{anyhow, Result}; use chrono::{DateTime, Utc}; -use common::{Channel, DigitizerId, FrameNumber, Intensity}; use hdf5::{Extents, File}; use log::error; use ndarray::{arr1, s, Array}; use std::path::{Path, PathBuf}; -use streaming_types::dat1_digitizer_analog_trace_v1_generated::DigitizerAnalogTraceMessage; +use supermusr_common::{Channel, DigitizerId, FrameNumber, Intensity}; +use supermusr_streaming_types::dat1_digitizer_analog_trace_v1_generated::DigitizerAnalogTraceMessage; /// Generate the filename for a HDF5 file for data from a trace message. fn generate_filename(msg: DigitizerAnalogTraceMessage<'_>) -> Result { @@ -121,10 +121,10 @@ pub(super) fn create(dir: &Path, msg: DigitizerAnalogTraceMessage<'_>) -> Result #[cfg(test)] mod test { use super::*; - use common::Intensity; use ndarray::{arr1, arr2, s}; use std::env; - use streaming_types::{ + use supermusr_common::Intensity; + use supermusr_streaming_types::{ dat1_digitizer_analog_trace_v1_generated::{ finish_digitizer_analog_trace_message_buffer, root_as_digitizer_analog_trace_message, ChannelTrace, ChannelTraceArgs, DigitizerAnalogTraceMessage, diff --git a/trace-archiver/src/main.rs b/trace-archiver/src/main.rs index 5b862d28..aecae9cc 100644 --- a/trace-archiver/src/main.rs +++ b/trace-archiver/src/main.rs @@ -9,7 +9,7 @@ use rdkafka::{ message::Message, }; use std::{net::SocketAddr, path::PathBuf}; -use streaming_types::dat1_digitizer_analog_trace_v1_generated::{ +use supermusr_streaming_types::dat1_digitizer_analog_trace_v1_generated::{ digitizer_analog_trace_message_buffer_has_identifier, root_as_digitizer_analog_trace_message, }; @@ -49,13 +49,16 @@ async fn main() -> Result<()> { metrics::register(&mut watcher); watcher.start_server(args.observability_address).await; - let consumer: StreamConsumer = - common::generate_kafka_client_config(&args.broker, &args.username, &args.password) - .set("group.id", &args.consumer_group) - .set("enable.partition.eof", "false") - .set("session.timeout.ms", "6000") - .set("enable.auto.commit", "false") - .create()?; + let consumer: StreamConsumer = supermusr_common::generate_kafka_client_config( + &args.broker, + &args.username, + &args.password, + ) + .set("group.id", &args.consumer_group) + .set("enable.partition.eof", "false") + .set("session.timeout.ms", "6000") + .set("enable.auto.commit", "false") + .create()?; consumer.subscribe(&[&args.trace_topic])?; diff --git a/trace-archiver/src/metrics.rs b/trace-archiver/src/metrics.rs index d5fb0806..7daf1a23 100644 --- a/trace-archiver/src/metrics.rs +++ b/trace-archiver/src/metrics.rs @@ -1,12 +1,12 @@ -pub(crate) use common::metrics::{ - failures::{FailureKind, FailureLabels}, - messages_received::{MessageKind, MessagesReceivedLabels}, -}; use kagiyama::{ prometheus::metrics::{counter::Counter, family::Family}, AlwaysReady, Watcher, }; use lazy_static::lazy_static; +pub(crate) use supermusr_common::metrics::{ + failures::{FailureKind, FailureLabels}, + messages_received::{MessageKind, MessagesReceivedLabels}, +}; pub(crate) fn register(watcher: &mut Watcher) { let mut registry = watcher.metrics_registry(); diff --git a/trace-reader/Cargo.toml b/trace-reader/Cargo.toml index 70f5471a..4a9172a2 100644 --- a/trace-reader/Cargo.toml +++ b/trace-reader/Cargo.toml @@ -5,14 +5,13 @@ license.workspace = true edition.workspace = true [dependencies] -common = { path = "../common" } -streaming-types = { path = "../streaming-types" } - anyhow.workspace = true +chrono.workspace = true clap.workspace = true +env_logger.workspace = true +log.workspace = true +rand.workspace = true rdkafka.workspace = true +supermusr-common.workspace = true +supermusr-streaming-types.workspace = true tokio.workspace = true -chrono = "0.4.22" -env_logger = "0.10" -log = "0.4" -rand = "0.8.5" diff --git a/trace-reader/src/main.rs b/trace-reader/src/main.rs index 704be2e3..e7a6f4f2 100644 --- a/trace-reader/src/main.rs +++ b/trace-reader/src/main.rs @@ -1,8 +1,8 @@ use clap::Parser; -use common::{DigitizerId, FrameNumber}; use rand::{seq::IteratorRandom, thread_rng}; use rdkafka::producer::FutureProducer; use std::path::PathBuf; +use supermusr_common::{DigitizerId, FrameNumber}; mod loader; mod processing; @@ -59,8 +59,11 @@ async fn main() { let args = Cli::parse(); - let client_config = - common::generate_kafka_client_config(&args.broker, &args.username, &args.password); + let client_config = supermusr_common::generate_kafka_client_config( + &args.broker, + &args.username, + &args.password, + ); let producer: FutureProducer = client_config .create() diff --git a/trace-reader/src/processing.rs b/trace-reader/src/processing.rs index 230b0d70..5ca5c431 100644 --- a/trace-reader/src/processing.rs +++ b/trace-reader/src/processing.rs @@ -11,8 +11,8 @@ use rdkafka::{ }; use std::time::Duration; -use common::{Channel, DigitizerId, FrameNumber, Intensity}; -use streaming_types::{ +use supermusr_common::{Channel, DigitizerId, FrameNumber, Intensity}; +use supermusr_streaming_types::{ dat1_digitizer_analog_trace_v1_generated::{ finish_digitizer_analog_trace_message_buffer, ChannelTrace, ChannelTraceArgs, DigitizerAnalogTraceMessage, DigitizerAnalogTraceMessageArgs, diff --git a/trace-to-events/Cargo.toml b/trace-to-events/Cargo.toml index 9d627bea..9caf2e05 100644 --- a/trace-to-events/Cargo.toml +++ b/trace-to-events/Cargo.toml @@ -5,20 +5,19 @@ license.workspace = true edition.workspace = true [dependencies] -common = { path = "../common" } -streaming-types = { path = "../streaming-types" } - anyhow.workspace = true clap.workspace = true env_logger.workspace = true kagiyama.workspace = true lazy_static.workspace = true log.workspace = true -num = "0.4.1" +num.workspace = true rdkafka.workspace = true +supermusr-common.workspace = true +supermusr-streaming-types.workspace = true tokio.workspace = true [dev-dependencies] -assert_approx_eq = "1.1.0" +assert_approx_eq.workspace = true chrono.workspace = true -rand = "0.8.5" +rand.workspace = true diff --git a/trace-to-events/src/main.rs b/trace-to-events/src/main.rs index 75577654..84f6c2fb 100644 --- a/trace-to-events/src/main.rs +++ b/trace-to-events/src/main.rs @@ -12,7 +12,7 @@ use rdkafka::{ producer::{FutureProducer, FutureRecord}, }; use std::{net::SocketAddr, path::PathBuf, time::Duration}; -use streaming_types::dat1_digitizer_analog_trace_v1_generated::{ +use supermusr_streaming_types::dat1_digitizer_analog_trace_v1_generated::{ digitizer_analog_trace_message_buffer_has_identifier, root_as_digitizer_analog_trace_message, }; @@ -57,8 +57,11 @@ async fn main() { metrics::register(&watcher); watcher.start_server(args.observability_address).await; - let mut client_config = - common::generate_kafka_client_config(&args.broker, &args.username, &args.password); + let mut client_config = supermusr_common::generate_kafka_client_config( + &args.broker, + &args.username, + &args.password, + ); let producer: FutureProducer = client_config .create() diff --git a/trace-to-events/src/metrics.rs b/trace-to-events/src/metrics.rs index b8cc1210..adf4d6f7 100644 --- a/trace-to-events/src/metrics.rs +++ b/trace-to-events/src/metrics.rs @@ -1,12 +1,12 @@ -pub(crate) use common::metrics::{ - failures::{FailureKind, FailureLabels}, - messages_received::{MessageKind, MessagesReceivedLabels}, -}; use kagiyama::{ prometheus::metrics::{counter::Counter, family::Family}, AlwaysReady, Watcher, }; use lazy_static::lazy_static; +pub(crate) use supermusr_common::metrics::{ + failures::{FailureKind, FailureLabels}, + messages_received::{MessageKind, MessagesReceivedLabels}, +}; pub(crate) fn register(watcher: &Watcher) { let mut registry = watcher.metrics_registry(); diff --git a/trace-to-events/src/processing.rs b/trace-to-events/src/processing.rs index 1a103b44..68bc6c4f 100644 --- a/trace-to-events/src/processing.rs +++ b/trace-to-events/src/processing.rs @@ -7,9 +7,9 @@ use crate::{ AssembleFilter, EventFilter, Real, SaveToFileFilter, }, }; -use common::{Channel, EventData, Intensity, Time}; use std::path::{Path, PathBuf}; -use streaming_types::{ +use supermusr_common::{Channel, EventData, Intensity, Time}; +use supermusr_streaming_types::{ dat1_digitizer_analog_trace_v1_generated::{ChannelTrace, DigitizerAnalogTraceMessage}, dev1_digitizer_event_v1_generated::{ finish_digitizer_event_list_message_buffer, DigitizerEventListMessage, @@ -246,7 +246,7 @@ mod tests { use crate::parameters::ThresholdDurationWrapper; use chrono::Utc; use std::str::FromStr; - use streaming_types::{ + use supermusr_streaming_types::{ dat1_digitizer_analog_trace_v1_generated::{ finish_digitizer_analog_trace_message_buffer, root_as_digitizer_analog_trace_message, ChannelTraceArgs, DigitizerAnalogTraceMessage, DigitizerAnalogTraceMessageArgs, diff --git a/trace-to-events/src/pulse_detection/datatype/mod.rs b/trace-to-events/src/pulse_detection/datatype/mod.rs index aaa8d902..b698d08c 100644 --- a/trace-to-events/src/pulse_detection/datatype/mod.rs +++ b/trace-to-events/src/pulse_detection/datatype/mod.rs @@ -1,6 +1,6 @@ use super::Real; -use common::Intensity; use std::fmt::{Debug, Display}; +use supermusr_common::Intensity; pub(crate) mod eventdata; pub(crate) mod eventpoint; diff --git a/trace-to-events/src/pulse_detection/window/finite_differences.rs b/trace-to-events/src/pulse_detection/window/finite_differences.rs index 83e40f08..2c5e834a 100644 --- a/trace-to-events/src/pulse_detection/window/finite_differences.rs +++ b/trace-to-events/src/pulse_detection/window/finite_differences.rs @@ -64,7 +64,7 @@ impl Window for FiniteDifferences { mod tests { use super::*; use crate::pulse_detection::window::WindowFilter; - use common::Intensity; + use supermusr_common::Intensity; #[test] fn sample_data() {