From 5a38775c0010869b860c24fc5cea8020ac729c93 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Tue, 25 Jun 2024 17:26:00 +1200 Subject: [PATCH 01/10] fix: Remove role restriction for removing functions (#833) Also runs `cargo format` (1st commit) closes: #822 --- registry/contract/src/lib.rs | 70 +++++++++++++++++------------------- 1 file changed, 33 insertions(+), 37 deletions(-) diff --git a/registry/contract/src/lib.rs b/registry/contract/src/lib.rs index d3d5d8023..91ffbd7f1 100644 --- a/registry/contract/src/lib.rs +++ b/registry/contract/src/lib.rs @@ -5,7 +5,8 @@ use near_sdk::store::UnorderedMap; use near_sdk::{env, log, near_bindgen, serde_json, AccountId, BorshStorageKey, CryptoHash}; use registry_types::{ - AccountIndexers, AccountOrAllIndexers, AllIndexers, IndexerConfig, IndexerIdentity, OldIndexerConfig, Rule, StartBlock, Status, + AccountIndexers, AccountOrAllIndexers, AllIndexers, IndexerConfig, IndexerIdentity, + OldIndexerConfig, Rule, StartBlock, Status, }; type FunctionName = String; @@ -123,7 +124,6 @@ impl Contract { StorageKeys::AccountV4(env::sha256_array(account_id.as_bytes())), ); - for (function_name, indexer_config) in indexers.iter() { let new_config: IndexerConfig = indexer_config.clone().into(); new_indexers.insert(function_name.to_string(), new_config); @@ -259,9 +259,7 @@ impl Contract { } account_id } - None => { - env::signer_account_id() - } + None => env::signer_account_id(), }; log!( @@ -279,7 +277,9 @@ impl Contract { affected_account_id, .. } => { - if affected_account_id.split(',').any(|account_id| ["*", "*.near", "*.kaiching", "*.tg"].contains(&account_id.trim())) { + if affected_account_id.split(',').any(|account_id| { + ["*", "*.near", "*.kaiching", "*.tg"].contains(&account_id.trim()) + }) { self.assert_roles(vec![Role::Owner]); } } @@ -329,10 +329,7 @@ impl Contract { env::panic_str(&format!("Account ID {} is invalid", account_id)); }) } - None => { - self.assert_roles(vec![Role::Owner, Role::User]); - env::signer_account_id() - } + None => env::signer_account_id(), }; log!( @@ -369,7 +366,7 @@ impl Contract { AccountOrAllIndexers::AccountIndexers(self.list_by_account(account_id)) } - None => AccountOrAllIndexers::AllIndexers(self.list_all()) + None => AccountOrAllIndexers::AllIndexers(self.list_all()), } } @@ -423,7 +420,7 @@ mod tests { rule: Rule::ActionFunctionCall { affected_account_id: String::from("social.near"), status: Status::Any, - function: String::from("set") + function: String::from("set"), }, updated_at_block_height: None, created_at_block_height: 10, @@ -437,7 +434,7 @@ mod tests { start_block: StartBlock::Height(100), rule: Rule::ActionAny { affected_account_id: String::from("social.near"), - status: Status::Success + status: Status::Success, }, updated_at_block_height: Some(20), created_at_block_height: 10, @@ -732,7 +729,7 @@ mod tests { rule: Rule::ActionFunctionCall { affected_account_id: String::from("social.near"), status: Status::Any, - function: String::from("set") + function: String::from("set"), }, updated_at_block_height: None, created_at_block_height: 0, @@ -962,7 +959,7 @@ mod tests { rule: Rule::ActionFunctionCall { affected_account_id: String::from("social.near"), status: Status::Any, - function: String::from("set") + function: String::from("set"), }, updated_at_block_height: None, created_at_block_height: 0, @@ -1007,7 +1004,7 @@ mod tests { start_block: StartBlock::Latest, code: "var x= 1;".to_string(), schema: String::new(), - rule: Rule:: ActionFunctionCall { + rule: Rule::ActionFunctionCall { affected_account_id: "test".to_string(), function: "test".to_string(), status: Status::Fail, @@ -1055,7 +1052,7 @@ mod tests { start_block: StartBlock::Latest, code: "var x= 1;".to_string(), schema: String::new(), - rule: Rule:: ActionFunctionCall { + rule: Rule::ActionFunctionCall { affected_account_id: "test".to_string(), function: "test".to_string(), status: Status::Fail, @@ -1078,7 +1075,7 @@ mod tests { start_block: StartBlock::Latest, code: "var x= 1;".to_string(), schema: String::new(), - rule: Rule:: ActionFunctionCall { + rule: Rule::ActionFunctionCall { affected_account_id: "test".to_string(), function: "test".to_string(), status: Status::Fail, @@ -1124,7 +1121,7 @@ mod tests { contract.register( "test_function".to_string(), None, - String::new(), + String::new(), String::new(), Rule::ActionFunctionCall { affected_account_id: String::from("*"), @@ -1148,7 +1145,7 @@ mod tests { contract.register( "test_function".to_string(), None, - String::new(), + String::new(), String::new(), Rule::ActionFunctionCall { affected_account_id: String::from("*.near"), @@ -1171,7 +1168,7 @@ mod tests { contract.register( "test_function".to_string(), None, - String::new(), + String::new(), String::new(), Rule::ActionAny { affected_account_id: String::from("*"), @@ -1196,7 +1193,7 @@ mod tests { start_block: StartBlock::Latest, code: "var x= 1;".to_string(), schema: String::new(), - rule: Rule:: ActionAny { + rule: Rule::ActionAny { affected_account_id: "social.near".to_string(), status: Status::Success, }, @@ -1235,7 +1232,7 @@ mod tests { start_block: StartBlock::Latest, code: "var x= 1;".to_string(), schema: String::new(), - rule: Rule:: ActionAny { + rule: Rule::ActionAny { affected_account_id: "social.near".to_string(), status: Status::Success, }, @@ -1275,7 +1272,7 @@ mod tests { start_block: StartBlock::Latest, code: "var x= 1;".to_string(), schema: String::new(), - rule: Rule:: ActionAny { + rule: Rule::ActionAny { affected_account_id: "social.near".to_string(), status: Status::Success, }, @@ -1309,7 +1306,7 @@ mod tests { start_block: StartBlock::Latest, code: "var x= 1;".to_string(), schema: String::new(), - rule: Rule:: ActionAny { + rule: Rule::ActionAny { affected_account_id: "social.near".to_string(), status: Status::Success, }, @@ -1337,8 +1334,7 @@ mod tests { } #[test] - #[should_panic(expected = "Account bob.near does not have any roles")] - fn anonymous_cannot_remove_functions() { + fn anonymous_can_remove_functions() { let account_id = "bob.near".parse::().unwrap(); let mut account_indexers = IndexerConfigByFunctionName::new(StorageKeys::Account( env::sha256_array(account_id.as_bytes()), @@ -1349,7 +1345,7 @@ mod tests { start_block: StartBlock::Latest, code: "var x= 1;".to_string(), schema: String::new(), - rule: Rule:: ActionAny { + rule: Rule::ActionAny { affected_account_id: "social.near".to_string(), status: Status::Success, }, @@ -1380,7 +1376,7 @@ mod tests { start_block: StartBlock::Latest, code: "var x= 1;".to_string(), schema: String::new(), - rule: Rule:: ActionAny { + rule: Rule::ActionAny { affected_account_id: "social.near".to_string(), status: Status::Success, }, @@ -1395,7 +1391,7 @@ mod tests { start_block: StartBlock::Latest, code: "var x= 1;".to_string(), schema: String::new(), - rule: Rule:: ActionAny { + rule: Rule::ActionAny { affected_account_id: "social.near".to_string(), status: Status::Success, }, @@ -1449,7 +1445,7 @@ mod tests { start_block: StartBlock::Latest, code: "var x= 1;".to_string(), schema: String::new(), - rule: Rule:: ActionAny { + rule: Rule::ActionAny { affected_account_id: "social.near".to_string(), status: Status::Success, }, @@ -1482,7 +1478,7 @@ mod tests { start_block: StartBlock::Latest, code: "var x= 1;".to_string(), schema: String::new(), - rule: Rule:: ActionAny { + rule: Rule::ActionAny { affected_account_id: "social.near".to_string(), status: Status::Success, }, @@ -1522,7 +1518,7 @@ mod tests { start_block: StartBlock::Latest, code: "var x= 1;".to_string(), schema: String::new(), - rule: Rule:: ActionAny { + rule: Rule::ActionAny { affected_account_id: "social.near".to_string(), status: Status::Success, }, @@ -1557,7 +1553,7 @@ mod tests { start_block: StartBlock::Latest, code: "var x= 1;".to_string(), schema: String::new(), - rule: Rule:: ActionAny { + rule: Rule::ActionAny { affected_account_id: "social.near".to_string(), status: Status::Success, }, @@ -1589,7 +1585,7 @@ mod tests { start_block: StartBlock::Latest, code: "var x= 1;".to_string(), schema: String::new(), - rule: Rule:: ActionAny { + rule: Rule::ActionAny { affected_account_id: "social.near".to_string(), status: Status::Success, }, @@ -1645,7 +1641,7 @@ mod tests { start_block: StartBlock::Latest, code: String::from("code"), schema: String::from("schema"), - rule: Rule:: ActionAny { + rule: Rule::ActionAny { affected_account_id: "social.near".to_string(), status: Status::Any, }, @@ -1715,7 +1711,7 @@ mod tests { start_block: StartBlock::Latest, code: String::from("code"), schema: String::from("schema"), - rule: Rule:: ActionAny { + rule: Rule::ActionAny { affected_account_id: "social.near".to_string(), status: Status::Any, }, From f52dfa331e0e3d4cccc847d248702ae081c2c85d Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Wed, 26 Jun 2024 09:27:45 +1200 Subject: [PATCH 02/10] feat: Limit Redis Stream length (#834) This PR introduces back pressure to the Redis Stream in Block Streamer, ensuring that the stream does not exceed a specified maximum length. This is achieved by blocking the `redis.publish_block()` call, intermittently polling the Stream length, and publishing once it falls below the configured limit. To aid testing, the current `RedisClient` struct has been split in to two: - `RedisCommands` - thin wrapper around redis commands to make mocking possible. - `RedisClient` - provides higher-level redis functionality, e.g. "publishing blocks", utilising the above. In most cases, `RedisClient` will be used. The split just allows us to test `RedisWrapper` itself. --- block-streamer/Cargo.toml | 2 +- block-streamer/src/block_stream.rs | 53 ++++---- block-streamer/src/main.rs | 4 +- block-streamer/src/redis.rs | 128 +++++++++++++++--- .../src/server/block_streamer_service.rs | 15 +- block-streamer/src/server/mod.rs | 9 +- 6 files changed, 145 insertions(+), 66 deletions(-) diff --git a/block-streamer/Cargo.toml b/block-streamer/Cargo.toml index 2d529d480..f4becc111 100644 --- a/block-streamer/Cargo.toml +++ b/block-streamer/Cargo.toml @@ -29,7 +29,7 @@ serde_json = "1.0.55" tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } tracing-stackdriver = "0.10.0" -tokio = { version = "1.28.0", features = ["full"]} +tokio = { version = "1.28.0", features = ["full", "test-util"]} tokio-util = "0.7.10" tokio-stream = "0.1.14" tonic = "0.10.2" diff --git a/block-streamer/src/block_stream.rs b/block-streamer/src/block_stream.rs index ee403a235..4dbb6b0e7 100644 --- a/block-streamer/src/block_stream.rs +++ b/block-streamer/src/block_stream.rs @@ -12,6 +12,7 @@ use registry_types::Rule; const LAKE_PREFETCH_SIZE: usize = 100; const MAX_STREAM_SIZE_WITH_CACHE: u64 = 100; const DELTA_LAKE_SKIP_ACCOUNTS: [&str; 4] = ["*", "*.near", "*.kaiching", "*.tg"]; +const MAX_STREAM_SIZE: u64 = 100; pub struct Task { handle: JoinHandle>, @@ -45,7 +46,7 @@ impl BlockStream { pub fn start( &mut self, start_block_height: near_indexer_primitives::types::BlockHeight, - redis_client: std::sync::Arc, + redis: std::sync::Arc, delta_lake_client: std::sync::Arc, lake_s3_client: crate::lake_s3_client::SharedLakeS3Client, ) -> anyhow::Result<()> { @@ -74,7 +75,7 @@ impl BlockStream { result = start_block_stream( start_block_height, &indexer_config, - redis_client, + redis, delta_lake_client, lake_s3_client, &chain_id, @@ -129,7 +130,7 @@ impl BlockStream { pub(crate) async fn start_block_stream( start_block_height: near_indexer_primitives::types::BlockHeight, indexer: &IndexerConfig, - redis_client: std::sync::Arc, + redis: std::sync::Arc, delta_lake_client: std::sync::Arc, lake_s3_client: crate::lake_s3_client::SharedLakeS3Client, chain_id: &ChainId, @@ -145,7 +146,7 @@ pub(crate) async fn start_block_stream( let last_indexed_delta_lake_block = process_delta_lake_blocks( start_block_height, delta_lake_client, - redis_client.clone(), + redis.clone(), indexer, redis_stream.clone(), ) @@ -156,7 +157,7 @@ pub(crate) async fn start_block_stream( last_indexed_delta_lake_block, lake_s3_client, lake_prefetch_size, - redis_client, + redis, indexer, redis_stream, chain_id, @@ -175,7 +176,7 @@ pub(crate) async fn start_block_stream( async fn process_delta_lake_blocks( start_block_height: near_indexer_primitives::types::BlockHeight, delta_lake_client: std::sync::Arc, - redis_client: std::sync::Arc, + redis: std::sync::Arc, indexer: &IndexerConfig, redis_stream: String, ) -> anyhow::Result { @@ -230,10 +231,10 @@ async fn process_delta_lake_blocks( for block_height in &blocks_from_index { let block_height = block_height.to_owned(); - redis_client - .publish_block(indexer, redis_stream.clone(), block_height) + redis + .publish_block(indexer, redis_stream.clone(), block_height, MAX_STREAM_SIZE) .await?; - redis_client + redis .set_last_processed_block(indexer, block_height) .await?; } @@ -253,7 +254,7 @@ async fn process_near_lake_blocks( start_block_height: near_indexer_primitives::types::BlockHeight, lake_s3_client: crate::lake_s3_client::SharedLakeS3Client, lake_prefetch_size: usize, - redis_client: std::sync::Arc, + redis: std::sync::Arc, indexer: &IndexerConfig, redis_stream: String, chain_id: &ChainId, @@ -278,7 +279,7 @@ async fn process_near_lake_blocks( let block_height = streamer_message.block.header.height; last_indexed_block = block_height; - redis_client + redis .set_last_processed_block(indexer, block_height) .await?; @@ -289,18 +290,14 @@ async fn process_near_lake_blocks( ); if !matches.is_empty() { - if let Ok(Some(stream_length)) = - redis_client.get_stream_length(redis_stream.clone()).await - { + if let Ok(Some(stream_length)) = redis.get_stream_length(redis_stream.clone()).await { if stream_length <= MAX_STREAM_SIZE_WITH_CACHE { - redis_client - .cache_streamer_message(&streamer_message) - .await?; + redis.cache_streamer_message(&streamer_message).await?; } } - redis_client - .publish_block(indexer, redis_stream.clone(), block_height) + redis + .publish_block(indexer, redis_stream.clone(), block_height, MAX_STREAM_SIZE) .await?; } } @@ -355,17 +352,18 @@ mod tests { .expect_list_matching_block_heights() .returning(|_, _| Ok(vec![107503702, 107503703])); - let mut mock_redis_client = crate::redis::RedisClient::default(); - mock_redis_client + let mut mock_redis = crate::redis::RedisClient::default(); + mock_redis .expect_publish_block() .with( predicate::always(), predicate::eq("stream key".to_string()), predicate::in_iter([107503702, 107503703, 107503705]), + predicate::always(), ) - .returning(|_, _, _| Ok(())) + .returning(|_, _, _, _| Ok(())) .times(3); - mock_redis_client + mock_redis .expect_set_last_processed_block() .with( predicate::always(), @@ -373,11 +371,11 @@ mod tests { ) .returning(|_, _| Ok(())) .times(4); - mock_redis_client + mock_redis .expect_cache_streamer_message() .with(predicate::always()) .returning(|_| Ok(())); - mock_redis_client + mock_redis .expect_get_stream_length() .with(predicate::eq("stream key".to_string())) .returning(|_| Ok(Some(10))); @@ -397,7 +395,7 @@ mod tests { start_block_stream( 91940840, &indexer_config, - std::sync::Arc::new(mock_redis_client), + std::sync::Arc::new(mock_redis), std::sync::Arc::new(mock_delta_lake_client), mock_lake_s3_client, &ChainId::Mainnet, @@ -435,8 +433,9 @@ mod tests { predicate::always(), predicate::eq("stream key".to_string()), predicate::in_iter([107503705]), + predicate::always(), ) - .returning(|_, _, _| Ok(())) + .returning(|_, _, _, _| Ok(())) .times(1); mock_redis_client .expect_set_last_processed_block() diff --git a/block-streamer/src/main.rs b/block-streamer/src/main.rs index f14c65706..7776f77d9 100644 --- a/block-streamer/src/main.rs +++ b/block-streamer/src/main.rs @@ -45,7 +45,7 @@ async fn main() -> anyhow::Result<()> { "Starting Block Streamer" ); - let redis_client = std::sync::Arc::new(redis::RedisClient::connect(&redis_url).await?); + let redis = std::sync::Arc::new(redis::RedisClient::connect(&redis_url).await?); let aws_config = aws_config::from_env().load().await; let s3_config = aws_sdk_s3::Config::from(&aws_config); @@ -58,7 +58,7 @@ async fn main() -> anyhow::Result<()> { tokio::spawn(metrics::init_server(metrics_port).expect("Failed to start metrics server")); - server::init(&grpc_port, redis_client, delta_lake_client, lake_s3_client).await?; + server::init(&grpc_port, redis, delta_lake_client, lake_s3_client).await?; Ok(()) } diff --git a/block-streamer/src/redis.rs b/block-streamer/src/redis.rs index 254e0c9db..03401a10a 100644 --- a/block-streamer/src/redis.rs +++ b/block-streamer/src/redis.rs @@ -10,18 +10,16 @@ use crate::metrics; use crate::utils; #[cfg(test)] -pub use MockRedisClientImpl as RedisClient; +use MockRedisCommandsImpl as RedisCommands; #[cfg(not(test))] -pub use RedisClientImpl as RedisClient; +use RedisCommandsImpl as RedisCommands; -pub struct RedisClientImpl { +struct RedisCommandsImpl { connection: ConnectionManager, } #[cfg_attr(test, mockall::automock)] -impl RedisClientImpl { - const STREAMER_MESSAGE_PREFIX: &'static str = "streamer_message:"; - +impl RedisCommandsImpl { pub async fn connect(redis_url: &str) -> Result { let connection = redis::Client::open(redis_url)? .get_tokio_connection_manager() @@ -91,6 +89,26 @@ impl RedisClientImpl { Ok(()) } +} + +#[cfg(test)] +pub use MockRedisClientImpl as RedisClient; +#[cfg(not(test))] +pub use RedisClientImpl as RedisClient; + +pub struct RedisClientImpl { + commands: RedisCommands, +} + +#[cfg_attr(test, mockall::automock)] +impl RedisClientImpl { + const STREAMER_MESSAGE_PREFIX: &'static str = "streamer_message:"; + + pub async fn connect(redis_url: &str) -> Result { + let commands = RedisCommands::connect(redis_url).await?; + + Ok(Self { commands }) + } pub async fn set_last_processed_block( &self, @@ -109,13 +127,14 @@ impl RedisClientImpl { .context("Failed to convert block height (u64) to metrics type (i64)")?, ); - self.set(indexer_config.last_processed_block_key(), height) + self.commands + .set(indexer_config.last_processed_block_key(), height) .await .context("Failed to set last processed block") } pub async fn get_stream_length(&self, stream: String) -> anyhow::Result> { - self.xlen(stream).await + self.commands.xlen(stream).await } pub async fn cache_streamer_message( @@ -128,13 +147,14 @@ impl RedisClientImpl { utils::snake_to_camel(&mut streamer_message); - self.set_ex( - format!("{}{}", Self::STREAMER_MESSAGE_PREFIX, height), - serde_json::to_string(&streamer_message)?, - 60, - ) - .await - .context("Failed to cache streamer message") + self.commands + .set_ex( + format!("{}{}", Self::STREAMER_MESSAGE_PREFIX, height), + serde_json::to_string(&streamer_message)?, + 60, + ) + .await + .context("Failed to cache streamer message") } pub async fn publish_block( @@ -142,16 +162,82 @@ impl RedisClientImpl { indexer: &IndexerConfig, stream: String, block_height: u64, + max_size: u64, ) -> anyhow::Result<()> { + loop { + let stream_length = self.get_stream_length(stream.clone()).await?; + + if stream_length.is_none() { + break; + } + + if stream_length.unwrap() < max_size { + break; + } + + println!("Waiting for stream to be consumed"); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } + metrics::PUBLISHED_BLOCKS_COUNT .with_label_values(&[&indexer.get_full_name()]) .inc(); - self.xadd( - stream.clone(), - &[(String::from("block_height"), block_height)], - ) - .await - .context("Failed to add block to Redis Stream") + self.commands + .xadd( + stream.clone(), + &[(String::from("block_height"), block_height)], + ) + .await + .context("Failed to add block to Redis Stream") + } +} + +#[cfg(test)] +mod test { + use super::*; + + use mockall::predicate; + use near_lake_framework::near_indexer_primitives; + + #[tokio::test] + async fn limits_block_stream_length() { + let mut mock_redis_commands = RedisCommands::default(); + mock_redis_commands + .expect_xadd::() + .with(predicate::eq("stream".to_string()), predicate::always()) + .returning(|_, _| Ok(())) + .once(); + let mut stream_len = 10; + mock_redis_commands + .expect_xlen::() + .with(predicate::eq("stream".to_string())) + .returning(move |_| { + stream_len -= 1; + Ok(Some(stream_len)) + }); + + let redis = RedisClientImpl { + commands: mock_redis_commands, + }; + + let indexer_config = crate::indexer_config::IndexerConfig { + account_id: near_indexer_primitives::types::AccountId::try_from( + "morgs.near".to_string(), + ) + .unwrap(), + function_name: "test".to_string(), + rule: registry_types::Rule::ActionAny { + affected_account_id: "queryapi.dataplatform.near".to_string(), + status: registry_types::Status::Success, + }, + }; + + tokio::time::pause(); + + redis + .publish_block(&indexer_config, "stream".to_string(), 0, 1) + .await + .unwrap(); } } diff --git a/block-streamer/src/server/block_streamer_service.rs b/block-streamer/src/server/block_streamer_service.rs index 7c2a6d45a..4c7d2c69d 100644 --- a/block-streamer/src/server/block_streamer_service.rs +++ b/block-streamer/src/server/block_streamer_service.rs @@ -13,7 +13,7 @@ use crate::server::blockstreamer; use blockstreamer::*; pub struct BlockStreamerService { - redis_client: std::sync::Arc, + redis: std::sync::Arc, delta_lake_client: std::sync::Arc, lake_s3_client: crate::lake_s3_client::SharedLakeS3Client, chain_id: ChainId, @@ -22,12 +22,12 @@ pub struct BlockStreamerService { impl BlockStreamerService { pub fn new( - redis_client: std::sync::Arc, + redis: std::sync::Arc, delta_lake_client: std::sync::Arc, lake_s3_client: crate::lake_s3_client::SharedLakeS3Client, ) -> Self { Self { - redis_client, + redis, delta_lake_client, lake_s3_client, chain_id: ChainId::Mainnet, @@ -113,7 +113,7 @@ impl blockstreamer::block_streamer_server::BlockStreamer for BlockStreamerServic block_stream .start( request.start_block_height, - self.redis_client.clone(), + self.redis.clone(), self.delta_lake_client.clone(), self.lake_s3_client.clone(), ) @@ -206,10 +206,7 @@ mod tests { .expect_list_matching_block_heights() .returning(|_, _| Ok(vec![])); - let mut mock_redis_client = crate::redis::RedisClient::default(); - mock_redis_client - .expect_xadd::() - .returning(|_, _| Ok(())); + let mock_redis = crate::redis::RedisClient::default(); let mut mock_lake_s3_client = crate::lake_s3_client::SharedLakeS3Client::default(); mock_lake_s3_client @@ -217,7 +214,7 @@ mod tests { .returning(crate::lake_s3_client::SharedLakeS3Client::default); BlockStreamerService::new( - std::sync::Arc::new(mock_redis_client), + std::sync::Arc::new(mock_redis), std::sync::Arc::new(mock_delta_lake_client), mock_lake_s3_client, ) diff --git a/block-streamer/src/server/mod.rs b/block-streamer/src/server/mod.rs index fcdb77068..0159550d4 100644 --- a/block-streamer/src/server/mod.rs +++ b/block-streamer/src/server/mod.rs @@ -6,7 +6,7 @@ pub mod blockstreamer { pub async fn init( port: &str, - redis_client: std::sync::Arc, + redis: std::sync::Arc, delta_lake_client: std::sync::Arc, lake_s3_client: crate::lake_s3_client::SharedLakeS3Client, ) -> anyhow::Result<()> { @@ -14,11 +14,8 @@ pub async fn init( tracing::info!("Starting gRPC server on {}", addr); - let block_streamer_service = block_streamer_service::BlockStreamerService::new( - redis_client, - delta_lake_client, - lake_s3_client, - ); + let block_streamer_service = + block_streamer_service::BlockStreamerService::new(redis, delta_lake_client, lake_s3_client); let block_streamer_server = blockstreamer::block_streamer_server::BlockStreamerServer::new(block_streamer_service); From 9ec0cedf9ea45e9fcb23087dba0192ed80330eea Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Wed, 26 Jun 2024 10:15:28 +1200 Subject: [PATCH 03/10] chore: Remove `println` (#838) --- block-streamer/src/redis.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/block-streamer/src/redis.rs b/block-streamer/src/redis.rs index 03401a10a..959b5ddc9 100644 --- a/block-streamer/src/redis.rs +++ b/block-streamer/src/redis.rs @@ -175,7 +175,6 @@ impl RedisClientImpl { break; } - println!("Waiting for stream to be consumed"); tokio::time::sleep(std::time::Duration::from_secs(1)).await; } From ec17eee85c44f185b1a5952ec6742c74ecbe8749 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Wed, 26 Jun 2024 10:54:38 +1200 Subject: [PATCH 04/10] feat: Add script to suspend Indexers (#829) This PR adds a Node script to Runner to suspend Indexers due to inactivity. The script will: 1. Call Coordinator to disable the indexer 2. Write to the Indexers logs table to notify of suspension Note that as Coordinator is in a private network, you must tunnel to the machine to expose the gRPC server. This can be achieved via running the following in a separate terminal: ```sh gcloud compute ssh ubuntu@queryapi-coordinator-mainnet -- -L 9003:0.0.0.0:9003 ``` The following environment variables are required: - `HASURA_ADMIN_SECRET` - `HASURA_ENDPOINT` - `PGPORT` - `PGHOST` All of which can be found in the Runner compute instance metadata: ```sh gcloud compute instances describe queryapi-runner-mainnet ``` Usage: `npm run script:suspend-indexer -- ` --- runner/package.json | 3 +- runner/scripts/suspend-indexer.ts | 100 ++++++++++++++++++++++++++++++ runner/src/indexer-meta/index.ts | 1 + runner/tsconfig.build.json | 2 +- runner/tsconfig.json | 4 +- 5 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 runner/scripts/suspend-indexer.ts diff --git a/runner/package.json b/runner/package.json index 87b21b967..3cc4f15bc 100644 --- a/runner/package.json +++ b/runner/package.json @@ -14,7 +14,8 @@ "start:dev": "tsc -p ./tsconfig.build.json && node ./dist", "start:docker": "node dist/index.js", "test": "npm run codegen && node --experimental-vm-modules ./node_modules/.bin/jest --silent", - "lint": "eslint -c .eslintrc.js" + "lint": "eslint -c .eslintrc.js", + "script:suspend-indexer": "tsc -p ./tsconfig.json && node ./dist/scripts/suspend-indexer.js" }, "keywords": [], "author": "", diff --git a/runner/scripts/suspend-indexer.ts b/runner/scripts/suspend-indexer.ts new file mode 100644 index 000000000..060339a2e --- /dev/null +++ b/runner/scripts/suspend-indexer.ts @@ -0,0 +1,100 @@ +/* + * This script is used to suspend an indexer for a given account. It will: + * 1. Call Coordinator to disable the indexer + * 2. Write to the Indexers logs table to notify of suspension + * + * Note that as Coordinator is in a private network, you must tunnel to the machine to expose the gRPC server. + * This can be achieved via running the following in a separate terminal: + * ```sh + * gcloud compute ssh ubuntu@queryapi-coordinator-mainnet -- -L 9003:0.0.0.0:9003 + * ``` + * + * The following environment variables are required: + * - `HASURA_ADMIN_SECRET` + * - `HASURA_ENDPOINT` + * - `PGPORT` + * - `PGHOST` + * + * All of which can be found in the Runner compute instance metadata: + * ```sh + * gcloud compute instances describe queryapi-runner-mainnet + * ``` + * + * + * Usage: npm run script:suspend-indexer -- +*/ + +import assert from 'assert' +import * as fs from 'fs' + +import * as grpc from '@grpc/grpc-js' +import * as protoLoader from '@grpc/proto-loader' + +import Provisioner from '../src/provisioner' +import IndexerConfig from '../src/indexer-config' +import IndexerMeta, { LogEntry } from '../src/indexer-meta'; + +const COORDINATOR_PROTO_PATH = '../coordinator/proto/indexer_manager.proto'; + +assert(exists(COORDINATOR_PROTO_PATH), 'Coordinator proto file not found. Make sure you run this script from the root directory.'); +assert(process.argv.length === 4, 'Usage: npm run script:suspend-indexer -- '); +assert(process.env.COORDINATOR_PORT, 'COORDINATOR_PORT env var is required'); +assert(process.env.HASURA_ADMIN_SECRET, 'HASURA_ADMIN_SECRET env var is required'); +assert(process.env.HASURA_ENDPOINT, 'HASURA_ENDPOINT env var is required'); +assert(process.env.PGPORT, 'PGPORT env var is required'); +assert(process.env.PGHOST, 'PGHOST env var is required'); + +const [_binary, _file, accountId, functionName] = process.argv; +const { COORDINATOR_PORT = 9003 } = process.env; + +main(); + +async function main() { + await suspendIndexer(); + await logSuspension(); + + console.log('Done') +} + +async function logSuspension() { + console.log('Logging suspension notification'); + + const config = new IndexerConfig('not needed', accountId, functionName, 0, 'not needed', 'not needed', 2); + + const pgCredentials = await new Provisioner().getPostgresConnectionParameters(config.userName()); + + await new IndexerMeta(config, pgCredentials).writeLogs([ + LogEntry.systemInfo('The indexer is suspended due to inactivity.'), + ]); +} + +async function suspendIndexer() { + console.log(`Suspending indexer: ${accountId}/${functionName}`); + + const indexerManager = createIndexerManagerClient(); + + return new Promise((resolve, reject) => { + indexerManager.disable({ accountId, functionName }, (err: any, response: any) => { + if (err) { + reject(err); + } else { + resolve(response); + } + }); + }) +} + +function exists(path: string): boolean { + try { + fs.statSync(path); + return true; + } catch (err) { + return false; + } +} + +function createIndexerManagerClient() { + const packageDefinition = protoLoader.loadSync(COORDINATOR_PROTO_PATH); + const protoDescriptor: any = grpc.loadPackageDefinition(packageDefinition); + return new protoDescriptor.indexer.IndexerManager(`localhost:${COORDINATOR_PORT}`, grpc.credentials.createInsecure()); +} diff --git a/runner/src/indexer-meta/index.ts b/runner/src/indexer-meta/index.ts index a6bf324cc..05406387c 100644 --- a/runner/src/indexer-meta/index.ts +++ b/runner/src/indexer-meta/index.ts @@ -1,2 +1,3 @@ export { default } from './indexer-meta'; export { IndexerStatus, METADATA_TABLE_UPSERT, MetadataFields } from './indexer-meta'; +export { default as LogEntry } from './log-entry'; diff --git a/runner/tsconfig.build.json b/runner/tsconfig.build.json index d37e1cd77..dfbf07a6b 100644 --- a/runner/tsconfig.build.json +++ b/runner/tsconfig.build.json @@ -1,5 +1,5 @@ { "extends": "./tsconfig.json", "include": ["./src"], - "exclude": ["node_modules", "dist", "**/*.test.*"] + "exclude": ["node_modules", "dist", "**/*.test.*", "scripts"] } diff --git a/runner/tsconfig.json b/runner/tsconfig.json index 4e64096ab..d4949c0e4 100644 --- a/runner/tsconfig.json +++ b/runner/tsconfig.json @@ -3,7 +3,7 @@ "target": "es2018", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ "lib": ["es2021"], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ "module": "commonjs", /* Specify what module code is generated. */ - "rootDirs": ["./src", "./tests"], + "rootDirs": ["./src", "./tests", "./scripts"], "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */ "resolveJsonModule": true, /* Enable importing .json files. */ "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ @@ -20,6 +20,6 @@ "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ "skipLibCheck": true /* Skip type checking all .d.ts files. */ }, - "include": ["./src", "./tests"], + "include": ["./src", "./tests", "./scripts"], "exclude": ["node_modules", "dist"] } From d6ec4112ac51f3b19f10dc0ca951ff44a93eee3d Mon Sep 17 00:00:00 2001 From: Kevin Zhang <42101107+Kevin101Zhang@users.noreply.github.com> Date: Wed, 26 Jun 2024 16:40:44 -0400 Subject: [PATCH 05/10] refactor: Refactor Editor into TypeScript with Smaller Components and Separate Concerns (#830) Refactored Editor Component to TypeScript. This refactoring involved breaking down the Editor file into smaller chunks and separating concerns into distinct components. Also did some minor work on validator to ts as it is a major consumer in the editor. It is setup to later iterate on some additional test for validators. --- frontend/.eslintrc.js | 4 + frontend/primitives.d.ts | 811 ---------------- frontend/src/classes/ValidationError.js | 6 - frontend/src/classes/ValidationError.ts | 17 + frontend/src/components/Common/Alert.tsx | 55 ++ .../Icons/CheckMarkIcon.js | 0 .../LogsView => Common}/Icons/ClearIcon.js | 0 .../src/components/Common/LatestBlock.tsx | 3 +- .../CreateNewIndexer/CreateNewIndexer.js | 2 +- .../DiffEditorComponent.jsx | 0 .../Editor.tsx} | 398 +++----- .../{ => EditorComponents}/FileSwitcher.jsx | 4 +- .../{ => EditorComponents}/GlyphContainer.js | 0 .../MonacoEditorComponent.jsx | 0 .../ResizableLayoutEditor.jsx | 4 +- .../{ => EditorComponents}/block_details.js | 0 .../Editor/EditorComponents/custom.d.ts | 4 + .../Editor/{ => EditorComponents}/index.js | 0 .../Editor/EditorComponents/primitives.d.ts | 908 ++++++++++++++++++ .../Editor/EditorView/DeveloperToolsView.jsx | 17 +- .../Editor/EditorView/EditorMenuView.jsx | 12 +- .../BlockPickerContainer.tsx | 1 + .../DeveloperToolsContainer.tsx | 47 +- .../EditorMenuContainer.jsx | 50 - .../EditorMenuContainer.tsx | 219 +++++ .../Editor/QueryApiStorageManager.tsx | 68 ++ .../Editor/__tests__/Editor.test.js | 82 -- frontend/src/components/Logs/LogsMenu.tsx | 10 +- .../Logs/LogsView/ClearButtonView.jsx | 2 +- .../Logs/LogsView/OptionSelectorView.jsx | 2 +- .../ClearButtonContainer.tsx | 3 +- .../DateSelectorContainer.tsx | 4 +- .../IndexerLogsContainer.tsx | 17 +- .../LogTypeSelectorContainer.tsx | 1 + .../OptionSelectorContainer.tsx | 1 + .../SeveritySelectorContainer.tsx | 1 + .../components/Modals/ForkIndexerModal.jsx | 15 +- .../ModalsContainer/PublishFormContainer.tsx | 3 +- ...resetChanges.jsx => ResetChangesModal.jsx} | 4 +- ...lsContext.js => IndexerDetailsContext.tsx} | 137 +-- frontend/src/contexts/ModalContext.js | 2 +- .../src/pages/create-new-indexer/index.js | 5 +- frontend/src/pages/query-api-editor/index.js | 5 +- frontend/src/utils/formatters.js | 2 +- frontend/src/utils/formatters.test.js | 2 +- frontend/src/utils/helpers.ts | 8 + frontend/src/utils/indexerRunner.js | 1 + frontend/src/utils/validators.js | 79 -- frontend/src/utils/validators.test.ts | 37 +- frontend/src/utils/validators.ts | 107 +++ 50 files changed, 1730 insertions(+), 1430 deletions(-) delete mode 100644 frontend/primitives.d.ts delete mode 100644 frontend/src/classes/ValidationError.js create mode 100644 frontend/src/classes/ValidationError.ts create mode 100644 frontend/src/components/Common/Alert.tsx rename frontend/src/components/{Logs/LogsView => Common}/Icons/CheckMarkIcon.js (100%) rename frontend/src/components/{Logs/LogsView => Common}/Icons/ClearIcon.js (100%) rename frontend/src/components/Editor/{ => EditorComponents}/DiffEditorComponent.jsx (100%) rename frontend/src/components/Editor/{Editor.jsx => EditorComponents/Editor.tsx} (50%) rename frontend/src/components/Editor/{ => EditorComponents}/FileSwitcher.jsx (87%) rename frontend/src/components/Editor/{ => EditorComponents}/GlyphContainer.js (100%) rename frontend/src/components/Editor/{ => EditorComponents}/MonacoEditorComponent.jsx (100%) rename frontend/src/components/Editor/{ => EditorComponents}/ResizableLayoutEditor.jsx (98%) rename frontend/src/components/Editor/{ => EditorComponents}/block_details.js (100%) create mode 100644 frontend/src/components/Editor/EditorComponents/custom.d.ts rename frontend/src/components/Editor/{ => EditorComponents}/index.js (100%) create mode 100644 frontend/src/components/Editor/EditorComponents/primitives.d.ts delete mode 100644 frontend/src/components/Editor/EditorViewContainer/EditorMenuContainer.jsx create mode 100644 frontend/src/components/Editor/EditorViewContainer/EditorMenuContainer.tsx create mode 100644 frontend/src/components/Editor/QueryApiStorageManager.tsx delete mode 100644 frontend/src/components/Editor/__tests__/Editor.test.js rename frontend/src/components/Modals/{resetChanges.jsx => ResetChangesModal.jsx} (84%) rename frontend/src/contexts/{IndexerDetailsContext.js => IndexerDetailsContext.tsx} (50%) delete mode 100644 frontend/src/utils/validators.js create mode 100644 frontend/src/utils/validators.ts diff --git a/frontend/.eslintrc.js b/frontend/.eslintrc.js index 64a016015..b9f962742 100644 --- a/frontend/.eslintrc.js +++ b/frontend/.eslintrc.js @@ -13,5 +13,9 @@ module.exports = { "@typescript-eslint/consistent-type-imports": "error", "@typescript-eslint/no-unused-vars": ['warn', { argsIgnorePattern: "^_", "varsIgnorePattern": "^_" }], '@typescript-eslint/no-empty-function': ['warn', { allow: ['methods'] }], + "@typescript-eslint/ban-ts-comment": ["error", { + "ts-ignore": "allow-with-description", + "minimumDescriptionLength": 5 + }], } }; diff --git a/frontend/primitives.d.ts b/frontend/primitives.d.ts deleted file mode 100644 index ab4ee9302..000000000 --- a/frontend/primitives.d.ts +++ /dev/null @@ -1,811 +0,0 @@ -import type * as borsh_lib_types_types from 'borsh/lib/types/types'; -import type * as borsh from 'borsh'; -import * as borsher from 'borsher'; - -function _mergeNamespaces(n, m) { - m.forEach(function (e) { - e && typeof e !== 'string' && !Array.isArray(e) && Object.keys(e).forEach(function (k) { - if (k !== 'default' && !(k in n)) { - const d = Object.getOwnPropertyDescriptor(e, k); - Object.defineProperty(n, k, d.get ? d : { - enumerable: true, - get: function () { return e[k]; } - }); - } - }); - }); - return Object.freeze(n); -} - -declare class LakeContext { -} - -type BlockHeight = number; -interface StreamerMessage { - block: BlockView; - shards: Shard[]; -} -interface BlockView { - author: string; - header: BlockHeaderView; - chunks: ChunkHeader[]; -} -interface BlockHeaderView { - author: any; - approvals: (string | null)[]; - blockMerkleRoot: string; - blockOrdinal: number; - challengesResult: ChallengeResult[]; - challengesRoot: string; - chunkHeadersRoot: string; - chunkMask: boolean[]; - chunkReceiptsRoot: string; - chunkTxRoot: string; - chunksIncluded: number; - epochId: string; - epochSyncDataHash: string | null; - gasPrice: string; - hash: string; - height: number; - lastDsFinalBlock: string; - lastFinalBlock: string; - latestProtocolVersion: number; - nextBpHash: string; - nextEpochId: string; - outcomeRoot: string; - prevHash: string; - prevHeight: number; - prevStateRoot: string; - randomValue: string; - rentPaid: string; - signature: string; - timestamp: number; - timestampNanosec: string; - totalSupply: string; - validatorProposals: []; - validatorReward: string; -} -interface Shard { - shardId: number; - chunk?: ChunkView; - receiptExecutionOutcomes: ExecutionOutcomeWithReceipt[]; - stateChanges: StateChangeWithCauseView[]; -} -type ValidatorStakeView = { - accountId: string; - publicKey: string; - stake: string; - validatorStakeStructVersion: string; -}; -type ChallengeResult = { - accountId: string; - isDoubleSign: boolean; -}; -interface ChunkHeader { - balanceBurnt: number; - chunkHash: string; - encodedLength: number; - encodedMerkleRoot: string; - gasLimit: number; - gasUsed: number; - heightCreated: number; - heightIncluded: number; - outcomeRoot: string; - outgoingReceiptsRoot: string; - prevBlockHash: string; - prevStateRoot: string; - rentPaid: string; - shardId: number; - signature: string; - txRoot: string; - validatorProposals: ValidatorProposal[]; - validatorReward: string; -} -type ValidatorProposal = { - accountId: string; - publicKey: string; - stake: string; - validatorStakeStructVersion: string; -}; -interface ChunkView { - author: string; - header: ChunkHeader; - receipts: ReceiptView[]; - transactions: IndexerTransactionWithOutcome[]; -} -type ActionReceipt = { - Action: { - actions: ActionView[]; - gasPrice: string; - inputDataIds: string[]; - outputDataReceivers: DataReceiver[]; - signerId: string; - signerPublicKey: string; - }; -}; -type DataReceipt = { - Data: { - data: string; - dataId: string; - }; -}; -type ReceiptEnum = ActionReceipt | DataReceipt; -type DataReceiver = { - dataId: string; - receiverId: string; -}; -type ReceiptView = { - predecessorId: string; - receiptId: string; - receiverId: string; - receipt: ReceiptEnum; -}; -/** - * `ExecutionStatus` is a simplified representation of the `ExecutionStatusView` from [near-primitives](https://github.com/near/nearcore/tree/master/core/primitives). Represent the execution outcome status for the `Receipt`. - */ -type ExecutionStatus = { - /** - * Execution succeeded with a value, value is represented by `Uint8Array` and can be anything. - */ - SuccessValue: Uint8Array; -} | { - /** - * Execution succeeded and a result of the execution is a new `Receipt` with the id. - */ - SuccessReceiptId: string; -} | { - /** - * Execution failed with an error represented by a `String`. - */ - Failure: string; -} | "Postponed"; -type ExecutionProof = { - direction: string; - hash: string; -}; -type ExecutionOutcomeWithReceipt = { - executionOutcome: { - blockHash: string; - id: string; - outcome: { - executorId: string; - gasBurnt: number; - logs: string[]; - metadata: { - gasProfile: string | null; - version: number; - }; - receiptIds: string[]; - status: ExecutionStatus; - tokensBurnt: string; - }; - proof: ExecutionProof[]; - }; - receipt: ReceiptView; -}; -type IndexerTransactionWithOutcome = { - transaction: Transaction$1; - outcome: ExecutionOutcomeWithReceipt; -}; -type Transaction$1 = { - signerId: string; - publicKey: string; - nonce: number; - receiverId: string; - actions: ActionView[]; - signature: string; - hash: string; -}; -type DeployContractAction = { - DeployContract: { - code: string; - }; -}; -type FunctionCallAction = { - FunctionCall: { - methodName: string; - args: string; - gas: number; - deposit: string; - }; -}; -type TransferAction = { - Transfer: { - deposit: string; - }; -}; -type StakeAction = { - Stake: { - stake: number; - publicKey: string; - }; -}; -type AddKeyAction = { - AddKey: { - publicKey: string; - accessKey: AccessKey$1; - }; -}; -interface AccessKey$1 { - nonce: number; - permission: string | AccessKeyFunctionCallPermission$1; -} -interface AccessKeyFunctionCallPermission$1 { - FunctionCall: { - allowance: string; - receiverId: string; - methodNames: string[]; - }; -} -type DeleteKeyAction = { - DeleteKey: { - publicKey: string; - }; -}; -type DeleteAccountAction = { - DeleteAccount: { - beneficiaryId: string; - }; -}; -type DelegateAction = { - Delegate: { - delegateAction: { - senderId: string; - receiverId: string; - actions: NonDelegateAction[]; - nonce: number; - maxBlockHeight: number; - publicKey: string; - }; - }; - signature: string; -}; -type NonDelegateAction = "CreateAccount" | DeployContractAction | FunctionCallAction | TransferAction | StakeAction | AddKeyAction | DeleteKeyAction | DeleteAccountAction; -type ActionView = "CreateAccount" | DeployContractAction | FunctionCallAction | TransferAction | StakeAction | AddKeyAction | DeleteKeyAction | DeleteAccountAction | DelegateAction; -type StateChangeWithCauseView = { - change: { - accountId: string; - keyBase64: string; - valueBase64: string; - }; - cause: { - receiptHash: string; - type: string; - }; - value: { - accountId: string; - keyBase64: string; - valueBase64: string; - }; - type: string; -}; - -type Log = { - log: string; - relatedReceiptId: string; -}; -/** - * This structure is an ephemeral entity to provide access to the [Events Standard](https://github.com/near/NEPs/blob/master/neps/nep-0297.md) structure and keep data about the related `Receipt` for convenience. - * - * #### Interface for Capturing Data About an Event in `handleStreamerMessage()` - * - * The interface to capture data about an event has the following arguments: - * - `standard`: name of standard, e.g. nep171 - * - `version`: e.g. 1.0.0 - * - `event`: type of the event, e.g. `nft_mint` - * - `data`: associate event data. Strictly typed for each set {standard, version, event} inside corresponding NEP - */ -declare class Event { - readonly relatedReceiptId: string; - readonly rawEvent: RawEvent; - constructor(relatedReceiptId: string, rawEvent: RawEvent); - static fromLog: (log: string) => Event; -} -/** - * This structure is a copy of the [JSON Events](https://github.com/near/NEPs/blob/master/neps/nep-0297.md) structure representation. - */ -declare class RawEvent { - readonly event: string; - readonly standard: string; - readonly version: string; - readonly data: JSON | undefined; - constructor(event: string, standard: string, version: string, data: JSON | undefined); - static isEvent: (log: string) => boolean; - static fromLog: (log: string) => RawEvent; -} -type Events = { - events: Event[]; -}; - -/** - * This field is a simplified representation of the `ReceiptView` structure from [near-primitives](https://github.com/near/nearcore/tree/master/core/primitives). - */ -declare class Receipt implements Events { - /** - * Defined the type of the `Receipt`: `Action` or `Data` representing the `ActionReceipt` and `DataReceipt`. - */ - readonly receiptKind: ReceiptKind; - /** - * The ID of the `Receipt` of the `CryptoHash` type. - */ - readonly receiptId: string; - /** - * The receiver account id of the `Receipt`. - */ - readonly receiverId: string; - /** - * The predecessor account id of the `Receipt`. - */ - readonly predecessorId: string; - /** - * Represents the status of `ExecutionOutcome` of the `Receipt`. - */ - readonly status: ExecutionStatus; - /** - * The id of the `ExecutionOutcome` for the `Receipt`. Returns `null` if the `Receipt` isn’t executed yet and has a postponed status. - */ - readonly executionOutcomeId?: string | undefined; - /** - * The original logs of the corresponding `ExecutionOutcome` of the `Receipt`. - * - * **Note:** not all of the logs might be parsed as JSON Events (`Events`). - */ - readonly logs: string[]; - constructor( - /** - * Defined the type of the `Receipt`: `Action` or `Data` representing the `ActionReceipt` and `DataReceipt`. - */ - receiptKind: ReceiptKind, - /** - * The ID of the `Receipt` of the `CryptoHash` type. - */ - receiptId: string, - /** - * The receiver account id of the `Receipt`. - */ - receiverId: string, - /** - * The predecessor account id of the `Receipt`. - */ - predecessorId: string, - /** - * Represents the status of `ExecutionOutcome` of the `Receipt`. - */ - status: ExecutionStatus, - /** - * The id of the `ExecutionOutcome` for the `Receipt`. Returns `null` if the `Receipt` isn’t executed yet and has a postponed status. - */ - executionOutcomeId?: string | undefined, - /** - * The original logs of the corresponding `ExecutionOutcome` of the `Receipt`. - * - * **Note:** not all of the logs might be parsed as JSON Events (`Events`). - */ - logs?: string[]); - /** - * Returns an Array of `Events` for the `Receipt`, if any. This might be empty if the `logs` field is empty or doesn’t contain JSON Events compatible log records. - */ - get events(): Event[]; - static fromOutcomeWithReceipt: (outcomeWithReceipt: ExecutionOutcomeWithReceipt) => Receipt; -} -/** - * `ReceiptKind` a simple `enum` to represent the `Receipt` type: either `Action` or `Data`. - */ -declare enum ReceiptKind { - Action = "Action", - Data = "Data" -} -/** - * `Action` is the structure with the fields and data relevant to an `ActionReceipt`. - * - * Basically, `Action` is the structure that indexer developers will be encouraged to work the most in their action-oriented indexers. - */ -declare class Action { - /** - * The id of the corresponding `Receipt` - */ - readonly receiptId: string; - /** - * The predecessor account id of the corresponding `Receipt`. - * This field is a piece of denormalization of the structures (`Receipt` and `Action`). - */ - readonly predecessorId: string; - /** - * The receiver account id of the corresponding `Receipt`. - * This field is a piece of denormalization of the structures (`Receipt` and `Action`). - */ - readonly receiverId: string; - /** - * The signer account id of the corresponding `Receipt` - */ - readonly signerId: string; - /** - * The signer’s PublicKey for the corresponding `Receipt` - */ - readonly signerPublicKey: string; - /** - * An array of `Operation` for this `ActionReceipt` - */ - readonly operations: Operation[]; - constructor( - /** - * The id of the corresponding `Receipt` - */ - receiptId: string, - /** - * The predecessor account id of the corresponding `Receipt`. - * This field is a piece of denormalization of the structures (`Receipt` and `Action`). - */ - predecessorId: string, - /** - * The receiver account id of the corresponding `Receipt`. - * This field is a piece of denormalization of the structures (`Receipt` and `Action`). - */ - receiverId: string, - /** - * The signer account id of the corresponding `Receipt` - */ - signerId: string, - /** - * The signer’s PublicKey for the corresponding `Receipt` - */ - signerPublicKey: string, - /** - * An array of `Operation` for this `ActionReceipt` - */ - operations: Operation[]); - static isActionReceipt: (receipt: ReceiptView) => boolean; - static fromReceiptView: (receipt: ReceiptView) => Action | null; -} -declare class DeployContract { - readonly code: Uint8Array; - constructor(code: Uint8Array); -} -declare class FunctionCall { - readonly methodName: string; - readonly args: Uint8Array; - readonly gas: number; - readonly deposit: string; - constructor(methodName: string, args: Uint8Array, gas: number, deposit: string); -} -declare class Transfer { - readonly deposit: string; - constructor(deposit: string); -} -declare class Stake { - readonly stake: number; - readonly publicKey: string; - constructor(stake: number, publicKey: string); -} -declare class AddKey { - readonly publicKey: string; - readonly accessKey: AccessKey; - constructor(publicKey: string, accessKey: AccessKey); -} -declare class DeleteKey { - readonly publicKey: string; - constructor(publicKey: string); -} -declare class DeleteAccount { - readonly beneficiaryId: string; - constructor(beneficiaryId: string); -} -/** - * A representation of the original `ActionView` from [near-primitives](https://github.com/near/nearcore/tree/master/core/primitives). - */ -type Operation = 'CreateAccount' | DeployContract | FunctionCall | Transfer | Stake | AddKey | DeleteKey | DeleteAccount; -declare class AccessKey { - readonly nonce: number; - readonly permission: string | AccessKeyFunctionCallPermission; - constructor(nonce: number, permission: string | AccessKeyFunctionCallPermission); -} -declare class AccessKeyFunctionCallPermission { - readonly allowance: string; - readonly receiverId: string; - readonly methodNames: string[]; - constructor(allowance: string, receiverId: string, methodNames: string[]); -} - -/** - * A representation of the `IndexerTransactionWithOutcome` from `near-indexer-primitives` which is an ephemeral structure combining `SignedTransactionView` from [near-primitives](https://github.com/near/nearcore/tree/master/core/primitives) and `IndexerExecutionOutcomeWithOptionalReceipt` from `near-indexer-primitives`. - * - * This structure is very similar to `Receipt`. Unlike `Receipt`, a `Transaction` has a few additional fields like `signerId`, `signature`, and `operations`. - */ -declare class Transaction { - /** - * Returns the hash of the `Transaction` in `CryptoHash`. - */ - readonly transactionHash: string; - /** - * Returns the signer account id of the `Transaction`. - */ - readonly signerId: string; - /** - * Returns the `PublicKey` of the signer of the `Transaction`. - */ - readonly signerPublicKey: string; - /** - * Returns the `Signature` the `Transaction` was signed with. - */ - readonly signature: string; - /** - * Returns the receiver account id of the `Transaction`. - */ - readonly receiverId: string; - /** - * Returns the status of the `Transaction` as `ExecutionStatus`. - */ - readonly status: ExecutionStatus; - /** - * Returns the id of the `ExecutionOutcome` for the `Transaction`. - */ - readonly executionOutcomeId: string; - /** - * Returns an Array of `Operation` for the `Transaction`. - */ - readonly operations: Operation[]; - constructor( - /** - * Returns the hash of the `Transaction` in `CryptoHash`. - */ - transactionHash: string, - /** - * Returns the signer account id of the `Transaction`. - */ - signerId: string, - /** - * Returns the `PublicKey` of the signer of the `Transaction`. - */ - signerPublicKey: string, - /** - * Returns the `Signature` the `Transaction` was signed with. - */ - signature: string, - /** - * Returns the receiver account id of the `Transaction`. - */ - receiverId: string, - /** - * Returns the status of the `Transaction` as `ExecutionStatus`. - */ - status: ExecutionStatus, - /** - * Returns the id of the `ExecutionOutcome` for the `Transaction`. - */ - executionOutcomeId: string, - /** - * Returns an Array of `Operation` for the `Transaction`. - */ - operations: Operation[]); -} - -/** - * This structure is almost an identical copy of the `StateChangeWithCauseView` from [near-primitives](https://github.com/near/nearcore/tree/master/core/primitives) with a propagated additional field `affectedAccountId`. - */ -declare class StateChange { - /** - * Returns the `cause` of the `StateChange`. - */ - readonly cause: StateChangeCause; - /** - * Returns the `value` of the `StateChange`. - */ - readonly value: StateChangeValue; - constructor( - /** - * Returns the `cause` of the `StateChange`. - */ - cause: StateChangeCause, - /** - * Returns the `value` of the `StateChange`. - */ - value: StateChangeValue); - /** - * Returns the account id of the `StateChange`. - */ - get affectedAccountId(): string; - /** - * Returns the `StateChange` from the `StateChangeWithCauseView`. Created for backward compatibility. - */ - static fromStateChangeView(stateChangeView: StateChangeWithCauseView): StateChange; -} -type TransactionProcessingCause = { - txHash: string; -}; -type ActionReceiptProcessingStartedCause = { - receiptHash: string; -}; -type ActionReceiptGasRewardCause = { - receiptHash: string; -}; -type ReceiptProcessingCause = { - receiptHash: string; -}; -type PostponedReceiptCause = { - receiptHash: string; -}; -type StateChangeCause = 'NotWritableToDisk' | 'InitialState' | TransactionProcessingCause | ActionReceiptProcessingStartedCause | ActionReceiptGasRewardCause | ReceiptProcessingCause | PostponedReceiptCause | 'UpdatedDelayedReceipts' | 'ValidatorAccountsUpdate' | 'Migration' | 'Resharding'; -declare class AccountUpdateValue { - readonly accountId: string; - readonly account: Account; - constructor(accountId: string, account: Account); -} -declare class AccountDeletionValue { - readonly accountId: string; - constructor(accountId: string); -} -declare class AccountKeyUpdateValue { - readonly accountId: string; - readonly publicKey: string; - readonly accessKey: AccessKey; - constructor(accountId: string, publicKey: string, accessKey: AccessKey); -} -declare class AccessKeyDeletionValue { - readonly accountId: string; - readonly publicKey: string; - constructor(accountId: string, publicKey: string); -} -declare class DataUpdateValue { - readonly accountId: string; - readonly key: Uint8Array; - readonly value: Uint8Array; - constructor(accountId: string, key: Uint8Array, value: Uint8Array); -} -declare class DataDeletionValue { - readonly accountId: string; - readonly key: Uint8Array; - constructor(accountId: string, key: Uint8Array); -} -declare class ContractCodeUpdateValue { - readonly accountId: string; - readonly code: Uint8Array; - constructor(accountId: string, code: Uint8Array); -} -declare class ContractCodeDeletionValue { - readonly accountId: string; - constructor(accountId: string); -} -type StateChangeValue = AccountUpdateValue | AccountDeletionValue | AccountKeyUpdateValue | AccessKeyDeletionValue | DataUpdateValue | DataDeletionValue | ContractCodeUpdateValue | ContractCodeDeletionValue; -declare class Account { - readonly amount: number; - readonly locked: number; - readonly codeHash: string; - readonly storageUsage: number; - readonly storagePaidAt: number; - constructor(amount: number, locked: number, codeHash: string, storageUsage: number, storagePaidAt: number); -} - -/** - * The `Block` type is used to represent a block in the NEAR Lake Framework. - * - * **Important Notes on `Block`:** - * - All the entities located on different shards were merged into one single list without differentiation. - * - `Block` is not the fairest name for this structure either. NEAR Protocol is a sharded blockchain, so its block is actually an ephemeral structure that represents a collection of real blocks called chunks in NEAR Protocol. - */ -declare class Block { - /** - * Low-level structure for backward compatibility. - * As implemented in previous versions of [`near-lake-framework`](https://www.npmjs.com/package/near-lake-framework). - */ - readonly streamerMessage: StreamerMessage; - private executedReceipts; - /** - * Receipts included on the chain but not executed yet marked as “postponed”: they are represented by the same structure `Receipt` (see the corresponding section in this doc for more details). - */ - readonly postponedReceipts: Receipt[]; - /** - * List of included `Transactions`, converted into `Receipts`. - * - * **_NOTE_:** Heads up! You might want to know about `Transactions` to know where the action chain has begun. Unlike Ethereum, where a Transaction contains everything you may want to know about a particular interaction on the Ethereum blockchain, Near Protocol because of its asynchronous nature converts a `Transaction` into a `Receipt` before executing it. Thus, On NEAR, `Receipts` are more important for figuring out what happened on-chain as a result of a Transaction signed by a user. Read more about [Transactions on Near](https://nomicon.io/RuntimeSpec/Transactions) here. - * - */ - readonly transactions: Transaction[]; - private _actions; - private _events; - private _stateChanges; - constructor( - /** - * Low-level structure for backward compatibility. - * As implemented in previous versions of [`near-lake-framework`](https://www.npmjs.com/package/near-lake-framework). - */ - streamerMessage: StreamerMessage, executedReceipts: Receipt[], - /** - * Receipts included on the chain but not executed yet marked as “postponed”: they are represented by the same structure `Receipt` (see the corresponding section in this doc for more details). - */ - postponedReceipts: Receipt[], - /** - * List of included `Transactions`, converted into `Receipts`. - * - * **_NOTE_:** Heads up! You might want to know about `Transactions` to know where the action chain has begun. Unlike Ethereum, where a Transaction contains everything you may want to know about a particular interaction on the Ethereum blockchain, Near Protocol because of its asynchronous nature converts a `Transaction` into a `Receipt` before executing it. Thus, On NEAR, `Receipts` are more important for figuring out what happened on-chain as a result of a Transaction signed by a user. Read more about [Transactions on Near](https://nomicon.io/RuntimeSpec/Transactions) here. - * - */ - transactions: Transaction[], _actions: Map, _events: Map, _stateChanges: StateChange[]); - /** - * Returns the block hash. A shortcut to get the data from the block header. - */ - get blockHash(): string; - /** - * Returns the previous block hash. A shortcut to get the data from the block header. - */ - get prevBlockHash(): string; - /** - * Returns the block height. A shortcut to get the data from the block header. - */ - get blockHeight(): number; - /** - * Returns a `BlockHeader` structure of the block - * See `BlockHeader` structure sections for details. - */ - header(): BlockHeader; - /** - * Returns a slice of `Receipts` executed in the block. - * Basically is a getter for the `executedReceipts` field. - */ - receipts(): Receipt[]; - /** - * Returns an Array of `Actions` executed in the block. - */ - actions(): Action[]; - /** - * Returns `Events` emitted in the block. - */ - events(): Event[]; - /** - * Returns raw logs regardless of the fact that they are standard events or not. - */ - logs(): Log[]; - /** - * Returns an Array of `StateChange` occurred in the block. - */ - stateChanges(): StateChange[]; - /** - * Returns `Action` of the provided `receipt_id` from the block if any. Returns `undefined` if there is no corresponding `Action`. - * - * This method uses the internal `Block` `action` field which is empty by default and will be filled with the block’s actions on the first call to optimize memory usage. - * - * The result is either `Action | undefined` since there might be a request for an `Action` by `receipt_id` from another block, in which case this method will be unable to find the `Action` in the current block. In the other case, the request might be for an `Action` for a `receipt_id` that belongs to a `DataReceipt` where an action does not exist. - */ - actionByReceiptId(receipt_id: string): Action | undefined; - /** - * Returns an Array of Events emitted by `ExecutionOutcome` for the given `receipt_id`. There might be more than one `Event` for the `Receipt` or there might be none of them. In the latter case, this method returns an empty Array. - */ - eventsByReceiptId(receipt_id: string): Event[]; - /** - * Returns an Array of Events emitted by `ExecutionOutcome` for the given `account_id`. There might be more than one `Event` for the `Receipt` or there might be none of them. In the latter case, this method returns an empty Array. - */ - eventsByAccountId(account_id: string): Event[]; - private buildActionsHashmap; - private buildEventsHashmap; - static fromStreamerMessage(streamerMessage: StreamerMessage): Block; -} -/** - * Replacement for `BlockHeaderView` from [near-primitives](https://github.com/near/nearcore/tree/master/core/primitives). Shrunken and simplified. - * - * **Note:** the original `BlockHeaderView` is still accessible via the `.streamerMessage` attribute. - */ -declare class BlockHeader { - readonly height: number; - readonly hash: string; - readonly prevHash: string; - readonly author: string; - readonly timestampNanosec: string; - readonly epochId: string; - readonly nextEpochId: string; - readonly gasPrice: string; - readonly totalSupply: string; - readonly latestProtocolVersion: number; - readonly randomValue: string; - readonly chunksIncluded: number; - readonly validatorProposals: ValidatorStakeView[]; - constructor(height: number, hash: string, prevHash: string, author: string, timestampNanosec: string, epochId: string, nextEpochId: string, gasPrice: string, totalSupply: string, latestProtocolVersion: number, randomValue: string, chunksIncluded: number, validatorProposals: ValidatorStakeView[]); - static fromStreamerMessage(streamerMessage: StreamerMessage): BlockHeader; -} - -declare const fromBorsh: (schema: borsh.Schema, encoded: Uint8Array) => borsh_lib_types_types.DecodeTypes; - -const fromBorsh$1 = /*#__PURE__*/_mergeNamespaces({ - __proto__: null, - fromBorsh: fromBorsh -}, [borsher]) as { fromBorsh: typeof fromBorsh }; - -export { Block, type BlockHeaderView, type BlockHeight, type BlockView, Event, LakeContext, Receipt, type Shard, StateChange, type StreamerMessage, Transaction, fromBorsh$1 as borsh }; diff --git a/frontend/src/classes/ValidationError.js b/frontend/src/classes/ValidationError.js deleted file mode 100644 index da308a9a2..000000000 --- a/frontend/src/classes/ValidationError.js +++ /dev/null @@ -1,6 +0,0 @@ -export class ValidationError extends Error { - constructor(message, type) { - super(message); - this.type = type; - } -} diff --git a/frontend/src/classes/ValidationError.ts b/frontend/src/classes/ValidationError.ts new file mode 100644 index 000000000..96a24ecbd --- /dev/null +++ b/frontend/src/classes/ValidationError.ts @@ -0,0 +1,17 @@ +export class ValidationError extends Error { + type: string; + location?: { + start: { line: number; column: number }; + end: { line: number; column: number }; + }; + + constructor( + message: string, + type: string, + location?: { start: { line: number; column: number }; end: { line: number; column: number } }, + ) { + super(message); + this.type = type; + this.location = location; + } +} diff --git a/frontend/src/components/Common/Alert.tsx b/frontend/src/components/Common/Alert.tsx new file mode 100644 index 000000000..efeb7e457 --- /dev/null +++ b/frontend/src/components/Common/Alert.tsx @@ -0,0 +1,55 @@ +import React, { useState } from 'react'; + +type AlertProps = { + type: 'success' | 'error' | 'info'; + message: string; + onClose?: () => void; +}; + +const Alert: React.FC = ({ type, message, onClose }) => { + const [closed, setClosed] = useState(false); + + const handleClose = () => { + setClosed(true); + if (onClose) { + onClose(); + } + }; + + if (closed) { + return null; + } + + return ( +
+ Alert: + {message} + + + Close + + + +
+ ); +}; + +export default Alert; diff --git a/frontend/src/components/Logs/LogsView/Icons/CheckMarkIcon.js b/frontend/src/components/Common/Icons/CheckMarkIcon.js similarity index 100% rename from frontend/src/components/Logs/LogsView/Icons/CheckMarkIcon.js rename to frontend/src/components/Common/Icons/CheckMarkIcon.js diff --git a/frontend/src/components/Logs/LogsView/Icons/ClearIcon.js b/frontend/src/components/Common/Icons/ClearIcon.js similarity index 100% rename from frontend/src/components/Logs/LogsView/Icons/ClearIcon.js rename to frontend/src/components/Common/Icons/ClearIcon.js diff --git a/frontend/src/components/Common/LatestBlock.tsx b/frontend/src/components/Common/LatestBlock.tsx index b03394f3e..46363a6bb 100644 --- a/frontend/src/components/Common/LatestBlock.tsx +++ b/frontend/src/components/Common/LatestBlock.tsx @@ -1,4 +1,5 @@ -import React, { useState, useEffect } from 'react'; +import React, { useEffect, useState } from 'react'; + import { calculateBlockTimeDifference } from '@/utils/calculateBlockTimeDifference'; interface LatestBlockProps { diff --git a/frontend/src/components/CreateNewIndexer/CreateNewIndexer.js b/frontend/src/components/CreateNewIndexer/CreateNewIndexer.js index 0d28d02e7..d3f2b9daa 100644 --- a/frontend/src/components/CreateNewIndexer/CreateNewIndexer.js +++ b/frontend/src/components/CreateNewIndexer/CreateNewIndexer.js @@ -1,4 +1,4 @@ -import Editor from '@/components/Editor'; +import Editor from '@/components/Editor/EditorComponents/Editor'; const CreateNewIndexer = () => { return ; diff --git a/frontend/src/components/Editor/DiffEditorComponent.jsx b/frontend/src/components/Editor/EditorComponents/DiffEditorComponent.jsx similarity index 100% rename from frontend/src/components/Editor/DiffEditorComponent.jsx rename to frontend/src/components/Editor/EditorComponents/DiffEditorComponent.jsx diff --git a/frontend/src/components/Editor/Editor.jsx b/frontend/src/components/Editor/EditorComponents/Editor.tsx similarity index 50% rename from frontend/src/components/Editor/Editor.jsx rename to frontend/src/components/Editor/EditorComponents/Editor.tsx index 8a4152a56..836eea02a 100644 --- a/frontend/src/components/Editor/Editor.jsx +++ b/frontend/src/components/Editor/EditorComponents/Editor.tsx @@ -1,82 +1,78 @@ -import React, { useEffect, useState, useRef, useMemo, useContext } from 'react'; -import { - formatSQL, - formatIndexingCode, - wrapCode, - defaultCode, - defaultSchema, - defaultSchemaTypes, -} from '../../utils/formatters'; -import { Alert } from 'react-bootstrap'; -import { queryIndexerFunctionDetails } from '../../utils/queryIndexerFunction'; - -import primitives from '!!raw-loader!../../../primitives.d.ts'; import { request, useInitialPayload } from 'near-social-bridge'; -import IndexerRunner from '../../utils/indexerRunner'; -import { block_details } from './block_details'; -import ResizableLayoutEditor from './ResizableLayoutEditor'; -import { ResetChangesModal } from '../Modals/resetChanges'; -import { FileSwitcher } from './FileSwitcher'; -import EditorMenuContainer from './EditorViewContainer/EditorMenuContainer'; -import DeveloperToolsContainer from './EditorViewContainer/DeveloperToolsContainer'; - -import { PublishModal } from '../Modals/PublishModal'; -import { ForkIndexerModal } from '../Modals/ForkIndexerModal'; -import { getLatestBlockHeight } from '../../utils/getLatestBlockHeight'; -import { IndexerDetailsContext } from '../../contexts/IndexerDetailsContext'; -import { PgSchemaTypeGen } from '../../utils/pgSchemaTypeGen'; -import { validateJSCode, validateSQLSchema } from '@/utils/validators'; +import React, { useContext, useEffect, useMemo, useRef, useState } from 'react'; +import type { ReactElement } from 'react'; +import { Alert } from 'react-bootstrap'; import { useDebouncedCallback } from 'use-debounce'; + +import primitives from '!!raw-loader!./primitives.d.ts'; import { - CODE_GENERAL_ERROR_MESSAGE, CODE_FORMATTING_ERROR_MESSAGE, - SCHEMA_TYPE_GENERATION_ERROR_MESSAGE, - SCHEMA_FORMATTING_ERROR_MESSAGE, + CODE_GENERAL_ERROR_MESSAGE, FORMATTING_ERROR_TYPE, + SCHEMA_FORMATTING_ERROR_MESSAGE, + SCHEMA_TYPE_GENERATION_ERROR_MESSAGE, TYPE_GENERATION_ERROR_TYPE, - INDEXER_REGISTER_TYPE_GENERATION_ERROR, -} from '../../constants/Strings'; -import { InfoModal } from '@/core/InfoModal'; +} from '@/constants/Strings'; +import { IndexerDetailsContext } from '@/contexts/IndexerDetailsContext'; import { useModal } from '@/contexts/ModalContext'; +import { InfoModal } from '@/core/InfoModal'; +import { defaultCode, defaultSchema, defaultSchemaTypes, formatIndexingCode, formatSQL } from '@/utils/formatters'; +import { getLatestBlockHeight } from '@/utils/getLatestBlockHeight'; +import IndexerRunner from '@/utils/indexerRunner'; +import { PgSchemaTypeGen } from '@/utils/pgSchemaTypeGen'; +import { validateJSCode, validateSQLSchema } from '@/utils/validators'; + +import DeveloperToolsContainer from '../EditorViewContainer/DeveloperToolsContainer'; +import EditorMenuContainer from '../EditorViewContainer/EditorMenuContainer'; +import QueryAPIStorageManager from '../QueryApiStorageManager'; +import { block_details } from './block_details'; +import { FileSwitcher } from './FileSwitcher'; import { GlyphContainer } from './GlyphContainer'; +import ResizableLayoutEditor from './ResizableLayoutEditor'; + +const INDEXER_TAB_NAME = 'indexer.js'; +const SCHEMA_TAB_NAME = 'schema.sql'; +declare const monaco: any; -const Editor = ({ actionButtonText }) => { - const { indexerDetails, setShowResetCodeModel, setShowPublishModal, debugMode, isCreateNewIndexer, setAccountId } = - useContext(IndexerDetailsContext); - - const DEBUG_LIST_STORAGE_KEY = `QueryAPI:debugList:${indexerDetails.accountId}#${ - indexerDetails.indexerName || 'new' - }`; - const SCHEMA_STORAGE_KEY = `QueryAPI:Schema:${indexerDetails.accountId}#${indexerDetails.indexerName || 'new'}`; - const SCHEMA_TYPES_STORAGE_KEY = `QueryAPI:Schema:Types:${indexerDetails.accountId}#${ - indexerDetails.indexerName || 'new' - }`; - const CODE_STORAGE_KEY = `QueryAPI:Code:${indexerDetails.accountId}#${indexerDetails.indexerName || 'new'}`; - const SCHEMA_TAB_NAME = 'schema.sql'; - const [blockHeightError, setBlockHeightError] = useState(undefined); - const [error, setError] = useState(); - - const [fileName, setFileName] = useState('indexingLogic.js'); - - const [originalSQLCode, setOriginalSQLCode] = useState(formatSQL(defaultSchema)); - const [originalIndexingCode, setOriginalIndexingCode] = useState(formatIndexingCode(defaultCode)); - const [indexingCode, setIndexingCode] = useState(originalIndexingCode); - const [schema, setSchema] = useState(originalSQLCode); - const [schemaTypes, setSchemaTypes] = useState(defaultSchemaTypes); - const [monacoMount, setMonacoMount] = useState(false); - - const [heights, setHeights] = useState(localStorage.getItem(DEBUG_LIST_STORAGE_KEY) || []); - - const [debugModeInfoDisabled, setDebugModeInfoDisabled] = useState(false); - const [diffView, setDiffView] = useState(false); - const [blockView, setBlockView] = useState(false); +const Editor: React.FC = (): ReactElement => { + const { indexerDetails, debugMode, isCreateNewIndexer } = useContext(IndexerDetailsContext); + const storageManager = useMemo(() => { + if (indexerDetails.accountId && indexerDetails.indexerName) { + return new QueryAPIStorageManager(indexerDetails.accountId, indexerDetails.indexerName); + } else return null; + }, [indexerDetails.accountId, indexerDetails.indexerName]); + + const [error, setError] = useState(); + const [fileName, setFileName] = useState(INDEXER_TAB_NAME); + + const [originalSQLCode, setOriginalSQLCode] = useState(formatSQL(defaultSchema)); + const [originalIndexingCode, setOriginalIndexingCode] = useState(formatIndexingCode(defaultCode)); + + const [indexingCode, setIndexingCode] = useState(originalIndexingCode); + const [schema, setSchema] = useState(originalSQLCode); + const [cursorPostion, setCursorPosition] = useState<{ lineNumber: number; column: number }>({ + lineNumber: 1, + column: 1, + }); + + const [schemaTypes, setSchemaTypes] = useState(defaultSchemaTypes); + const [monacoMount, setMonacoMount] = useState(false); + + const initialHeights = storageManager ? storageManager.getDebugList() || [] : []; + const [heights, setHeights] = useState(initialHeights); + + const [debugModeInfoDisabled, setDebugModeInfoDisabled] = useState(false); + const [diffView, setDiffView] = useState(false); + const [blockView, setBlockView] = useState(false); const { openModal, showModal, data, message, hideModal } = useModal(); - const [isExecutingIndexerFunction, setIsExecutingIndexerFunction] = useState(false); - const { height, currentUserAccountId } = useInitialPayload(); + const [isExecutingIndexerFunction, setIsExecutingIndexerFunction] = useState(false); + const { height, currentUserAccountId }: { height?: number; currentUserAccountId?: string } = + useInitialPayload() || {}; + + const [decorations, setDecorations] = useState([]); - const [decorations, setDecorations] = useState([]); - const handleLog = (_, log, callback) => { + const handleLog = (_: any, log: string, callback: () => void) => { if (log) console.log(log); if (callback) { callback(); @@ -85,10 +81,13 @@ const Editor = ({ actionButtonText }) => { const indexerRunner = useMemo(() => new IndexerRunner(handleLog), []); const pgSchemaTypeGen = new PgSchemaTypeGen(); - const disposableRef = useRef(null); - const monacoEditorRef = useRef(null); + const disposableRef = useRef(null); + const monacoEditorRef = useRef(null); - const parseGlyphError = (error, line) => { + const parseGlyphError = ( + error?: { message: string }, + line?: { start: { line: number; column: number }; end: { line: number; column: number } }, + ) => { const { line: startLine, column: startColumn } = line?.start || { line: 1, column: 1 }; const { line: endLine, column: endColumn } = line?.end || { line: 1, column: 1 }; const displayedError = error?.message || 'No Errors'; @@ -97,7 +96,6 @@ const Editor = ({ actionButtonText }) => { [decorations], [ { - // eslint-disable-next-line no-undef range: new monaco.Range(startLine, startColumn, endLine, endColumn), options: { isWholeLine: true, @@ -109,15 +107,15 @@ const Editor = ({ actionButtonText }) => { ); }; - const debouncedValidateSQLSchema = useDebouncedCallback((_schema) => { + const debouncedValidateSQLSchema = useDebouncedCallback((_schema: string) => { const { error, location } = validateSQLSchema(_schema); - error ? parseGlyphError(error, location) : parseGlyphError(); + error ? parseGlyphError(error as any, location as any) : parseGlyphError(); return; }, 500); - const debouncedValidateCode = useDebouncedCallback((_code) => { + const debouncedValidateCode = useDebouncedCallback((_code: string) => { const { error: codeError } = validateJSCode(_code); - codeError ? setError(CODE_FORMATTING_ERROR_MESSAGE) : setError(); + codeError ? setError(CODE_FORMATTING_ERROR_MESSAGE) : setError(undefined); }, 500); useEffect(() => { @@ -128,8 +126,10 @@ const Editor = ({ actionButtonText }) => { setError(CODE_FORMATTING_ERROR_MESSAGE); } - setOriginalIndexingCode(formattedCode); - setIndexingCode(formattedCode); + if (formattedCode) { + setOriginalIndexingCode(formattedCode); + setIndexingCode(formattedCode); + } } }, [indexerDetails.code]); @@ -143,12 +143,12 @@ const Editor = ({ actionButtonText }) => { setError(SCHEMA_TYPE_GENERATION_ERROR_MESSAGE); } - setSchema(formattedSchema); + formattedSchema && setSchema(formattedSchema); } }, [indexerDetails.schema]); useEffect(() => { - const { error: schemaError, location } = validateSQLSchema(schema); + const { error: schemaError } = validateSQLSchema(schema); const { error: codeError } = validateJSCode(indexingCode); if (schemaError?.type === FORMATTING_ERROR_TYPE) { @@ -157,47 +157,44 @@ const Editor = ({ actionButtonText }) => { setError(SCHEMA_TYPE_GENERATION_ERROR_MESSAGE); } else if (codeError) setError(CODE_GENERAL_ERROR_MESSAGE); else { - setError(); + setError(undefined); handleCodeGen(); } if (fileName === SCHEMA_TAB_NAME) debouncedValidateSQLSchema(schema); }, [fileName]); useEffect(() => { - const savedSchema = localStorage.getItem(SCHEMA_STORAGE_KEY); - const savedCode = localStorage.getItem(CODE_STORAGE_KEY); - - if (savedSchema) { - setSchema(savedSchema); - } + const savedCode = storageManager?.getIndexerCode(); + const savedSchema = storageManager?.getSchemaCode(); + if (savedSchema) setSchema(savedSchema); if (savedCode) setIndexingCode(savedCode); }, [indexerDetails.accountId, indexerDetails.indexerName]); useEffect(() => { - localStorage.setItem(SCHEMA_STORAGE_KEY, schema); - localStorage.setItem(CODE_STORAGE_KEY, indexingCode); - }, [schema, indexingCode]); + storageManager?.setSchemaCode(schema); + storageManager?.setIndexerCode(indexingCode); + storageManager?.setCursorPosition(cursorPostion); + }, [schema, indexingCode, cursorPostion]); useEffect(() => { - localStorage.setItem(SCHEMA_TYPES_STORAGE_KEY, schemaTypes); + storageManager?.setSchemaTypes(schemaTypes); handleCodeGen(); }, [schemaTypes, monacoMount]); useEffect(() => { - localStorage.setItem(DEBUG_LIST_STORAGE_KEY, heights); + storageManager?.setDebugList(heights); }, [heights]); const attachTypesToMonaco = () => { - // If types has been added already, dispose of them first + // If types have been added already, dispose of them first if (disposableRef.current) { disposableRef.current.dispose(); disposableRef.current = null; } - if (window.monaco) { + if ((window as any).monaco) { // Add generated types to monaco and store disposable to clear them later - // eslint-disable-next-line no-undef - const newDisposable = monaco.languages.typescript.typescriptDefaults.addExtraLib(schemaTypes); + const newDisposable = (window as any).monaco.languages.typescript.typescriptDefaults.addExtraLib(schemaTypes); if (newDisposable != null) { console.log('Types successfully imported to Editor'); } @@ -206,68 +203,6 @@ const Editor = ({ actionButtonText }) => { } }; - const forkIndexer = async (indexerName) => { - let code = indexingCode; - setAccountId(currentUserAccountId); - let prevAccountId = indexerDetails.accountId.replaceAll('.', '_'); - let newAccountId = currentUserAccountId.replaceAll('.', '_'); - let prevIndexerName = indexerDetails.indexerName.replaceAll('-', '_').trim().toLowerCase(); - let newIndexerName = indexerName.replaceAll('-', '_').trim().toLowerCase(); - code = code.replaceAll(prevAccountId, newAccountId); - code = code.replaceAll(prevIndexerName, newIndexerName); - setIndexingCode(formatIndexingCode(code)); - }; - - const registerFunction = async (indexerName, indexerConfig) => { - const { data: validatedSchema, error: schemaValidationError } = validateSQLSchema(schema); - const { data: validatedCode, error: codeValidationError } = validateJSCode(indexingCode); - - if (codeValidationError) { - setError(CODE_FORMATTING_ERROR_MESSAGE); - return; - } - - let innerCode = validatedCode.match(/getBlock\s*\([^)]*\)\s*{([\s\S]*)}/)[1]; - indexerName = indexerName.replaceAll(' ', '_'); - let forkedFrom = - indexerDetails.forkedAccountId && indexerDetails.forkedIndexerName - ? { account_id: indexerDetails.forkedAccountId, function_name: indexerDetails.forkedIndexerName } - : null; - - const startBlock = - indexerConfig.startBlock === 'startBlockHeight' - ? { HEIGHT: indexerConfig.height } - : indexerConfig.startBlock === 'startBlockLatest' - ? 'LATEST' - : 'CONTINUE'; - - if (schemaValidationError?.type === FORMATTING_ERROR_TYPE) { - setError(SCHEMA_FORMATTING_ERROR_MESSAGE); - return; - } else if (schemaValidationError?.type === TYPE_GENERATION_ERROR_TYPE) { - showModal(INDEXER_REGISTER_TYPE_GENERATION_ERROR, { - indexerName, - code: innerCode, - schema: validatedSchema, - startBlock, - contractFilter: indexerConfig.filter, - forkedFrom, - }); - return; - } - - request('register-function', { - indexerName: indexerName, - code: innerCode, - schema: validatedSchema, - startBlock, - contractFilter: indexerConfig.filter, - ...(forkedFrom && { forkedFrom }), - }); - - setShowPublishModal(false); - }; - const handleDeleteIndexer = () => { request('delete-indexer', { accountId: indexerDetails.accountId, @@ -275,53 +210,12 @@ const Editor = ({ actionButtonText }) => { }); }; - const handleReload = async () => { - if (isCreateNewIndexer) { - setShowResetCodeModel(false); - setIndexingCode(originalIndexingCode); - setSchema(originalSQLCode); - setSchemaTypes(defaultSchemaTypes); - return; - } - - const data = await queryIndexerFunctionDetails(indexerDetails.accountId, indexerDetails.indexerName); - if (data == null) { - setIndexingCode(defaultCode); - setSchema(defaultSchema); - setSchemaTypes(defaultSchemaTypes); - } else { - try { - let unformatted_wrapped_indexing_code = wrapCode(data.code); - let unformatted_schema = data.schema; - if (unformatted_wrapped_indexing_code !== null) { - setOriginalIndexingCode(() => unformatted_wrapped_indexing_code); - setIndexingCode(() => unformatted_wrapped_indexing_code); - } - if (unformatted_schema !== null) { - setOriginalSQLCode(unformatted_schema); - setSchema(unformatted_schema); - } - - const { formattedCode, formattedSchema } = reformatAll(unformatted_wrapped_indexing_code, unformatted_schema); - setIndexingCode(formattedCode); - setSchema(formattedSchema); - } catch (formattingError) { - console.log(formattingError); - } - } - setShowResetCodeModel(false); - }; - - const getActionButtonText = () => { - const isUserIndexer = indexerDetails.accountId === currentUserAccountId; - if (isCreateNewIndexer) return 'Create New Indexer'; - return isUserIndexer ? actionButtonText : 'Fork Indexer'; - }; - - const reformatAll = (indexingCode, schema) => { - let { data: formattedCode, error: codeError } = validateJSCode(indexingCode); - let { data: formattedSchema, error: schemaError } = validateSQLSchema(schema); + const reformatAll = (indexingCode: string, schema: string) => { + const { data: validatedCode, error: codeError } = validateJSCode(indexingCode); + const { data: validatedSchema, error: schemaError } = validateSQLSchema(schema); + let formattedCode = validatedCode; + let formattedSchema = validatedSchema; if (codeError) { formattedCode = indexingCode; setError(CODE_FORMATTING_ERROR_MESSAGE); @@ -332,13 +226,13 @@ const Editor = ({ actionButtonText }) => { formattedSchema = schema; setError(SCHEMA_TYPE_GENERATION_ERROR_MESSAGE); } else { - setError(); + setError(undefined); } return { formattedCode, formattedSchema }; }; - function handleCodeGen() { + const handleCodeGen = () => { try { setSchemaTypes(pgSchemaTypeGen.generateTypes(schema)); attachTypesToMonaco(); // Just in case schema types have been updated but weren't added to monaco @@ -346,15 +240,15 @@ const Editor = ({ actionButtonText }) => { console.error('Error generating types for saved schema.\n', _error); setError(SCHEMA_TYPE_GENERATION_ERROR_MESSAGE); } - } + }; - function handleFormating() { + const handleFormating = () => { const { formattedCode, formattedSchema } = reformatAll(indexingCode, schema); - setIndexingCode(formattedCode); - setSchema(formattedSchema); - } + formattedCode && setIndexingCode(formattedCode); + formattedSchema && setSchema(formattedSchema); + }; - function handleEditorWillMount(editor, monaco) { + const handleEditorWillMount = (editor: any, monaco: any) => { if (!diffView) { const decorations = editor.deltaDecorations( [], @@ -373,15 +267,18 @@ const Editor = ({ actionButtonText }) => { 'file:///node_modules/@near-lake/primitives/index.d.ts', ); setMonacoMount(true); - } + }; - async function executeIndexerFunction(option = 'latest', startingBlockHeight = null) { + const executeIndexerFunction = async (option = 'latest', startingBlockHeight: number | null = null) => { setIsExecutingIndexerFunction(() => true); - const schemaName = indexerDetails.accountId.concat('_', indexerDetails.indexerName).replace(/[^a-zA-Z0-9]/g, '_'); + const accountId = indexerDetails?.accountId ?? ''; + const indexerName = indexerDetails?.indexerName ?? ''; + const schemaName = accountId.concat('_', indexerName).replace(/[^a-zA-Z0-9]/g, '_'); + let latestHeight; switch (option) { case 'debugList': - await indexerRunner.executeIndexerFunctionOnHeights(heights, indexingCode, schema, schemaName, option); + await indexerRunner.executeIndexerFunctionOnHeights(heights, indexingCode, schema, schemaName); break; case 'specific': if (startingBlockHeight === null && Number(startingBlockHeight) === 0) { @@ -396,21 +293,21 @@ const Editor = ({ actionButtonText }) => { if (latestHeight) await indexerRunner.start(latestHeight - 10, indexingCode, schema, schemaName, option); } setIsExecutingIndexerFunction(() => false); - } + }; - function handleOnChangeSchema(_schema) { + const handleOnChangeSchema = (_schema: string) => { setSchema(_schema); debouncedValidateSQLSchema(_schema); - } + }; - function handleOnChangeCode(_code) { + const handleOnChangeCode = (_code: string) => { setIndexingCode(_code); debouncedValidateCode(_code); - } + }; - function handleRegisterIndexerWithErrors(args) { + const handleRegisterIndexerWithErrors = (args: any) => { request('register-function', args); - } + }; return ( <> @@ -433,46 +330,37 @@ const Editor = ({ actionButtonText }) => { {(indexerDetails.code || isCreateNewIndexer) && ( <> indexerRunner.stop()} - latestHeight={height} isUserIndexer={indexerDetails.accountId === currentUserAccountId} handleDeleteIndexer={handleDeleteIndexer} + isCreateNewIndexer={isCreateNewIndexer} + error={error} + //Fork Indexer Modal + indexingCode={indexingCode} + setIndexingCode={setIndexingCode} + currentUserAccountId={currentUserAccountId} + //Reset Indexer Modal + setSchema={setSchema} + setSchemaTypes={setSchemaTypes} + setOriginalIndexingCode={setOriginalIndexingCode} + setOriginalSQLCode={setOriginalSQLCode} + //Publish Modal + actionButtonText={'publish'} + schema={schema} + setError={setError} + showModal={showModal} /> indexerRunner.stop()} latestHeight={height} - isUserIndexer={indexerDetails.accountId === currentUserAccountId} - handleDeleteIndexer={handleDeleteIndexer} - fileName={fileName} - setFileName={setFileName} diffView={diffView} setDiffView={setDiffView} /> - - - - -
{ > {error && ( setError()} + dismissible={true} + onClose={() => setError(undefined)} className="px-4 py-3 mb-4 font-semibold text-red-700 text-sm text-center border border-red-300 bg-red-50 rounded-lg shadow-md" variant="danger" > @@ -495,20 +383,16 @@ const Editor = ({ actionButtonText }) => { {debugMode && !debugModeInfoDisabled && ( setDebugModeInfoDisabled(true)} variant="info" > To debug, you will need to open your browser console window in order to see the logs. )} - + + {/* @ts-ignore remove after refactoring Resizable Editor to ts*/} diff --git a/frontend/src/components/Editor/GlyphContainer.js b/frontend/src/components/Editor/EditorComponents/GlyphContainer.js similarity index 100% rename from frontend/src/components/Editor/GlyphContainer.js rename to frontend/src/components/Editor/EditorComponents/GlyphContainer.js diff --git a/frontend/src/components/Editor/MonacoEditorComponent.jsx b/frontend/src/components/Editor/EditorComponents/MonacoEditorComponent.jsx similarity index 100% rename from frontend/src/components/Editor/MonacoEditorComponent.jsx rename to frontend/src/components/Editor/EditorComponents/MonacoEditorComponent.jsx diff --git a/frontend/src/components/Editor/ResizableLayoutEditor.jsx b/frontend/src/components/Editor/EditorComponents/ResizableLayoutEditor.jsx similarity index 98% rename from frontend/src/components/Editor/ResizableLayoutEditor.jsx rename to frontend/src/components/Editor/EditorComponents/ResizableLayoutEditor.jsx index ed3e5be74..ef31508e2 100644 --- a/frontend/src/components/Editor/ResizableLayoutEditor.jsx +++ b/frontend/src/components/Editor/EditorComponents/ResizableLayoutEditor.jsx @@ -2,7 +2,7 @@ import { DiffEditorComponent } from './DiffEditorComponent'; import { MonacoEditorComponent } from './MonacoEditorComponent'; import { defaultCode, defaultSchema } from '@/utils/formatters'; import { useDragResize } from '@/utils/resize'; -import GraphqlPlayground from './../Playground'; +import GraphqlPlayground from '../../Playground'; const containerStyle = { display: 'flex', @@ -51,7 +51,7 @@ const ResizableEditor = ({ // Render logic based on fileName const editorComponents = { GraphiQL: () => , - 'indexingLogic.js': () => + 'indexer.js': () => diffView ? ( Event; +} +/** + * This structure is a copy of the [JSON Events](https://github.com/near/NEPs/blob/master/neps/nep-0297.md) structure representation. + */ +declare class RawEvent { + readonly event: string; + readonly standard: string; + readonly version: string; + readonly data: JSON | undefined; + constructor(event: string, standard: string, version: string, data: JSON | undefined); + static isEvent: (log: string) => boolean; + static fromLog: (log: string) => RawEvent; +} +type Events = { + events: Event[]; +}; + +/** + * This field is a simplified representation of the `ReceiptView` structure from [near-primitives](https://github.com/near/nearcore/tree/master/core/primitives). + */ +declare class Receipt implements Events { + /** + * Defined the type of the `Receipt`: `Action` or `Data` representing the `ActionReceipt` and `DataReceipt`. + */ + readonly receiptKind: ReceiptKind; + /** + * The ID of the `Receipt` of the `CryptoHash` type. + */ + readonly receiptId: string; + /** + * The receiver account id of the `Receipt`. + */ + readonly receiverId: string; + /** + * The predecessor account id of the `Receipt`. + */ + readonly predecessorId: string; + /** + * Represents the status of `ExecutionOutcome` of the `Receipt`. + */ + readonly status: ExecutionStatus; + /** + * The id of the `ExecutionOutcome` for the `Receipt`. Returns `null` if the `Receipt` isn’t executed yet and has a postponed status. + */ + readonly executionOutcomeId?: string | undefined; + /** + * The original logs of the corresponding `ExecutionOutcome` of the `Receipt`. + * + * **Note:** not all of the logs might be parsed as JSON Events (`Events`). + */ + readonly logs: string[]; + constructor( + /** + * Defined the type of the `Receipt`: `Action` or `Data` representing the `ActionReceipt` and `DataReceipt`. + */ + receiptKind: ReceiptKind, + /** + * The ID of the `Receipt` of the `CryptoHash` type. + */ + receiptId: string, + /** + * The receiver account id of the `Receipt`. + */ + receiverId: string, + /** + * The predecessor account id of the `Receipt`. + */ + predecessorId: string, + /** + * Represents the status of `ExecutionOutcome` of the `Receipt`. + */ + status: ExecutionStatus, + /** + * The id of the `ExecutionOutcome` for the `Receipt`. Returns `null` if the `Receipt` isn’t executed yet and has a postponed status. + */ + executionOutcomeId?: string | undefined, + /** + * The original logs of the corresponding `ExecutionOutcome` of the `Receipt`. + * + * **Note:** not all of the logs might be parsed as JSON Events (`Events`). + */ + logs?: string[], + ); + /** + * Returns an Array of `Events` for the `Receipt`, if any. This might be empty if the `logs` field is empty or doesn’t contain JSON Events compatible log records. + */ + get events(): Event[]; + static fromOutcomeWithReceipt: (outcomeWithReceipt: ExecutionOutcomeWithReceipt) => Receipt; +} +/** + * `ReceiptKind` a simple `enum` to represent the `Receipt` type: either `Action` or `Data`. + */ +declare enum ReceiptKind { + Action = 'Action', + Data = 'Data', +} +/** + * `Action` is the structure with the fields and data relevant to an `ActionReceipt`. + * + * Basically, `Action` is the structure that indexer developers will be encouraged to work the most in their action-oriented indexers. + */ +declare class Action { + /** + * The id of the corresponding `Receipt` + */ + readonly receiptId: string; + /** + * The predecessor account id of the corresponding `Receipt`. + * This field is a piece of denormalization of the structures (`Receipt` and `Action`). + */ + readonly predecessorId: string; + /** + * The receiver account id of the corresponding `Receipt`. + * This field is a piece of denormalization of the structures (`Receipt` and `Action`). + */ + readonly receiverId: string; + /** + * The signer account id of the corresponding `Receipt` + */ + readonly signerId: string; + /** + * The signer’s PublicKey for the corresponding `Receipt` + */ + readonly signerPublicKey: string; + /** + * An array of `Operation` for this `ActionReceipt` + */ + readonly operations: Operation[]; + constructor( + /** + * The id of the corresponding `Receipt` + */ + receiptId: string, + /** + * The predecessor account id of the corresponding `Receipt`. + * This field is a piece of denormalization of the structures (`Receipt` and `Action`). + */ + predecessorId: string, + /** + * The receiver account id of the corresponding `Receipt`. + * This field is a piece of denormalization of the structures (`Receipt` and `Action`). + */ + receiverId: string, + /** + * The signer account id of the corresponding `Receipt` + */ + signerId: string, + /** + * The signer’s PublicKey for the corresponding `Receipt` + */ + signerPublicKey: string, + /** + * An array of `Operation` for this `ActionReceipt` + */ + operations: Operation[], + ); + static isActionReceipt: (receipt: ReceiptView) => boolean; + static fromReceiptView: (receipt: ReceiptView) => Action | null; +} +declare class DeployContract { + readonly code: Uint8Array; + constructor(code: Uint8Array); +} +declare class FunctionCall { + readonly methodName: string; + readonly args: Uint8Array; + readonly gas: number; + readonly deposit: string; + constructor(methodName: string, args: Uint8Array, gas: number, deposit: string); +} +declare class Transfer { + readonly deposit: string; + constructor(deposit: string); +} +declare class Stake { + readonly stake: number; + readonly publicKey: string; + constructor(stake: number, publicKey: string); +} +declare class AddKey { + readonly publicKey: string; + readonly accessKey: AccessKey; + constructor(publicKey: string, accessKey: AccessKey); +} +declare class DeleteKey { + readonly publicKey: string; + constructor(publicKey: string); +} +declare class DeleteAccount { + readonly beneficiaryId: string; + constructor(beneficiaryId: string); +} +/** + * A representation of the original `ActionView` from [near-primitives](https://github.com/near/nearcore/tree/master/core/primitives). + */ +type Operation = + | 'CreateAccount' + | DeployContract + | FunctionCall + | Transfer + | Stake + | AddKey + | DeleteKey + | DeleteAccount; +declare class AccessKey { + readonly nonce: number; + readonly permission: string | AccessKeyFunctionCallPermission; + constructor(nonce: number, permission: string | AccessKeyFunctionCallPermission); +} +declare class AccessKeyFunctionCallPermission { + readonly allowance: string; + readonly receiverId: string; + readonly methodNames: string[]; + constructor(allowance: string, receiverId: string, methodNames: string[]); +} + +/** + * A representation of the `IndexerTransactionWithOutcome` from `near-indexer-primitives` which is an ephemeral structure combining `SignedTransactionView` from [near-primitives](https://github.com/near/nearcore/tree/master/core/primitives) and `IndexerExecutionOutcomeWithOptionalReceipt` from `near-indexer-primitives`. + * + * This structure is very similar to `Receipt`. Unlike `Receipt`, a `Transaction` has a few additional fields like `signerId`, `signature`, and `operations`. + */ +declare class Transaction { + /** + * Returns the hash of the `Transaction` in `CryptoHash`. + */ + readonly transactionHash: string; + /** + * Returns the signer account id of the `Transaction`. + */ + readonly signerId: string; + /** + * Returns the `PublicKey` of the signer of the `Transaction`. + */ + readonly signerPublicKey: string; + /** + * Returns the `Signature` the `Transaction` was signed with. + */ + readonly signature: string; + /** + * Returns the receiver account id of the `Transaction`. + */ + readonly receiverId: string; + /** + * Returns the status of the `Transaction` as `ExecutionStatus`. + */ + readonly status: ExecutionStatus; + /** + * Returns the id of the `ExecutionOutcome` for the `Transaction`. + */ + readonly executionOutcomeId: string; + /** + * Returns an Array of `Operation` for the `Transaction`. + */ + readonly operations: Operation[]; + constructor( + /** + * Returns the hash of the `Transaction` in `CryptoHash`. + */ + transactionHash: string, + /** + * Returns the signer account id of the `Transaction`. + */ + signerId: string, + /** + * Returns the `PublicKey` of the signer of the `Transaction`. + */ + signerPublicKey: string, + /** + * Returns the `Signature` the `Transaction` was signed with. + */ + signature: string, + /** + * Returns the receiver account id of the `Transaction`. + */ + receiverId: string, + /** + * Returns the status of the `Transaction` as `ExecutionStatus`. + */ + status: ExecutionStatus, + /** + * Returns the id of the `ExecutionOutcome` for the `Transaction`. + */ + executionOutcomeId: string, + /** + * Returns an Array of `Operation` for the `Transaction`. + */ + operations: Operation[], + ); +} + +/** + * This structure is almost an identical copy of the `StateChangeWithCauseView` from [near-primitives](https://github.com/near/nearcore/tree/master/core/primitives) with a propagated additional field `affectedAccountId`. + */ +declare class StateChange { + /** + * Returns the `cause` of the `StateChange`. + */ + readonly cause: StateChangeCause; + /** + * Returns the `value` of the `StateChange`. + */ + readonly value: StateChangeValue; + constructor( + /** + * Returns the `cause` of the `StateChange`. + */ + cause: StateChangeCause, + /** + * Returns the `value` of the `StateChange`. + */ + value: StateChangeValue, + ); + /** + * Returns the account id of the `StateChange`. + */ + get affectedAccountId(): string; + /** + * Returns the `StateChange` from the `StateChangeWithCauseView`. Created for backward compatibility. + */ + static fromStateChangeView(stateChangeView: StateChangeWithCauseView): StateChange; +} +type TransactionProcessingCause = { + txHash: string; +}; +type ActionReceiptProcessingStartedCause = { + receiptHash: string; +}; +type ActionReceiptGasRewardCause = { + receiptHash: string; +}; +type ReceiptProcessingCause = { + receiptHash: string; +}; +type PostponedReceiptCause = { + receiptHash: string; +}; +type StateChangeCause = + | 'NotWritableToDisk' + | 'InitialState' + | TransactionProcessingCause + | ActionReceiptProcessingStartedCause + | ActionReceiptGasRewardCause + | ReceiptProcessingCause + | PostponedReceiptCause + | 'UpdatedDelayedReceipts' + | 'ValidatorAccountsUpdate' + | 'Migration' + | 'Resharding'; +declare class AccountUpdateValue { + readonly accountId: string; + readonly account: Account; + constructor(accountId: string, account: Account); +} +declare class AccountDeletionValue { + readonly accountId: string; + constructor(accountId: string); +} +declare class AccountKeyUpdateValue { + readonly accountId: string; + readonly publicKey: string; + readonly accessKey: AccessKey; + constructor(accountId: string, publicKey: string, accessKey: AccessKey); +} +declare class AccessKeyDeletionValue { + readonly accountId: string; + readonly publicKey: string; + constructor(accountId: string, publicKey: string); +} +declare class DataUpdateValue { + readonly accountId: string; + readonly key: Uint8Array; + readonly value: Uint8Array; + constructor(accountId: string, key: Uint8Array, value: Uint8Array); +} +declare class DataDeletionValue { + readonly accountId: string; + readonly key: Uint8Array; + constructor(accountId: string, key: Uint8Array); +} +declare class ContractCodeUpdateValue { + readonly accountId: string; + readonly code: Uint8Array; + constructor(accountId: string, code: Uint8Array); +} +declare class ContractCodeDeletionValue { + readonly accountId: string; + constructor(accountId: string); +} +type StateChangeValue = + | AccountUpdateValue + | AccountDeletionValue + | AccountKeyUpdateValue + | AccessKeyDeletionValue + | DataUpdateValue + | DataDeletionValue + | ContractCodeUpdateValue + | ContractCodeDeletionValue; +declare class Account { + readonly amount: number; + readonly locked: number; + readonly codeHash: string; + readonly storageUsage: number; + readonly storagePaidAt: number; + constructor(amount: number, locked: number, codeHash: string, storageUsage: number, storagePaidAt: number); +} + +/** + * The `Block` type is used to represent a block in the NEAR Lake Framework. + * + * **Important Notes on `Block`:** + * - All the entities located on different shards were merged into one single list without differentiation. + * - `Block` is not the fairest name for this structure either. NEAR Protocol is a sharded blockchain, so its block is actually an ephemeral structure that represents a collection of real blocks called chunks in NEAR Protocol. + */ +declare class Block { + /** + * Low-level structure for backward compatibility. + * As implemented in previous versions of [`near-lake-framework`](https://www.npmjs.com/package/near-lake-framework). + */ + readonly streamerMessage: StreamerMessage; + private executedReceipts; + /** + * Receipts included on the chain but not executed yet marked as “postponed”: they are represented by the same structure `Receipt` (see the corresponding section in this doc for more details). + */ + readonly postponedReceipts: Receipt[]; + /** + * List of included `Transactions`, converted into `Receipts`. + * + * **_NOTE_:** Heads up! You might want to know about `Transactions` to know where the action chain has begun. Unlike Ethereum, where a Transaction contains everything you may want to know about a particular interaction on the Ethereum blockchain, Near Protocol because of its asynchronous nature converts a `Transaction` into a `Receipt` before executing it. Thus, On NEAR, `Receipts` are more important for figuring out what happened on-chain as a result of a Transaction signed by a user. Read more about [Transactions on Near](https://nomicon.io/RuntimeSpec/Transactions) here. + * + */ + readonly transactions: Transaction[]; + private _actions; + private _events; + private _stateChanges; + constructor( + /** + * Low-level structure for backward compatibility. + * As implemented in previous versions of [`near-lake-framework`](https://www.npmjs.com/package/near-lake-framework). + */ + streamerMessage: StreamerMessage, + executedReceipts: Receipt[], + /** + * Receipts included on the chain but not executed yet marked as “postponed”: they are represented by the same structure `Receipt` (see the corresponding section in this doc for more details). + */ + postponedReceipts: Receipt[], + /** + * List of included `Transactions`, converted into `Receipts`. + * + * **_NOTE_:** Heads up! You might want to know about `Transactions` to know where the action chain has begun. Unlike Ethereum, where a Transaction contains everything you may want to know about a particular interaction on the Ethereum blockchain, Near Protocol because of its asynchronous nature converts a `Transaction` into a `Receipt` before executing it. Thus, On NEAR, `Receipts` are more important for figuring out what happened on-chain as a result of a Transaction signed by a user. Read more about [Transactions on Near](https://nomicon.io/RuntimeSpec/Transactions) here. + * + */ + transactions: Transaction[], + _actions: Map, + _events: Map, + _stateChanges: StateChange[], + ); + /** + * Returns the block hash. A shortcut to get the data from the block header. + */ + get blockHash(): string; + /** + * Returns the previous block hash. A shortcut to get the data from the block header. + */ + get prevBlockHash(): string; + /** + * Returns the block height. A shortcut to get the data from the block header. + */ + get blockHeight(): number; + /** + * Returns a `BlockHeader` structure of the block + * See `BlockHeader` structure sections for details. + */ + header(): BlockHeader; + /** + * Returns a slice of `Receipts` executed in the block. + * Basically is a getter for the `executedReceipts` field. + */ + receipts(): Receipt[]; + /** + * Returns an Array of `Actions` executed in the block. + */ + actions(): Action[]; + /** + * Returns `Events` emitted in the block. + */ + events(): Event[]; + /** + * Returns raw logs regardless of the fact that they are standard events or not. + */ + logs(): Log[]; + /** + * Returns an Array of `StateChange` occurred in the block. + */ + stateChanges(): StateChange[]; + /** + * Returns `Action` of the provided `receipt_id` from the block if any. Returns `undefined` if there is no corresponding `Action`. + * + * This method uses the internal `Block` `action` field which is empty by default and will be filled with the block’s actions on the first call to optimize memory usage. + * + * The result is either `Action | undefined` since there might be a request for an `Action` by `receipt_id` from another block, in which case this method will be unable to find the `Action` in the current block. In the other case, the request might be for an `Action` for a `receipt_id` that belongs to a `DataReceipt` where an action does not exist. + */ + actionByReceiptId(receipt_id: string): Action | undefined; + /** + * Returns an Array of Events emitted by `ExecutionOutcome` for the given `receipt_id`. There might be more than one `Event` for the `Receipt` or there might be none of them. In the latter case, this method returns an empty Array. + */ + eventsByReceiptId(receipt_id: string): Event[]; + /** + * Returns an Array of Events emitted by `ExecutionOutcome` for the given `account_id`. There might be more than one `Event` for the `Receipt` or there might be none of them. In the latter case, this method returns an empty Array. + */ + eventsByAccountId(account_id: string): Event[]; + private buildActionsHashmap; + private buildEventsHashmap; + static fromStreamerMessage(streamerMessage: StreamerMessage): Block; +} +/** + * Replacement for `BlockHeaderView` from [near-primitives](https://github.com/near/nearcore/tree/master/core/primitives). Shrunken and simplified. + * + * **Note:** the original `BlockHeaderView` is still accessible via the `.streamerMessage` attribute. + */ +declare class BlockHeader { + readonly height: number; + readonly hash: string; + readonly prevHash: string; + readonly author: string; + readonly timestampNanosec: string; + readonly epochId: string; + readonly nextEpochId: string; + readonly gasPrice: string; + readonly totalSupply: string; + readonly latestProtocolVersion: number; + readonly randomValue: string; + readonly chunksIncluded: number; + readonly validatorProposals: ValidatorStakeView[]; + constructor( + height: number, + hash: string, + prevHash: string, + author: string, + timestampNanosec: string, + epochId: string, + nextEpochId: string, + gasPrice: string, + totalSupply: string, + latestProtocolVersion: number, + randomValue: string, + chunksIncluded: number, + validatorProposals: ValidatorStakeView[], + ); + static fromStreamerMessage(streamerMessage: StreamerMessage): BlockHeader; +} + +declare const fromBorsh: (schema: borsh.Schema, encoded: Uint8Array) => borsh_lib_types_types.DecodeTypes; + +const fromBorsh$1 = /*#__PURE__*/ _mergeNamespaces( + { + __proto__: null, + fromBorsh: fromBorsh, + }, + [borsher], +) as { fromBorsh: typeof fromBorsh }; + +export { + Block, + type BlockHeaderView, + type BlockHeight, + type BlockView, + Event, + LakeContext, + Receipt, + type Shard, + StateChange, + type StreamerMessage, + Transaction, + fromBorsh$1 as borsh, +}; diff --git a/frontend/src/components/Editor/EditorView/DeveloperToolsView.jsx b/frontend/src/components/Editor/EditorView/DeveloperToolsView.jsx index d9d05a2ac..26cbd0f0d 100644 --- a/frontend/src/components/Editor/EditorView/DeveloperToolsView.jsx +++ b/frontend/src/components/Editor/EditorView/DeveloperToolsView.jsx @@ -4,20 +4,23 @@ import BlockPickerContainer from '../EditorViewContainer/BlockPickerContainer'; import CustomTooltip, { TooltipDirection } from '@/components/Common/CustomTooltip'; const DeveloperToolsView = ({ + // Props handleFormating, handleCodeGen, - setShowResetCodeModel, - debugMode, - setDebugMode, + executeIndexerFunction, + isExecuting, + stopExecution, heights, setHeights, latestHeight, - isExecuting, - stopExecution, - removeHeight, - executeIndexerFunction, diffView, setDiffView, + // Context + setShowResetCodeModel, + debugMode, + setDebugMode, + // Functions + removeHeight, }) => { const [hoveredIndex, setHoveredIndex] = useState(null); diff --git a/frontend/src/components/Editor/EditorView/EditorMenuView.jsx b/frontend/src/components/Editor/EditorView/EditorMenuView.jsx index f1df07d9e..457f070c9 100644 --- a/frontend/src/components/Editor/EditorView/EditorMenuView.jsx +++ b/frontend/src/components/Editor/EditorView/EditorMenuView.jsx @@ -4,18 +4,18 @@ import { Braces, ArrowCounterclockwise, FileText, TrashFill } from 'react-bootst import CustomTooltip, { TooltipDirection } from '@/components/Common/CustomTooltip'; const EditorMenuView = ({ + // Props + isUserIndexer, + handleDeleteIndexer, + isCreateNewIndexer, + error, + // Context indexerName, accountId, indexerDetails, setShowPublishModal, - setShowResetCodeModel, setShowForkIndexerModal, - handleDeleteIndexer, - debugMode, - isCreateNewIndexer, setShowLogsView, - isUserIndexer, - error, }) => { return ( diff --git a/frontend/src/components/Editor/EditorViewContainer/BlockPickerContainer.tsx b/frontend/src/components/Editor/EditorViewContainer/BlockPickerContainer.tsx index 754d1ad1f..95aa74742 100644 --- a/frontend/src/components/Editor/EditorViewContainer/BlockPickerContainer.tsx +++ b/frontend/src/components/Editor/EditorViewContainer/BlockPickerContainer.tsx @@ -1,4 +1,5 @@ import React, { useState } from 'react'; + import BlockPickerView from '../EditorView/BlockPickerView'; interface BlockPickerContainerProps { diff --git a/frontend/src/components/Editor/EditorViewContainer/DeveloperToolsContainer.tsx b/frontend/src/components/Editor/EditorViewContainer/DeveloperToolsContainer.tsx index c7013b565..c4f8d2850 100644 --- a/frontend/src/components/Editor/EditorViewContainer/DeveloperToolsContainer.tsx +++ b/frontend/src/components/Editor/EditorViewContainer/DeveloperToolsContainer.tsx @@ -1,19 +1,17 @@ import React, { useContext } from 'react'; -import DeveloperToolsView from '../EditorView/DeveloperToolsView'; + import { IndexerDetailsContext } from '../../../contexts/IndexerDetailsContext'; +import DeveloperToolsView from '../EditorView/DeveloperToolsView'; interface DeveloperToolsContainerProps { handleFormating: () => void; handleCodeGen: () => void; - executeIndexerFunction: () => void; isExecuting: boolean; + executeIndexerFunction: () => void; + heights: number[]; + setHeights: React.Dispatch>; stopExecution: () => void; - heights: string[]; - setHeights: React.Dispatch>; - latestHeight: number; - isUserIndexer: boolean; - handleDeleteIndexer: () => void; - error: string; + latestHeight: number | undefined; diffView: boolean; setDiffView: React.Dispatch>; } @@ -27,24 +25,10 @@ const DeveloperToolsContainer: React.FC = ({ heights, setHeights, latestHeight, - isUserIndexer, - handleDeleteIndexer, - error, diffView, setDiffView, }) => { - const { - indexerName, - accountId, - indexerDetails, - setShowPublishModal, - setShowResetCodeModel, - setShowForkIndexerModal, - debugMode, - setDebugMode, - isCreateNewIndexer, - setShowLogsView, - } = useContext(IndexerDetailsContext); + const { setShowResetCodeModel, debugMode, setDebugMode } = useContext(IndexerDetailsContext); const removeHeight = (index: number): void => { setHeights(heights.filter((_, i) => i !== index)); @@ -53,20 +37,23 @@ const DeveloperToolsContainer: React.FC = ({ return ( ); diff --git a/frontend/src/components/Editor/EditorViewContainer/EditorMenuContainer.jsx b/frontend/src/components/Editor/EditorViewContainer/EditorMenuContainer.jsx deleted file mode 100644 index fa381d5cc..000000000 --- a/frontend/src/components/Editor/EditorViewContainer/EditorMenuContainer.jsx +++ /dev/null @@ -1,50 +0,0 @@ -import React, { useContext } from 'react'; -import EditorMenuView from '../EditorView/EditorMenuView'; -import { IndexerDetailsContext } from '../../../contexts/IndexerDetailsContext'; - -const EditorMenuContainer = (props) => { - const { - handleFormating, - handleCodeGen, - error, - executeIndexerFunction, - heights, - setHeights, - isCreateNewIndexer, - isExecuting, - stopExecution, - latestHeight, - isUserIndexer, - handleDeleteIndexer, - } = props; - const { - indexerName, - accountId, - indexerDetails, - setShowPublishModal, - setShowResetCodeModel, - setShowForkIndexerModal, - debugMode, - setShowLogsView, - } = useContext(IndexerDetailsContext); - - return ( - - ); -}; - -export default EditorMenuContainer; diff --git a/frontend/src/components/Editor/EditorViewContainer/EditorMenuContainer.tsx b/frontend/src/components/Editor/EditorViewContainer/EditorMenuContainer.tsx new file mode 100644 index 000000000..3c6f23734 --- /dev/null +++ b/frontend/src/components/Editor/EditorViewContainer/EditorMenuContainer.tsx @@ -0,0 +1,219 @@ +import { request } from 'near-social-bridge'; +import React, { useContext } from 'react'; + +import { + CODE_FORMATTING_ERROR_MESSAGE, + FORMATTING_ERROR_TYPE, + INDEXER_REGISTER_TYPE_GENERATION_ERROR, + SCHEMA_FORMATTING_ERROR_MESSAGE, + TYPE_GENERATION_ERROR_TYPE, +} from '@/constants/Strings'; +import { IndexerDetailsContext } from '@/contexts/IndexerDetailsContext'; +import { + defaultCode, + defaultSchema, + defaultSchemaTypes, + formatIndexingCode, + formatSQL, + wrapCode, +} from '@/utils/formatters'; +import { sanitizeAccountId, sanitizeIndexerName } from '@/utils/helpers'; +import { queryIndexerFunctionDetails as PreviousSavedCode } from '@/utils/queryIndexerFunction'; +import { validateJSCode, validateSQLSchema } from '@/utils/validators'; + +import { ForkIndexerModal } from '@/components/Modals/ForkIndexerModal'; +import { PublishModal } from '@/components/Modals/PublishModal'; +import { ResetChangesModal } from '@/components/Modals/ResetChangesModal'; +import EditorMenuView from '../EditorView/EditorMenuView'; + +interface EditorMenuContainerProps { + isUserIndexer: boolean; + handleDeleteIndexer: () => void; + isCreateNewIndexer: boolean; + error: string | undefined; + indexingCode: string; + setIndexingCode: (code: string) => void; + currentUserAccountId: string | undefined; + // reset code + setSchema: (schema: string) => void; + setSchemaTypes: (schemaTypes: string) => void; + setOriginalIndexingCode: (code: string) => void; + setOriginalSQLCode: (code: string) => void; + // publish + actionButtonText: string; + schema: string; + setError: (error: string) => void; + showModal: (modalName: string, modalProps: any) => void; +} + +const EditorMenuContainer: React.FC = ({ + isUserIndexer, + handleDeleteIndexer, + isCreateNewIndexer, + error, + indexingCode, + setIndexingCode, + currentUserAccountId, + // reset code + setSchema, + setSchemaTypes, + setOriginalIndexingCode, + setOriginalSQLCode, + // publish + actionButtonText, + schema, + setError, + showModal, +}) => { + const { + indexerName, + accountId, + indexerDetails, + setShowForkIndexerModal, + setShowLogsView, + setAccountId, + // reset code + setShowResetCodeModel, + // publish + setShowPublishModal, + } = useContext(IndexerDetailsContext); + + const forkIndexer = async (indexerName: string): Promise => { + if (!indexerDetails.accountId || !indexerDetails.indexerName || !indexerName || !currentUserAccountId) return; + const sanitizedForkedFromAccountId = sanitizeAccountId(indexerDetails.accountId); + const sanitizedForkedFromIndexerName = sanitizeIndexerName(indexerDetails.indexerName); + const sanitizedIndexerNameInput = sanitizeIndexerName(indexerName); + const sanitizedCurrentAccountId = sanitizeAccountId(currentUserAccountId); + + const sanitizedCode = indexingCode + .replaceAll(sanitizedForkedFromAccountId, sanitizedCurrentAccountId) + .replaceAll(sanitizedForkedFromIndexerName, sanitizedIndexerNameInput); + + setAccountId(currentUserAccountId); + setIndexingCode(sanitizedCode); + }; + + const handleResetCodeChanges = async (): Promise => { + if (isCreateNewIndexer) { + setShowResetCodeModel(false); + setIndexingCode(formatIndexingCode(defaultCode)); + setSchema(formatSQL(defaultSchema)); + setSchemaTypes(defaultSchemaTypes); + return; + } + loadDataFromPreviousSaved().catch((err) => { + console.log(err); + }); + setShowResetCodeModel(false); + }; + + const loadDataFromPreviousSaved = async (): Promise => { + try { + const data = await PreviousSavedCode(indexerDetails.accountId, indexerDetails.indexerName); + if (data == null) { + setIndexingCode(defaultCode); + setSchema(defaultSchema); + setSchemaTypes(defaultSchemaTypes); + } else { + const unformattedIndexerCode = wrapCode(data.code); + const unformattedSchemaCode = data.schema; + if (unformattedIndexerCode !== null) { + setOriginalIndexingCode(unformattedIndexerCode); + setIndexingCode(unformattedIndexerCode); + } + if (unformattedSchemaCode !== null) { + setOriginalSQLCode(unformattedSchemaCode); + setSchema(unformattedSchemaCode); + } + // todo add reformatting (reformatAll) + } + } catch (error) { + console.error('Error loading data:', error); + } + }; + + const registerFunction = async (indexerName: string, indexerConfig: any): Promise => { + const { data: validatedSchema, error: schemaValidationError } = validateSQLSchema(schema); + const { data: validatedCode, error: codeValidationError } = validateJSCode(indexingCode); + + if (codeValidationError) { + setError(CODE_FORMATTING_ERROR_MESSAGE); + return; + } + + const innerCode = validatedCode?.match(/getBlock\s*\([^)]*\)\s*{([\s\S]*)}/)?.[1] || ''; + indexerName = indexerName.replaceAll(' ', '_'); + const forkedFrom = + indexerDetails.forkedAccountId && indexerDetails.forkedIndexerName + ? { + account_id: indexerDetails.forkedAccountId, + function_name: indexerDetails.forkedIndexerName, + } + : null; + + const startBlock = + indexerConfig.startBlock === 'startBlockHeight' + ? { HEIGHT: indexerConfig.height } + : indexerConfig.startBlock === 'startBlockLatest' + ? 'LATEST' + : 'CONTINUE'; + + if (schemaValidationError?.type === FORMATTING_ERROR_TYPE) { + setError(SCHEMA_FORMATTING_ERROR_MESSAGE); + return; + } else if (schemaValidationError?.type === TYPE_GENERATION_ERROR_TYPE) { + showModal(INDEXER_REGISTER_TYPE_GENERATION_ERROR, { + indexerName, + code: innerCode, + schema: validatedSchema, + startBlock, + contractFilter: indexerConfig.filter, + forkedFrom, + }); + return; + } + + request('register-function', { + indexerName, + code: innerCode, + schema: validatedSchema, + startBlock, + contractFilter: indexerConfig.filter, + ...(forkedFrom && { forkedFrom }), + }); + + setShowPublishModal(false); + }; + + const getActionButtonText = () => { + const isUserIndexer = indexerDetails.accountId === currentUserAccountId; + if (isCreateNewIndexer) return 'Create New Indexer'; + return isUserIndexer ? actionButtonText : 'Fork Indexer'; + }; + + return ( + <> + + + + + + ); +}; + +export default EditorMenuContainer; diff --git a/frontend/src/components/Editor/QueryApiStorageManager.tsx b/frontend/src/components/Editor/QueryApiStorageManager.tsx new file mode 100644 index 000000000..a52c8acf1 --- /dev/null +++ b/frontend/src/components/Editor/QueryApiStorageManager.tsx @@ -0,0 +1,68 @@ +export default class QueryAPIStorageManager { + private indexerCodeStorageKey: string; + private schemaCodeStorageKey: string; + private schemaTypesStorageKey: string; + private cursorPositionKey: string; + private debugListStorageKey: string; + + constructor(accountID: string, indexerName: string) { + this.indexerCodeStorageKey = this.createStorageKey('IndexerCode', accountID, indexerName); + this.schemaCodeStorageKey = this.createStorageKey('SchemaCode', accountID, indexerName); + this.schemaTypesStorageKey = this.createStorageKey('SchemaTypes', accountID, indexerName); + this.cursorPositionKey = this.createStorageKey('CursorPosition', accountID, indexerName); + this.debugListStorageKey = this.createStorageKey('DebugList', accountID, indexerName); + } + + private createStorageKey(type: string, accountID: string, indexerName: string): string { + return `QueryAPI:${type}:${accountID}#${indexerName || 'new'}`; + } + + private saveToLocalStorage(key: string, data: any): void { + localStorage.setItem(key, JSON.stringify(data)); + } + + private getFromLocalStorage(key: string): any { + const data = localStorage.getItem(key); + return data ? JSON.parse(data) : null; + } + + setIndexerCode(data: any): void { + this.saveToLocalStorage(this.indexerCodeStorageKey, data); + } + + getIndexerCode(): any { + return this.getFromLocalStorage(this.indexerCodeStorageKey); + } + + setSchemaCode(data: any): void { + this.saveToLocalStorage(this.schemaCodeStorageKey, data); + } + + getSchemaCode(): any { + return this.getFromLocalStorage(this.schemaCodeStorageKey); + } + + setSchemaTypes(data: any): void { + this.saveToLocalStorage(this.schemaTypesStorageKey, data); + } + + getSchemaTypes(): any { + return this.getFromLocalStorage(this.schemaTypesStorageKey); + } + + setCursorPosition(data: any): void { + this.saveToLocalStorage(this.cursorPositionKey, data); + } + + getCursorPosition(): any { + return this.getFromLocalStorage(this.cursorPositionKey); + } + + setDebugList(data: any): void { + this.saveToLocalStorage(this.debugListStorageKey, data); + } + + getDebugList(): any { + return this.getFromLocalStorage(this.debugListStorageKey); + } +} diff --git a/frontend/src/components/Editor/__tests__/Editor.test.js b/frontend/src/components/Editor/__tests__/Editor.test.js deleted file mode 100644 index d4c4ef396..000000000 --- a/frontend/src/components/Editor/__tests__/Editor.test.js +++ /dev/null @@ -1,82 +0,0 @@ -import React from 'react'; -import { render, screen, fireEvent, waitFor } from '@testing-library/react'; -import '@testing-library/jest-dom/extend-expect'; -import userEvent from '@testing-library/user-event'; -import Editor from './Editor'; - -describe('Editor for creating a new indexer', () => { - beforeEach(() => { - render( - , - ); - }); - - test('renders and displays the component without errors', () => { - expect(screen.getByTestId('action-button-group')).toBeInTheDocument(); - expect(screen.getByTestId('indexing-logic-file-button')).toBeInTheDocument(); - expect(screen.getByTestId('schema-file-button')).toBeInTheDocument(); - expect(screen.getByTestId('diff-view-switch')).toBeInTheDocument(); - }); - - test('verifies the visibility and functionality of buttons: Reset, Format Code, and Register Function', async () => { - const resetButton = screen.getByTestId('reset-button'); - const formatButton = screen.getByTestId('format-code-button'); - const registerButton = screen.getByTestId('submit-code-button'); - - expect(resetButton).toBeInTheDocument(); - expect(formatButton).toBeInTheDocument(); - expect(registerButton).toBeInTheDocument(); - - fireEvent.click(resetButton); - await waitFor(() => expect(screen.queryByText('Are you sure?')).toBeInTheDocument()); - - fireEvent.click(formatButton); - await waitFor(() => - expect( - screen.queryByText('Oh snap! We could not format your code. Make sure it is proper Javascript code.'), - ).not.toBeInTheDocument(), - ); - - fireEvent.click(registerButton); - }); - - test('ensures that the component loads the default or stored values for the indexing code and SQL schema', () => { - expect(screen.getByTestId('code-editor-indexing-logic')).toBeInTheDocument(); - expect(screen.getByTestId('schema-editor-schema')).toBeInTheDocument(); - }); - - test('confirming that the component handles formatting errors and displays an error message when the indexing code or SQL schema is not valid', async () => { - const invalidCode = 'function invalidCode) {}'; - // await new Promise((r) => setTimeout(r, 3000)); - const indexingEditor = screen.getByTestId('code-editor-indexing-logic'); - // fireEvent.click(screen.getByTestId('indexing-logic-file-button')); - // userEvent.type(indexingEditor, invalidCode); - - // fireEvent.click(screen.getByTestId('format-code-button')); - // await waitFor(() => expect(screen.queryByText('Oh snap! We could not format your code. Make sure it is proper Javascript code.')).toBeInTheDocument()); - }); - - test('testing the Diff View switch and making sure the component switches between normal and diff view as expected', () => { - const diffViewSwitch = screen.getByTestId('diff-view-switch'); - fireEvent.click(diffViewSwitch); - expect(screen.getByTestId('diff-editor-indexing-logic')).toBeInTheDocument(); - fireEvent.click(diffViewSwitch); - expect(screen.getByTestId('code-editor-indexing-logic')).toBeInTheDocument(); - }); - - test('checking the component behavior when resetting the code and reloading the original code and schema', async () => { - fireEvent.click(screen.getByTestId('reset-button')); - await waitFor(() => expect(screen.queryByText('Are you sure?')).toBeInTheDocument()); - fireEvent.click(screen.getByText('Reload')); - await waitFor(() => expect(screen.queryByText('Are you sure?')).not.toBeInTheDocument()); - expect(screen.getByTestId('code-editor-indexing-logic')).toBeInTheDocument(); - expect(screen.getByTestId('schema-editor-schema')).toBeInTheDocument(); - }); -}); diff --git a/frontend/src/components/Logs/LogsMenu.tsx b/frontend/src/components/Logs/LogsMenu.tsx index 5d44e936d..37c56b48e 100644 --- a/frontend/src/components/Logs/LogsMenu.tsx +++ b/frontend/src/components/Logs/LogsMenu.tsx @@ -1,8 +1,10 @@ -import React, { useState, useEffect, useContext } from 'react'; -import { useQuery, gql } from '@apollo/client'; -import { Button, Navbar, Container, ButtonGroup, Spinner } from 'react-bootstrap'; +import { gql, useQuery } from '@apollo/client'; +import React, { useContext, useEffect, useState } from 'react'; +import { Button, ButtonGroup, Container, Navbar, Spinner } from 'react-bootstrap'; import { ArrowCounterclockwise, Code } from 'react-bootstrap-icons'; + import { IndexerDetailsContext } from '@/contexts/IndexerDetailsContext'; + import LatestBlock from '../Common/LatestBlock'; interface LogsMenuProps { @@ -118,7 +120,7 @@ const LogsMenu: React.FC = ({ variant="outline-primary" className="d-flex align-items-center" onClick={() => { - setShowLogsView(); + setShowLogsView(!showLogsView); }} > diff --git a/frontend/src/components/Logs/LogsView/ClearButtonView.jsx b/frontend/src/components/Logs/LogsView/ClearButtonView.jsx index 28e00eee5..2083aebe9 100644 --- a/frontend/src/components/Logs/LogsView/ClearButtonView.jsx +++ b/frontend/src/components/Logs/LogsView/ClearButtonView.jsx @@ -1,6 +1,6 @@ import React from 'react'; import { Button } from 'react-bootstrap'; -import { ClearIcon } from '../LogsView/Icons/ClearIcon'; +import { ClearIcon } from '@/components/Common/Icons/ClearIcon'; const ClearButtonView = ({ onClick }) => { return ( diff --git a/frontend/src/components/Logs/LogsView/OptionSelectorView.jsx b/frontend/src/components/Logs/LogsView/OptionSelectorView.jsx index 0439d7140..e5bd7d08b 100644 --- a/frontend/src/components/Logs/LogsView/OptionSelectorView.jsx +++ b/frontend/src/components/Logs/LogsView/OptionSelectorView.jsx @@ -1,6 +1,6 @@ import React from 'react'; import { Row, Col, Form, Button } from 'react-bootstrap'; -import { CheckmarkIcon } from './Icons/CheckMarkIcon'; +import { CheckmarkIcon } from '@/components/Common/Icons/CheckMarkIcon'; import ClearButtonContainer from '../LogsViewContainer/ClearButtonContainer'; const OptionSelectorView = ({ options, selectedOption, onOptionChange, handleOptionChange, handleClearSelection }) => { diff --git a/frontend/src/components/Logs/LogsViewContainer/ClearButtonContainer.tsx b/frontend/src/components/Logs/LogsViewContainer/ClearButtonContainer.tsx index 77edd1b8e..8deca26e1 100644 --- a/frontend/src/components/Logs/LogsViewContainer/ClearButtonContainer.tsx +++ b/frontend/src/components/Logs/LogsViewContainer/ClearButtonContainer.tsx @@ -1,4 +1,5 @@ -import React, { type MouseEvent, type FC } from 'react'; +import React, { type FC, type MouseEvent } from 'react'; + import ClearButtonView from '../LogsView/ClearButtonView'; interface ClearButtonProps { diff --git a/frontend/src/components/Logs/LogsViewContainer/DateSelectorContainer.tsx b/frontend/src/components/Logs/LogsViewContainer/DateSelectorContainer.tsx index cb05ba389..0ca996fc2 100644 --- a/frontend/src/components/Logs/LogsViewContainer/DateSelectorContainer.tsx +++ b/frontend/src/components/Logs/LogsViewContainer/DateSelectorContainer.tsx @@ -1,7 +1,9 @@ import React from 'react'; -import DateSelectorView from '../LogsView/DateSelectorView'; + import { TIME_INTERVALS_MAP } from '@/constants/DurationMap'; +import DateSelectorView from '../LogsView/DateSelectorView'; + interface DateSelectorProps { selectedDate: string; onDateChange: (selectedDate: Date) => void; diff --git a/frontend/src/components/Logs/LogsViewContainer/IndexerLogsContainer.tsx b/frontend/src/components/Logs/LogsViewContainer/IndexerLogsContainer.tsx index af358681f..10bdebf71 100644 --- a/frontend/src/components/Logs/LogsViewContainer/IndexerLogsContainer.tsx +++ b/frontend/src/components/Logs/LogsViewContainer/IndexerLogsContainer.tsx @@ -1,13 +1,14 @@ -import React, { useContext, useState, useEffect, useRef } from 'react'; +import { Grid } from 'gridjs'; import { useInitialPayload } from 'near-social-bridge'; -import { sanitizeString } from '@/utils/helpers'; +import React, { useContext, useEffect, useRef, useState } from 'react'; + import { IndexerDetailsContext } from '@/contexts/IndexerDetailsContext'; -import IndexerLogsView from '../LogsView/IndexerLogsView'; -import { Grid } from 'gridjs'; +import { formatTimestamp } from '@/utils/formatTimestamp'; +import { sanitizeString } from '@/utils/helpers'; -import { QueryValidation } from '../GraphQL/QueryValidation'; import { Query } from '../GraphQL/Query'; -import { formatTimestamp } from '@/utils/formatTimestamp'; +import { QueryValidation } from '../GraphQL/QueryValidation'; +import IndexerLogsView from '../LogsView/IndexerLogsView'; interface GridConfig { columns: string[]; @@ -29,8 +30,8 @@ const IndexerLogsContainer: React.FC = () => { const { indexerDetails, latestHeight } = useContext(IndexerDetailsContext); const { currentUserAccountId } = useInitialPayload(); - const sanitizedAccountId: string = sanitizeString(indexerDetails.accountId); - const sanitizedIndexerName: string = sanitizeString(indexerDetails.indexerName); + const sanitizedAccountId: string = indexerDetails.accountId ? sanitizeString(indexerDetails.accountId) : ''; + const sanitizedIndexerName: string = indexerDetails.indexerName ? sanitizeString(indexerDetails.indexerName) : ''; const functionName = `${indexerDetails.accountId}/${indexerDetails.indexerName}`; const schemaName = `${sanitizedAccountId}_${sanitizedIndexerName}`; diff --git a/frontend/src/components/Logs/LogsViewContainer/LogTypeSelectorContainer.tsx b/frontend/src/components/Logs/LogsViewContainer/LogTypeSelectorContainer.tsx index 4b8f7ffe2..01af34f1f 100644 --- a/frontend/src/components/Logs/LogsViewContainer/LogTypeSelectorContainer.tsx +++ b/frontend/src/components/Logs/LogsViewContainer/LogTypeSelectorContainer.tsx @@ -1,4 +1,5 @@ import React from 'react'; + import LogTypeSelectorView from '../LogsView/LogTypeSelectorView'; interface LogTypeSelectorContainerProps { diff --git a/frontend/src/components/Logs/LogsViewContainer/OptionSelectorContainer.tsx b/frontend/src/components/Logs/LogsViewContainer/OptionSelectorContainer.tsx index 2cf3dd171..92fd5c891 100644 --- a/frontend/src/components/Logs/LogsViewContainer/OptionSelectorContainer.tsx +++ b/frontend/src/components/Logs/LogsViewContainer/OptionSelectorContainer.tsx @@ -1,4 +1,5 @@ import React from 'react'; + import OptionSelectorView from '../LogsView/OptionSelectorView'; interface OptionSelectorContainerProps { options: string[]; diff --git a/frontend/src/components/Logs/LogsViewContainer/SeveritySelectorContainer.tsx b/frontend/src/components/Logs/LogsViewContainer/SeveritySelectorContainer.tsx index cb7635ac6..aa08914fe 100644 --- a/frontend/src/components/Logs/LogsViewContainer/SeveritySelectorContainer.tsx +++ b/frontend/src/components/Logs/LogsViewContainer/SeveritySelectorContainer.tsx @@ -1,4 +1,5 @@ import React from 'react'; + import SeverityRadioButtonGroupView from '../LogsView/SeveritySelectorView'; interface SeveritySelectorProps { diff --git a/frontend/src/components/Modals/ForkIndexerModal.jsx b/frontend/src/components/Modals/ForkIndexerModal.jsx index e1ef53a43..e6204dfea 100644 --- a/frontend/src/components/Modals/ForkIndexerModal.jsx +++ b/frontend/src/components/Modals/ForkIndexerModal.jsx @@ -1,8 +1,8 @@ import React, { useContext, useState } from 'react'; import { Button, Modal, Alert, InputGroup, Form } from 'react-bootstrap'; -import { IndexerDetailsContext } from '../../contexts/IndexerDetailsContext'; +import { IndexerDetailsContext } from '@/contexts/IndexerDetailsContext'; -export const ForkIndexerModal = ({ registerFunction, forkIndexer }) => { +export const ForkIndexerModal = ({ forkIndexer }) => { const { indexerDetails, showForkIndexerModal, @@ -11,8 +11,6 @@ export const ForkIndexerModal = ({ registerFunction, forkIndexer }) => { setIndexerName, setForkedAccountId, setForkedIndexerName, - setIndexerConfig, - isCreateNewIndexer, } = useContext(IndexerDetailsContext); const [indexerName, setIndexerNameField] = useState(''); const [error, setError] = useState(null); @@ -38,12 +36,7 @@ export const ForkIndexerModal = ({ registerFunction, forkIndexer }) => { }; return ( - setShowForkIndexerModal(false)} - className="bg-gray-50" - > + setShowForkIndexerModal(false)} className="bg-gray-50"> Enter Indexer Details @@ -61,8 +54,8 @@ export const ForkIndexerModal = ({ registerFunction, forkIndexer }) => { {error && ( {error} diff --git a/frontend/src/components/Modals/ModalsContainer/PublishFormContainer.tsx b/frontend/src/components/Modals/ModalsContainer/PublishFormContainer.tsx index ca56fc627..b43b269f6 100644 --- a/frontend/src/components/Modals/ModalsContainer/PublishFormContainer.tsx +++ b/frontend/src/components/Modals/ModalsContainer/PublishFormContainer.tsx @@ -1,4 +1,5 @@ -import React, { useContext, useState, useEffect, type ChangeEvent } from 'react'; +import React, { type ChangeEvent, useContext, useEffect, useState } from 'react'; + import { IndexerDetailsContext } from '../../../contexts/IndexerDetailsContext'; import { validateContractIds } from '../../../utils/validators'; import PublishFormView from '../ModalsView/PublishFormView'; diff --git a/frontend/src/components/Modals/resetChanges.jsx b/frontend/src/components/Modals/ResetChangesModal.jsx similarity index 84% rename from frontend/src/components/Modals/resetChanges.jsx rename to frontend/src/components/Modals/ResetChangesModal.jsx index 15c2369b7..d56e73114 100644 --- a/frontend/src/components/Modals/resetChanges.jsx +++ b/frontend/src/components/Modals/ResetChangesModal.jsx @@ -1,7 +1,7 @@ import { IndexerDetailsContext } from '../../contexts/IndexerDetailsContext'; import React, { useContext } from 'react'; import { Button, Modal } from 'react-bootstrap'; -export const ResetChangesModal = ({ handleReload }) => { +export const ResetChangesModal = ({ handleResetCodeChanges }) => { const { showResetCodeModel, setShowResetCodeModel } = useContext(IndexerDetailsContext); return ( setShowResetCodeModel(false)}> @@ -13,7 +13,7 @@ export const ResetChangesModal = ({ handleReload }) => { - diff --git a/frontend/src/contexts/IndexerDetailsContext.js b/frontend/src/contexts/IndexerDetailsContext.tsx similarity index 50% rename from frontend/src/contexts/IndexerDetailsContext.js rename to frontend/src/contexts/IndexerDetailsContext.tsx index e0f58a3cd..5bedbf0da 100644 --- a/frontend/src/contexts/IndexerDetailsContext.js +++ b/frontend/src/contexts/IndexerDetailsContext.tsx @@ -1,30 +1,57 @@ -import React, { useState, useEffect } from 'react'; -import { queryIndexerFunctionDetails } from '@/utils/queryIndexerFunction'; -import { defaultCode, defaultSchema, wrapCode } from '../utils/formatters'; - import { useInitialPayload } from 'near-social-bridge'; +import React, { createContext, useEffect, useState } from 'react'; + import { getLatestBlockHeight } from '@/utils/getLatestBlockHeight'; -// interface IndexerDetails { -// accountId: String, -// indexerName: String, -// code: String, -// schema: String, -// config: IndexerConfig, -// } -// -// type IndexerConfig = { -// startBlockHeight?: Number, -// filter: String, -// } +import { queryIndexerFunctionDetails } from '@/utils/queryIndexerFunction'; -export const IndexerDetailsContext = React.createContext({ +import { wrapCode } from '../utils/formatters'; + +interface IndexerDetails { + code?: string; + schema?: string; + rule: { affected_account_id: string }; + startBlock: string; + accountId?: string; + indexerName?: string; + forkedAccountId: string | null; + forkedIndexerName: string | null; +} + +interface IndexerDetailsContextProps { + indexerDetails: IndexerDetails; + showResetCodeModel: boolean; + setShowResetCodeModel: (bool: boolean) => void; + showPublishModal: boolean; + setShowPublishModal: (bool: boolean) => void; + showForkIndexerModal: boolean; + setShowForkIndexerModal: (bool: boolean) => void; + debugMode: boolean; + setDebugMode: (mode: boolean) => void; + latestHeight: number; + setLatestHeight: (height: number) => void; + isCreateNewIndexer: boolean; + setIsCreateNewIndexer: (bool: boolean) => void; + accountId?: string; + setAccountId: (accountId?: string) => void; + indexerName?: string; + setIndexerName: (indexerName?: string) => void; + forkedAccountId?: string; + setForkedAccountId: (accountId?: string) => void; + forkedIndexerName?: string; + setForkedIndexerName: (indexerName?: string) => void; + setIndexerDetails: (details: IndexerDetails) => void; + showLogsView: boolean; + setShowLogsView: (showLogsView: boolean) => void; +} + +export const IndexerDetailsContext = createContext({ indexerDetails: { code: undefined, schema: undefined, rule: { affected_account_id: 'social.near' }, startBlock: 'LATEST', - accountId: '', - indexerName: '', + accountId: undefined, + indexerName: undefined, forkedAccountId: null, forkedIndexerName: null, }, @@ -40,33 +67,33 @@ export const IndexerDetailsContext = React.createContext({ setLatestHeight: () => {}, isCreateNewIndexer: false, setIsCreateNewIndexer: () => {}, - accountId: undefined, setAccountId: () => {}, - indexerName: undefined, setIndexerName: () => {}, - forkedAccountId: undefined, setForkedAccountId: () => {}, - forkedIndexerName: undefined, setForkedIndexerName: () => {}, setIndexerDetails: () => {}, showLogsView: false, setShowLogsView: () => {}, }); -export const IndexerDetailsProvider = ({ children }) => { - const [accountId, setAccountId] = useState(undefined); - const [indexerName, setIndexerName] = useState(undefined); - const [forkedAccountId, setForkedAccountId] = useState(undefined); - const [forkedIndexerName, setForkedIndexerName] = useState(undefined); - const [indexerDetails, setIndexerDetails] = useState({ +interface IndexerDetailsProviderProps { + children: React.ReactNode; +} + +export const IndexerDetailsProvider: React.FC = ({ children }) => { + const [accountId, setAccountId] = useState(undefined); + const [indexerName, setIndexerName] = useState(undefined); + const [forkedAccountId, setForkedAccountId] = useState(undefined); + const [forkedIndexerName, setForkedIndexerName] = useState(undefined); + const [indexerDetails, setIndexerDetails] = useState({ code: undefined, schema: undefined, rule: { affected_account_id: 'social.near' }, startBlock: 'LATEST', - accountId: accountId, - indexerName: indexerName, - forkedAccountId: forkedAccountId, - forkedIndexerName: forkedIndexerName, + accountId, + indexerName, + forkedAccountId: forkedAccountId ?? null, + forkedIndexerName: forkedIndexerName ?? null, }); const [showResetCodeModel, setShowResetCodeModel] = useState(false); const [showPublishModal, setShowPublishModal] = useState(false); @@ -76,20 +103,21 @@ export const IndexerDetailsProvider = ({ children }) => { const [latestHeight, setLatestHeight] = useState(0); const [isCreateNewIndexer, setIsCreateNewIndexer] = useState(false); - const { activeView } = useInitialPayload(); + const activeView = useInitialPayload(); // Adjust the type according to what useInitialPayload returns useEffect(() => { - if (activeView == 'status') setShowLogsView(true); - }, []); + if (activeView === 'status') setShowLogsView(true); + }, [activeView]); - const requestIndexerDetails = async () => { + const requestIndexerDetails = async (): Promise => { + if (!accountId || !indexerName) return undefined; const data = await queryIndexerFunctionDetails(accountId, indexerName); if (data) { - const details = { - accountId: accountId, - indexerName: indexerName, - forkedAccountId: data.forked_from?.account_id, - forkedIndexerName: data.forked_from?.function_name, + const details: IndexerDetails = { + accountId, + indexerName, + forkedAccountId: data.forked_from?.account_id ?? null, + forkedIndexerName: data.forked_from?.function_name ?? null, code: wrapCode(data.code), schema: data.schema, startBlock: data.start_block, @@ -98,6 +126,7 @@ export const IndexerDetailsProvider = ({ children }) => { return details; } }; + useEffect(() => { (async () => { const latestHeight = await getLatestBlockHeight(); @@ -109,26 +138,18 @@ export const IndexerDetailsProvider = ({ children }) => { if (isCreateNewIndexer || !accountId || !indexerName) { setIndexerDetails((prevDetails) => ({ ...prevDetails, - accountId: accountId, - indexerName: indexerName, - forkedAccountId: forkedAccountId, - forkedIndexerName: forkedIndexerName, + accountId, + indexerName, + forkedAccountId: forkedAccountId ?? null, + forkedIndexerName: forkedIndexerName ?? null, })); return; } (async () => { const indexer = await requestIndexerDetails(); - const details = { - accountId: indexer.accountId, - indexerName: indexer.indexerName, - forkedAccountId: indexer.forkedAccountId, - forkedIndexerName: indexer.forkedIndexerName, - code: indexer.code, - schema: indexer.schema, - startBlock: indexer.startBlock, - rule: indexer.rule, - }; - setIndexerDetails(details); + if (indexer) { + setIndexerDetails(indexer); + } })(); }, [accountId, indexerName, forkedAccountId, forkedIndexerName, isCreateNewIndexer]); @@ -151,8 +172,10 @@ export const IndexerDetailsProvider = ({ children }) => { debugMode, setDebugMode, latestHeight, + setLatestHeight, isCreateNewIndexer, setIsCreateNewIndexer, + setIndexerDetails, showLogsView, setShowLogsView, }} diff --git a/frontend/src/contexts/ModalContext.js b/frontend/src/contexts/ModalContext.js index e13d54c50..d26a60ec4 100644 --- a/frontend/src/contexts/ModalContext.js +++ b/frontend/src/contexts/ModalContext.js @@ -1,4 +1,4 @@ -import React, { createContext, useState, useContext } from 'react'; +import React, { createContext, useContext, useState } from 'react'; const ModalContext = createContext({ openModal: false, diff --git a/frontend/src/pages/create-new-indexer/index.js b/frontend/src/pages/create-new-indexer/index.js index 6597ac1ed..ba9c04684 100644 --- a/frontend/src/pages/create-new-indexer/index.js +++ b/frontend/src/pages/create-new-indexer/index.js @@ -1,8 +1,9 @@ -import React, { useContext, useEffect } from 'react'; import { withRouter } from 'next/router'; +import React, { useContext, useEffect } from 'react'; import { Alert } from 'react-bootstrap'; -import { IndexerDetailsContext } from '@/contexts/IndexerDetailsContext'; + import CreateNewIndexer from '@/components/CreateNewIndexer'; +import { IndexerDetailsContext } from '@/contexts/IndexerDetailsContext'; const CreateNewIndexerPage = ({ router }) => { const { accountId } = router.query; diff --git a/frontend/src/pages/query-api-editor/index.js b/frontend/src/pages/query-api-editor/index.js index 887b0fe64..5e047f8db 100644 --- a/frontend/src/pages/query-api-editor/index.js +++ b/frontend/src/pages/query-api-editor/index.js @@ -1,7 +1,8 @@ -import React, { useEffect, useContext } from 'react'; import { withRouter } from 'next/router'; +import React, { useContext, useEffect } from 'react'; import { Alert } from 'react-bootstrap'; -import Editor from '@/components/Editor'; + +import Editor from '@/components/Editor/EditorComponents/Editor'; import IndexerLogsContainer from '@/components/Logs/LogsViewContainer/IndexerLogsContainer'; import { IndexerDetailsContext } from '@/contexts/IndexerDetailsContext'; diff --git a/frontend/src/utils/formatters.js b/frontend/src/utils/formatters.js index a6de20b03..5c48d08b4 100644 --- a/frontend/src/utils/formatters.js +++ b/frontend/src/utils/formatters.js @@ -1,6 +1,6 @@ import prettier from 'prettier'; -import SqlPlugin from 'prettier-plugin-sql'; import parserBabel from 'prettier/parser-babel'; +import SqlPlugin from 'prettier-plugin-sql'; let wrap_code = (code) => `import * as primitives from "@near-lake/primitives" /** diff --git a/frontend/src/utils/formatters.test.js b/frontend/src/utils/formatters.test.js index 5ed9f451b..5e86a6f0d 100644 --- a/frontend/src/utils/formatters.test.js +++ b/frontend/src/utils/formatters.test.js @@ -1,4 +1,4 @@ -import { formatSQL, formatIndexingCode } from './formatters'; +import { formatIndexingCode, formatSQL } from './formatters'; const inputSQL1 = `CREATE TABLE\n "indexer_storage" (\n "function_name" TEXT NOT NULL,\n "key_name" TEXT NOT NULL,\n "value" TEXT NOT NULL,\n PRIMARY KEY ("function_name", "key_name")\n )\n`; const expectedOutput1 = `CREATE TABLE diff --git a/frontend/src/utils/helpers.ts b/frontend/src/utils/helpers.ts index dff61ae3a..88aa4986e 100644 --- a/frontend/src/utils/helpers.ts +++ b/frontend/src/utils/helpers.ts @@ -1,3 +1,11 @@ export const sanitizeString = (str: string): string => { return str.replace(/[^a-zA-Z0-9]/g, '_').replace(/^([0-9])/, '_$1'); }; + +export const sanitizeIndexerName = (name: string): string => { + return name.replaceAll('-', '_').trim().toLowerCase(); +}; + +export const sanitizeAccountId = (accountId: string): string => { + return accountId.replaceAll('.', '_'); +}; diff --git a/frontend/src/utils/indexerRunner.js b/frontend/src/utils/indexerRunner.js index b61cb7a9d..c8cd4d9a9 100644 --- a/frontend/src/utils/indexerRunner.js +++ b/frontend/src/utils/indexerRunner.js @@ -1,5 +1,6 @@ import { Block } from '@near-lake/primitives'; import { Buffer } from 'buffer'; + import { fetchBlockDetails } from './fetchBlock'; import { PgSchemaTypeGen } from './pgSchemaTypeGen'; diff --git a/frontend/src/utils/validators.js b/frontend/src/utils/validators.js deleted file mode 100644 index 5b115ab23..000000000 --- a/frontend/src/utils/validators.js +++ /dev/null @@ -1,79 +0,0 @@ -import { defaultSchema, formatIndexingCode, formatSQL } from './formatters'; -import { PgSchemaTypeGen } from './pgSchemaTypeGen'; -import { CONTRACT_NAME_REGEX, WILD_CARD_REGEX, WILD_CARD } from '../constants/RegexExp'; -import { ValidationError } from '../classes/ValidationError'; -import { FORMATTING_ERROR_TYPE, TYPE_GENERATION_ERROR_TYPE } from '../constants/Strings'; - -export const validateContractId = (accountId) => { - accountId = accountId.trim(); - if (accountId === WILD_CARD) return true; - - const isLengthValid = accountId.length >= 2 && accountId.length <= 64; - if (!isLengthValid) return false; - - //test if the string starts with a '*.' and remove it if it does - const isWildCard = WILD_CARD_REGEX.test(accountId); - accountId = isWildCard ? accountId.slice(2) : accountId; - - //test if rest of string is valid accounting for/not isWildCard - const isRegexValid = CONTRACT_NAME_REGEX.test(accountId); - return isRegexValid; -}; - -export const validateContractIds = (accountIds) => { - const ids = accountIds.split(',').map((id) => id.trim()); - return ids.every((accountId) => validateContractId(accountId)); -}; - -/** - * Validates formatting and type generation from a SQL schema. - * - * @param {string} schema - The SQL schema to validate and format. - * @returns {{ data: string | null, error: string | null }} - An object containing the formatted schema and error (if any). - */ -export function validateSQLSchema(schema) { - if (!schema) return { data: null, error: null }; - if (schema === formatSQL(defaultSchema)) return { data: schema, error: null }; - - const pgSchemaTypeGen = new PgSchemaTypeGen(); - let formattedSchema; - - try { - formattedSchema = formatSQL(schema); - } catch (error) { - //todo: add error handling for location - return { data: schema, error: new ValidationError(error.message, FORMATTING_ERROR_TYPE) }; - } - - if (formattedSchema) { - try { - pgSchemaTypeGen.generateTypes(formattedSchema); // Sanity check - return { data: formattedSchema, error: null }; - } catch (error) { - console.log(error); - return { - data: schema, - error: new ValidationError(error.message, TYPE_GENERATION_ERROR_TYPE), - location: error.location, - }; - } - } -} - -/** - * Asynchronously validates and formats JavaScript code. - * - * @param {string} code - The JavaScript code to be validated and formatted. - * @returns {{ data: string | null, error: string | null }} An object containing either the formatted code or an error. - */ -export function validateJSCode(code) { - if (!code) return { data: null, error: null }; - - try { - const formattedCode = formatIndexingCode(code); - return { data: formattedCode, error: null }; - } catch (error) { - console.error(error.message); - return { data: code, error }; - } -} diff --git a/frontend/src/utils/validators.test.ts b/frontend/src/utils/validators.test.ts index 5d52906b8..adf2c302e 100644 --- a/frontend/src/utils/validators.test.ts +++ b/frontend/src/utils/validators.test.ts @@ -1,5 +1,5 @@ -import * as Validator from './validators'; -const { validateContractId, validateContractIds } = Validator; +import { defaultSchema } from './formatters'; +import { validateContractId, validateContractIds, validateSQLSchema } from './validators'; describe('validateContractId', () => { test('it should return true for valid contract ID', () => { @@ -218,3 +218,36 @@ describe('validateContractIds', () => { expect(validateContractIds(validWildCardwithInvalid)).toBe(false); }); }); + +const formattedDefaultSchema = `CREATE TABLE + \"indexer_storage\" ( + \"function_name\" TEXT NOT NULL, + \"key_name\" TEXT NOT NULL, + \"value\" TEXT NOT NULL, + PRIMARY KEY (\"function_name\", \"key_name\") + )\n`; + +jest.mock('./validators.ts', () => { + const originalModule = jest.requireActual('./validators.ts'); + return { + formatSchema: jest.fn(), + generateTypes: jest.fn(), + ...originalModule, + }; +}); + +describe('validateSQLSchema', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should return null data and error for empty schema', () => { + const result = validateSQLSchema(''); + expect(result).toEqual({ data: null, error: null }); + }); + + it('should return formatted data and null error for default schema', () => { + const result = validateSQLSchema(defaultSchema); + expect(result).toEqual({ data: formattedDefaultSchema, error: null }); + }); +}); diff --git a/frontend/src/utils/validators.ts b/frontend/src/utils/validators.ts new file mode 100644 index 000000000..685fac29f --- /dev/null +++ b/frontend/src/utils/validators.ts @@ -0,0 +1,107 @@ +import { ValidationError } from '../classes/ValidationError'; +import { CONTRACT_NAME_REGEX, WILD_CARD, WILD_CARD_REGEX } from '../constants/RegexExp'; +import { FORMATTING_ERROR_TYPE, TYPE_GENERATION_ERROR_TYPE } from '../constants/Strings'; +import { defaultSchema, formatIndexingCode, formatSQL } from './formatters'; +import { PgSchemaTypeGen } from './pgSchemaTypeGen'; + +interface ValidationResult { + data: string | null; + error: ValidationError | null; + location?: string; +} + +interface FormatResult { + data: string | null; + error: ValidationError | null; +} + +interface GenerateTypesResult { + error: ValidationError | null; + location?: string; +} + +export const validateContractId = (accountId: string) => { + accountId = accountId.trim(); + if (accountId === WILD_CARD) return true; + + const isLengthValid = accountId.length >= 2 && accountId.length <= 64; + if (!isLengthValid) return false; + + //test if the string starts with a '*.' and remove it if it does + const isWildCard = WILD_CARD_REGEX.test(accountId); + accountId = isWildCard ? accountId.slice(2) : accountId; + + //test if rest of string is valid accounting for/not isWildCard + const isRegexValid = CONTRACT_NAME_REGEX.test(accountId); + return isRegexValid; +}; + +export const validateContractIds = (accountIds: string) => { + const ids = accountIds.split(',').map((id) => id.trim()); + return ids.every((accountId) => validateContractId(accountId)); +}; + +export function validateJSCode(code: string): { data: string | null; error: Error | null } { + if (!code) return { data: null, error: null }; + + try { + const formattedCode = formatIndexingCode(code); + return { data: formattedCode, error: null }; + } catch (error: unknown) { + if (error instanceof Error) { + console.error(error.message); + return { data: code, error }; + } else { + throw error; + } + } +} + +export function validateSQLSchema(schema: string): ValidationResult { + try { + if (!schema || isDefaultSchema(schema)) return { data: null, error: null }; + const formattedSchemaResult = formatSchema(schema); + if (formattedSchemaResult.error) return { data: formattedSchemaResult.data, error: formattedSchemaResult.error }; + + const validationResult = generateTypes(formattedSchemaResult.data); + if (validationResult.error) + return { data: schema, error: validationResult.error, location: validationResult.location }; + + return { data: formattedSchemaResult.data, error: null }; + } catch (error: any) { + return { data: schema, error: new ValidationError(error.message, FORMATTING_ERROR_TYPE) }; + } +} + +export const formatSchema = (schema: string): FormatResult => { + try { + const formattedSchema = formatSQL(schema); + return { data: formattedSchema, error: null }; + } catch (error: any) { + return { + data: schema, + error: new ValidationError(error.message, FORMATTING_ERROR_TYPE), + }; + } +}; + +function generateTypes(formattedSchema: string | null): GenerateTypesResult { + if (!formattedSchema) return { error: null }; + + const pgSchemaTypeGen = new PgSchemaTypeGen(); + + try { + pgSchemaTypeGen.generateTypes(formattedSchema); + return { error: null }; + } catch (error: any) { + const location = error.location ? error.location : undefined; + return { + error: new ValidationError(error.message, TYPE_GENERATION_ERROR_TYPE, error.location), + location: location, + }; + } +} + +export const isDefaultSchema = (schema: string): boolean => { + return schema === formatSQL(defaultSchema); +}; From e3a835bca30300e8dd65300c374f449272d97ea2 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Fri, 28 Jun 2024 08:19:10 +1200 Subject: [PATCH 06/10] fix: Add `catch` blocks to prevent unhandled rejections (#842) Promises without rejection handlers, i.e. `.catch` or `try catch`, will throw "unhandled rejection" errors, which bubble up to the worker thread causing it to exit. This PR adds handlers to the various `simultaneousPromises` triggered within the Executor, to avoid the described. --- runner/src/indexer/indexer.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/runner/src/indexer/indexer.ts b/runner/src/indexer/indexer.ts index 97d9f6289..3d6ad3a9a 100644 --- a/runner/src/indexer/indexer.ts +++ b/runner/src/indexer/indexer.ts @@ -107,7 +107,9 @@ export default class Indexer { }, this.tracer, 'get database connection parameters'); const resourceCreationSpan = this.tracer.startSpan('prepare vm and context to run indexer code'); - simultaneousPromises.push(this.setStatus(IndexerStatus.RUNNING)); + simultaneousPromises.push(this.setStatus(IndexerStatus.RUNNING).catch((e: Error) => { + this.logger.error('Failed to set status to RUNNING', e); + })); const vm = new VM({ allowAsync: true }); const context = this.buildContext(blockHeight, logEntries); @@ -130,10 +132,14 @@ export default class Indexer { runIndexerCodeSpan.end(); } }); - simultaneousPromises.push(this.updateIndexerBlockHeight(blockHeight)); + simultaneousPromises.push(this.updateIndexerBlockHeight(blockHeight).catch((e: Error) => { + this.logger.error('Failed to update block height', e); + })); } catch (e) { // TODO: Prevent unnecesary reruns of set status - simultaneousPromises.push(await this.setStatus(IndexerStatus.FAILING)); + simultaneousPromises.push(await this.setStatus(IndexerStatus.FAILING).catch((e: Error) => { + this.logger.error('Failed to set status to FAILING', e); + })); throw e; } finally { const results = await Promise.allSettled([(this.deps.indexerMeta as IndexerMeta).writeLogs(logEntries), ...simultaneousPromises]); From adaadfc3b2c94ad7bfdf54cd7d9d887b0b55899a Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Fri, 28 Jun 2024 08:48:50 +1200 Subject: [PATCH 07/10] fix: Add metrics for reliably measuring Block Stream/Executor health (#843) The current methods for determining both Block Stream and Executor health is flawed. This PR addresses these flaws by adding new, more reliable, metrics for use within Grafana. ### Block Streams A Block Stream is considered healthy if `LAST_PROCESSED_BLOCK` is continuously incremented, i.e. we are continuously downloading blocks from S3. This is flawed for the following reasons: 1. When the Redis Stream if full, we halt the Block Stream, preventing it from processing more blocks 2. When a Block Stream is intentionally stopped, we no longer process blocks To address these flaws, I've introduced a new dedicated metric: `BLOCK_STREAM_UP`, which: - is incremented every time the Block Stream future is polled, i.e. the task is doing work. A static value means unhealthy. - is removed when the Block Stream is stopped, so that it doesn't trigger the false positive described above ### Executors An Executor is considered unhealthy if: it has messages in the Redis Stream, and no reported execution durations. The latter only being recorded on success. The inverse of this is used to determine "healthy". This is flawed for the following reasons: 1. We distinguish the difference between a genuinely broken Indexer, and one broken due to system failures 2. "health" is only determined when there are messages in Redis, meaning we catch the issue later than possible To address these I have added the following metrics: 1. `EXECUTOR_UP` which is incremented on every Executor loop, like above, a static value means unhealthy. 2. `SUCCESSFUL_EXECUTIONS`/`FAILED_EXECUTIONS` which track successful/failed executions directly, rather than tracking using durations. This will be useful for tracking health of specific Indexers, e.g. the `staking` indexer should never have failed executions. --- block-streamer/Cargo.lock | 1 + block-streamer/Cargo.toml | 1 + block-streamer/src/block_stream.rs | 96 ++++++++++++++----- block-streamer/src/metrics.rs | 6 ++ .../src/server/block_streamer_service.rs | 16 +++- coordinator/Cargo.lock | 9 +- runner/src/metrics.ts | 23 ++++- runner/src/stream-handler/worker.ts | 4 + 8 files changed, 120 insertions(+), 36 deletions(-) diff --git a/block-streamer/Cargo.lock b/block-streamer/Cargo.lock index ce9cb859d..26a1e34ff 100644 --- a/block-streamer/Cargo.lock +++ b/block-streamer/Cargo.lock @@ -980,6 +980,7 @@ dependencies = [ "lazy_static", "mockall", "near-lake-framework", + "pin-project", "prometheus", "prost 0.12.4", "redis", diff --git a/block-streamer/Cargo.toml b/block-streamer/Cargo.toml index f4becc111..d7d4a42f5 100644 --- a/block-streamer/Cargo.toml +++ b/block-streamer/Cargo.toml @@ -19,6 +19,7 @@ graphql_client = { version = "0.14.0", features = ["reqwest"] } lazy_static = "1.4.0" mockall = "0.11.4" near-lake-framework = "0.7.8" +pin-project = "1.1.5" prometheus = "0.13.3" prost = "0.12.3" redis = { version = "0.21.5", features = ["tokio-comp", "connection-manager"] } diff --git a/block-streamer/src/block_stream.rs b/block-streamer/src/block_stream.rs index 4dbb6b0e7..6698881a9 100644 --- a/block-streamer/src/block_stream.rs +++ b/block-streamer/src/block_stream.rs @@ -1,3 +1,7 @@ +use std::future::Future; +use std::pin::Pin; +use std::task::Poll; + use anyhow::Context; use near_lake_framework::near_indexer_primitives; use tokio::task::JoinHandle; @@ -14,6 +18,35 @@ const MAX_STREAM_SIZE_WITH_CACHE: u64 = 100; const DELTA_LAKE_SKIP_ACCOUNTS: [&str; 4] = ["*", "*.near", "*.kaiching", "*.tg"]; const MAX_STREAM_SIZE: u64 = 100; +#[pin_project::pin_project] +pub struct PollCounter { + #[pin] + inner: F, + indexer_name: String, +} + +impl PollCounter { + pub fn new(inner: F, indexer_name: String) -> Self { + Self { + inner, + indexer_name, + } + } +} + +impl Future for PollCounter { + type Output = F::Output; + + fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { + metrics::BLOCK_STREAM_UP + .with_label_values(&[&self.indexer_name]) + .inc(); + + let this = self.project(); + this.inner.poll(cx) + } +} + pub struct Task { handle: JoinHandle>, cancellation_token: tokio_util::sync::CancellationToken, @@ -55,24 +88,15 @@ impl BlockStream { } let cancellation_token = tokio_util::sync::CancellationToken::new(); - let cancellation_token_clone = cancellation_token.clone(); - - let indexer_config = self.indexer_config.clone(); - let chain_id = self.chain_id.clone(); - let redis_stream = self.redis_stream.clone(); - - let handle = tokio::spawn(async move { - tokio::select! { - _ = cancellation_token_clone.cancelled() => { - tracing::info!( - account_id = indexer_config.account_id.as_str(), - function_name = indexer_config.function_name, - "Cancelling block stream task", - ); - - Ok(()) - }, - result = start_block_stream( + + let handle = tokio::spawn({ + let cancellation_token = cancellation_token.clone(); + let indexer_config = self.indexer_config.clone(); + let chain_id = self.chain_id.clone(); + let redis_stream = self.redis_stream.clone(); + + async move { + let block_stream_future = start_block_stream( start_block_height, &indexer_config, redis, @@ -80,17 +104,33 @@ impl BlockStream { lake_s3_client, &chain_id, LAKE_PREFETCH_SIZE, - redis_stream - ) => { - result.map_err(|err| { - tracing::error!( + redis_stream, + ); + + let block_stream_future = + PollCounter::new(block_stream_future, indexer_config.get_full_name()); + + tokio::select! { + _ = cancellation_token.cancelled() => { + tracing::info!( account_id = indexer_config.account_id.as_str(), function_name = indexer_config.function_name, - "Block stream task stopped due to error: {:?}", - err, + "Cancelling block stream task", ); - err - }) + + Ok(()) + }, + result = block_stream_future => { + result.map_err(|err| { + tracing::error!( + account_id = indexer_config.account_id.as_str(), + function_name = indexer_config.function_name, + "Block stream task stopped due to error: {:?}", + err, + ); + err + }) + } } } }); @@ -108,6 +148,10 @@ impl BlockStream { task.cancellation_token.cancel(); let _ = task.handle.await?; + // Fails if metric doesn't exist, i.e. task was never polled + let _ = metrics::BLOCK_STREAM_UP + .remove_label_values(&[&self.indexer_config.get_full_name()]); + return Ok(()); } diff --git a/block-streamer/src/metrics.rs b/block-streamer/src/metrics.rs index 31c4f84c2..1f01a071e 100644 --- a/block-streamer/src/metrics.rs +++ b/block-streamer/src/metrics.rs @@ -57,6 +57,12 @@ lazy_static! { &["level"] ) .unwrap(); + pub static ref BLOCK_STREAM_UP: IntCounterVec = register_int_counter_vec!( + "queryapi_block_streamer_block_stream_up", + "A continuously increasing counter to indicate the block stream is up", + &["indexer"] + ) + .unwrap(); } pub struct LogCounter; diff --git a/block-streamer/src/server/block_streamer_service.rs b/block-streamer/src/server/block_streamer_service.rs index 4c7d2c69d..6a9037047 100644 --- a/block-streamer/src/server/block_streamer_service.rs +++ b/block-streamer/src/server/block_streamer_service.rs @@ -58,6 +58,7 @@ impl BlockStreamerService { #[tonic::async_trait] impl blockstreamer::block_streamer_server::BlockStreamer for BlockStreamerService { + #[tracing::instrument(skip(self))] async fn start_stream( &self, request: Request, @@ -117,7 +118,11 @@ impl blockstreamer::block_streamer_server::BlockStreamer for BlockStreamerServic self.delta_lake_client.clone(), self.lake_s3_client.clone(), ) - .map_err(|_| Status::internal("Failed to start block stream"))?; + .map_err(|err| { + tracing::error!(?err, "Failed to start block stream"); + + Status::internal("Failed to start block stream") + })?; let mut lock = self.get_block_streams_lock()?; lock.insert(indexer_config.get_hash_id(), block_stream); @@ -127,6 +132,7 @@ impl blockstreamer::block_streamer_server::BlockStreamer for BlockStreamerServic })) } + #[tracing::instrument(skip(self))] async fn stop_stream( &self, request: Request, @@ -148,10 +154,10 @@ impl blockstreamer::block_streamer_server::BlockStreamer for BlockStreamerServic ))) } Some(mut block_stream) => { - block_stream - .cancel() - .await - .map_err(|_| Status::internal("Failed to cancel block stream"))?; + block_stream.cancel().await.map_err(|err| { + tracing::error!(?err, "Failed to cancel block stream"); + Status::internal("Failed to cancel block stream") + })?; } } diff --git a/coordinator/Cargo.lock b/coordinator/Cargo.lock index 5b118095c..0e4e5e24f 100644 --- a/coordinator/Cargo.lock +++ b/coordinator/Cargo.lock @@ -939,6 +939,7 @@ dependencies = [ "lazy_static", "mockall", "near-lake-framework", + "pin-project", "prometheus", "prost 0.12.3", "redis 0.21.7", @@ -3133,18 +3134,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.3" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.3" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", diff --git a/runner/src/metrics.ts b/runner/src/metrics.ts index fd99d88e9..493da4119 100644 --- a/runner/src/metrics.ts +++ b/runner/src/metrics.ts @@ -63,6 +63,24 @@ const LOGS_COUNT = new Counter({ labelNames: ['level'], }); +const EXECUTOR_UP = new Counter({ + name: 'queryapi_runner_executor_up', + help: 'Incremented each time the executor loop runs to indicate whether the job is functional', + labelNames: ['indexer'], +}); + +const SUCCESSFUL_EXECUTIONS = new Counter({ + name: 'queryapi_runner_successful_executions', + help: 'Count of successful executions of an indexer function', + labelNames: ['indexer'], +}); + +const FAILED_EXECUTIONS = new Counter({ + name: 'queryapi_runner_failed_executions', + help: 'Count of failed executions of an indexer function', + labelNames: ['indexer'], +}); + export const METRICS = { HEAP_TOTAL_ALLOCATION, HEAP_USED, @@ -73,7 +91,10 @@ export const METRICS = { UNPROCESSED_STREAM_MESSAGES, LAST_PROCESSED_BLOCK_HEIGHT, EXECUTION_DURATION, - LOGS_COUNT + LOGS_COUNT, + EXECUTOR_UP, + SUCCESSFUL_EXECUTIONS, + FAILED_EXECUTIONS, }; const aggregatorRegistry = new AggregatorRegistry(); diff --git a/runner/src/stream-handler/worker.ts b/runner/src/stream-handler/worker.ts index 6be274822..6c0759bb9 100644 --- a/runner/src/stream-handler/worker.ts +++ b/runner/src/stream-handler/worker.ts @@ -101,6 +101,8 @@ async function blockQueueConsumer (workerContext: WorkerContext): Promise let currBlockHeight = 0; while (true) { + METRICS.EXECUTOR_UP.labels({ indexer: indexerConfig.fullName() }).inc(); + if (workerContext.queue.length === 0) { await sleep(100); continue; @@ -150,9 +152,11 @@ async function blockQueueConsumer (workerContext: WorkerContext): Promise executionDurationTimer(); + METRICS.SUCCESSFUL_EXECUTIONS.labels({ indexer: indexerConfig.fullName() }).inc(); METRICS.LAST_PROCESSED_BLOCK_HEIGHT.labels({ indexer: indexerConfig.fullName() }).set(currBlockHeight); postRunSpan.end(); } catch (err) { + METRICS.FAILED_EXECUTIONS.labels({ indexer: indexerConfig.fullName() }).inc(); parentSpan.setAttribute('status', 'failed'); parentPort?.postMessage({ type: WorkerMessageType.STATUS, data: { status: IndexerStatus.FAILING } }); const error = err as Error; From a73331a70a6ae65e826a9c26515ca37991e01128 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Fri, 28 Jun 2024 12:49:41 +1200 Subject: [PATCH 08/10] fix: Report worker metrics even when no messages in Stream (#844) We skip reporting metrics if there are no messages in the pre-fetch queue/Redis Stream. This is especially problematic for `EXECUTOR_UP`, as we won't increment the metric even though we are processing. This PR moves the metrics logic so that it is always reported, even when no messages in the stream. --- runner/src/stream-handler/worker.ts | 30 +++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/runner/src/stream-handler/worker.ts b/runner/src/stream-handler/worker.ts index 6c0759bb9..d9e77184d 100644 --- a/runner/src/stream-handler/worker.ts +++ b/runner/src/stream-handler/worker.ts @@ -103,10 +103,26 @@ async function blockQueueConsumer (workerContext: WorkerContext): Promise while (true) { METRICS.EXECUTOR_UP.labels({ indexer: indexerConfig.fullName() }).inc(); + const metricsSpan = tracer.startSpan('Record metrics after processing block', {}, context.active()); + + const unprocessedMessageCount = await workerContext.redisClient.getUnprocessedStreamMessageCount(indexerConfig.redisStreamKey); + METRICS.UNPROCESSED_STREAM_MESSAGES.labels({ indexer: indexerConfig.fullName() }).set(unprocessedMessageCount); + + const memoryUsage = process.memoryUsage(); + METRICS.HEAP_TOTAL_ALLOCATION.labels({ indexer: indexerConfig.fullName() }).set(memoryUsage.heapTotal / (1024 * 1024)); + METRICS.HEAP_USED.labels({ indexer: indexerConfig.fullName() }).set(memoryUsage.heapUsed / (1024 * 1024)); + METRICS.PREFETCH_QUEUE_COUNT.labels({ indexer: indexerConfig.fullName() }).set(workerContext.queue.length); + + const metricsMessage: WorkerMessage = { type: WorkerMessageType.METRICS, data: await promClient.register.getMetricsAsJSON() }; + parentPort?.postMessage(metricsMessage); + + metricsSpan.end(); + if (workerContext.queue.length === 0) { await sleep(100); continue; } + await tracer.startActiveSpan(`${indexerConfig.fullName()}`, async (parentSpan: Span) => { parentSpan.setAttribute('indexer', indexerConfig.fullName()); parentSpan.setAttribute('account', indexerConfig.accountId); @@ -168,20 +184,6 @@ async function blockQueueConsumer (workerContext: WorkerContext): Promise await sleep(10000); sleepSpan.end(); } finally { - const metricsSpan = tracer.startSpan('Record metrics after processing block', {}, context.active()); - - const unprocessedMessageCount = await workerContext.redisClient.getUnprocessedStreamMessageCount(indexerConfig.redisStreamKey); - METRICS.UNPROCESSED_STREAM_MESSAGES.labels({ indexer: indexerConfig.fullName() }).set(unprocessedMessageCount); - - const memoryUsage = process.memoryUsage(); - METRICS.HEAP_TOTAL_ALLOCATION.labels({ indexer: indexerConfig.fullName() }).set(memoryUsage.heapTotal / (1024 * 1024)); - METRICS.HEAP_USED.labels({ indexer: indexerConfig.fullName() }).set(memoryUsage.heapUsed / (1024 * 1024)); - METRICS.PREFETCH_QUEUE_COUNT.labels({ indexer: indexerConfig.fullName() }).set(workerContext.queue.length); - - const metricsMessage: WorkerMessage = { type: WorkerMessageType.METRICS, data: await promClient.register.getMetricsAsJSON() }; - parentPort?.postMessage(metricsMessage); - - metricsSpan.end(); parentSpan.end(); } }); From 791afc2f6e3e32018408969b22f35b50b38dbedf Mon Sep 17 00:00:00 2001 From: Kevin Zhang <42101107+Kevin101Zhang@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:18:48 -0400 Subject: [PATCH 09/10] feat: line position now persist for indexer.js when switching tabs (#835) Added logic to monaco lifecycle method and react lifecycle methods to persist line/cursor position between the swapped files. --- .../Editor/EditorComponents/Editor.tsx | 58 ++++++++++++++++--- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/frontend/src/components/Editor/EditorComponents/Editor.tsx b/frontend/src/components/Editor/EditorComponents/Editor.tsx index 836eea02a..c3a33c0a3 100644 --- a/frontend/src/components/Editor/EditorComponents/Editor.tsx +++ b/frontend/src/components/Editor/EditorComponents/Editor.tsx @@ -50,7 +50,7 @@ const Editor: React.FC = (): ReactElement => { const [indexingCode, setIndexingCode] = useState(originalIndexingCode); const [schema, setSchema] = useState(originalSQLCode); - const [cursorPostion, setCursorPosition] = useState<{ lineNumber: number; column: number }>({ + const [cursorPosition, setCursorPosition] = useState<{ lineNumber: number; column: number }>({ lineNumber: 1, column: 1, }); @@ -164,17 +164,37 @@ const Editor: React.FC = (): ReactElement => { }, [fileName]); useEffect(() => { - const savedCode = storageManager?.getIndexerCode(); - const savedSchema = storageManager?.getSchemaCode(); - if (savedSchema) setSchema(savedSchema); - if (savedCode) setIndexingCode(savedCode); + if (storageManager) { + const savedSchema = storageManager.getSchemaCode(); + const savedIndexingCode = storageManager.getIndexerCode(); + const savedCursorPosition = storageManager.getCursorPosition(); + + if (savedSchema) setSchema(savedSchema); + if (savedIndexingCode) setIndexingCode(savedIndexingCode); + if (savedCursorPosition) setCursorPosition(savedCursorPosition); + + if (monacoEditorRef.current && fileName === INDEXER_TAB_NAME) { + monacoEditorRef.current.setValue(savedIndexingCode || ''); + monacoEditorRef.current.setPosition(savedCursorPosition || { lineNumber: 1, column: 1 }); + monacoEditorRef.current.focus(); + } + } }, [indexerDetails.accountId, indexerDetails.indexerName]); useEffect(() => { - storageManager?.setSchemaCode(schema); - storageManager?.setIndexerCode(indexingCode); - storageManager?.setCursorPosition(cursorPostion); - }, [schema, indexingCode, cursorPostion]); + cacheToLocal(); + }, [indexingCode, schema]); + + useEffect(() => { + if (!monacoEditorRef.current) return; + + const editorInstance = monacoEditorRef.current; + editorInstance.onDidChangeCursorPosition(handleCursorChange); + + return () => { + editorInstance.dispose(); + }; + }, [monacoEditorRef.current]); useEffect(() => { storageManager?.setSchemaTypes(schemaTypes); @@ -185,6 +205,23 @@ const Editor: React.FC = (): ReactElement => { storageManager?.setDebugList(heights); }, [heights]); + const cacheToLocal = () => { + if (!storageManager || !monacoEditorRef.current) return; + + storageManager.setSchemaCode(schema); + storageManager.setIndexerCode(indexingCode); + + const newCursorPosition = monacoEditorRef.current.getPosition(); + storageManager.setCursorPosition(newCursorPosition); + }; + + const handleCursorChange = () => { + if (monacoEditorRef.current && fileName === INDEXER_TAB_NAME) { + const position = monacoEditorRef.current.getPosition(); + setCursorPosition(position); + } + }; + const attachTypesToMonaco = () => { // If types have been added already, dispose of them first if (disposableRef.current) { @@ -261,6 +298,9 @@ const Editor: React.FC = (): ReactElement => { ); monacoEditorRef.current = editor; setDecorations(decorations); + + editor.setPosition(fileName === INDEXER_TAB_NAME ? cursorPosition : { lineNumber: 1, column: 1 }); + editor.focus(); } monaco.languages.typescript.typescriptDefaults.addExtraLib( `${primitives}}`, From a9c25c3cbfe6511cec3d4d0b51a3e064c231fd06 Mon Sep 17 00:00:00 2001 From: Kevin Zhang <42101107+Kevin101Zhang@users.noreply.github.com> Date: Tue, 2 Jul 2024 18:25:58 -0400 Subject: [PATCH 10/10] feat: enable scroll past last line in monaco (#846) set scroll past last line in monaco flag to true --- .../Editor/EditorComponents/ResizableLayoutEditor.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/Editor/EditorComponents/ResizableLayoutEditor.jsx b/frontend/src/components/Editor/EditorComponents/ResizableLayoutEditor.jsx index ef31508e2..f9c60ba92 100644 --- a/frontend/src/components/Editor/EditorComponents/ResizableLayoutEditor.jsx +++ b/frontend/src/components/Editor/EditorComponents/ResizableLayoutEditor.jsx @@ -76,7 +76,7 @@ const ResizableEditor = ({ minimap: { enabled: false }, folding: false, lineNumberMinChars: 3, - scrollBeyondLastLine: false, + scrollBeyondLastLine: true, automaticLayout: true, formatOnPaste: true, definitionLinkOpensInPeek: true, @@ -109,7 +109,7 @@ const ResizableEditor = ({ minimap: { enabled: false }, folding: false, lineNumberMinChars: 3, - scrollBeyondLastLine: false, + scrollBeyondLastLine: true, automaticLayout: true, formatOnPaste: true, definitionLinkOpensInPeek: true,