From ea08366f9343f53f906d50965bb4c8f2585c024d Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Tue, 3 Oct 2023 15:37:31 -0300 Subject: [PATCH 1/5] feat(sinks): add log on receiveing message from maestro. --- sinks/redis/consumer/sink_status_listener.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sinks/redis/consumer/sink_status_listener.go b/sinks/redis/consumer/sink_status_listener.go index fa4374891..fd47601cd 100644 --- a/sinks/redis/consumer/sink_status_listener.go +++ b/sinks/redis/consumer/sink_status_listener.go @@ -72,6 +72,8 @@ func (s *sinkStatusListener) ReceiveMessage(ctx context.Context, message redis.X logger := s.logger.Named(fmt.Sprintf("sink_status_msg:%s", message.ID)) go func(ctx context.Context, logger *zap.Logger, message redis.XMessage) { event := s.decodeMessage(message.Values) + logger.Info("received message from maestro", zap.String("owner_id", event.OwnerID), + zap.String("sink_id", event.SinkID), zap.String("state", event.State), zap.String("msg", event.Msg)) gotSink, err := s.sinkService.ViewSinkInternal(ctx, event.OwnerID, event.SinkID) if err != nil { logger.Error("failed to get sink for sink_id from message", zap.String("owner_id", event.OwnerID), From f2436a048386ebd16fd57ab5bc78054db3fa6298 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Tue, 3 Oct 2023 15:59:18 -0300 Subject: [PATCH 2/5] feat(sinks): add log on receiveing message from maestro. --- sinks/redis/consumer/sink_status_listener.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sinks/redis/consumer/sink_status_listener.go b/sinks/redis/consumer/sink_status_listener.go index fd47601cd..c24831c11 100644 --- a/sinks/redis/consumer/sink_status_listener.go +++ b/sinks/redis/consumer/sink_status_listener.go @@ -54,6 +54,9 @@ func (s *sinkStatusListener) SubscribeToMaestroSinkStatus(ctx context.Context) e Count: 1000, }).Result() if err != nil || len(streams) == 0 { + if err != nil { + rLogger.Error("failed to read group", zap.Error(err)) + } continue } for _, msg := range streams[0].Messages { From f09969e9159a1d785657a4d3371220785ba723cf Mon Sep 17 00:00:00 2001 From: "Everton H. Taques" <97463920+etaques@users.noreply.github.com> Date: Tue, 3 Oct 2023 16:35:59 -0300 Subject: [PATCH 3/5] fix: (maestro) monitor to kill orphaned otelcollectors (#2705) * set sink status when deployment status change * fix: (maestro) monitor to kill orphaned otelcollectors * fix: (sinks) update state --------- Co-authored-by: etaques --- maestro/monitor/monitor.go | 7 ++++--- sinks/redis/consumer/sink_status_listener.go | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/maestro/monitor/monitor.go b/maestro/monitor/monitor.go index b6e20463a..8e55b7ba7 100644 --- a/maestro/monitor/monitor.go +++ b/maestro/monitor/monitor.go @@ -5,12 +5,13 @@ import ( "context" "encoding/json" "errors" - "github.com/orb-community/orb/maestro/deployment" - "github.com/orb-community/orb/maestro/redis/producer" "io" "strings" "time" + "github.com/orb-community/orb/maestro/deployment" + "github.com/orb-community/orb/maestro/redis/producer" + maestroconfig "github.com/orb-community/orb/maestro/config" "github.com/orb-community/orb/maestro/kubecontrol" sinkspb "github.com/orb-community/orb/sinks/pb" @@ -172,7 +173,7 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { deploymentName := "otel-" + sinkId svc.logger.Debug("compare deploymentName with collector name", zap.String("deploy name", deploymentName), zap.String("collector name", collector.Name)) - err = svc.kubecontrol.KillOtelCollector(ctx, collector.Name, sinkId) + err = svc.kubecontrol.KillOtelCollector(ctx, deploymentName, sinkId) if err != nil { svc.logger.Error("error removing otel collector", zap.Error(err)) } diff --git a/sinks/redis/consumer/sink_status_listener.go b/sinks/redis/consumer/sink_status_listener.go index fd47601cd..efd3a77b1 100644 --- a/sinks/redis/consumer/sink_status_listener.go +++ b/sinks/redis/consumer/sink_status_listener.go @@ -3,6 +3,7 @@ package consumer import ( "context" "fmt" + "github.com/go-redis/redis/v8" "github.com/orb-community/orb/sinks" redis2 "github.com/orb-community/orb/sinks/redis" @@ -84,6 +85,7 @@ func (s *sinkStatusListener) ReceiveMessage(ctx context.Context, message redis.X if newState == sinks.Error || newState == sinks.ProvisioningError || newState == sinks.Warning { gotSink.Error = event.Msg } + gotSink.State = newState _, err = s.sinkService.UpdateSinkInternal(ctx, gotSink) if err != nil { logger.Error("failed to update sink", zap.String("owner_id", event.OwnerID), From bed581ab389ba4d25d7c1c92ac7cbc6beff4be67 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Tue, 3 Oct 2023 16:49:27 -0300 Subject: [PATCH 4/5] feat(sinker): add debounce of 1 min per sinker to not publish every metric on redis sink_activity. --- sinker/otel/bridgeservice/bridge.go | 33 ++++++++++++------- .../kafkafanoutexporter/kafka_exporter.go | 6 ++-- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/sinker/otel/bridgeservice/bridge.go b/sinker/otel/bridgeservice/bridge.go index f9602db07..bd348758e 100644 --- a/sinker/otel/bridgeservice/bridge.go +++ b/sinker/otel/bridgeservice/bridge.go @@ -67,19 +67,28 @@ func (bs *SinkerOtelBridgeService) IncrementMessageCounter(publisher, subtopic, // NotifyActiveSink notify the sinker that a sink is active func (bs *SinkerOtelBridgeService) NotifyActiveSink(ctx context.Context, mfOwnerId, sinkId, size string) error { - bs.logger.Debug("notifying active sink", zap.String("sink_id", sinkId), zap.String("owner_id", mfOwnerId), - zap.String("payload_size", size)) - event := producer.SinkActivityEvent{ - OwnerID: mfOwnerId, - SinkID: sinkId, - State: "active", - Size: size, - Timestamp: time.Now(), - } - err := bs.sinkerActivitySvc.PublishSinkActivity(ctx, event) - if err != nil { - bs.logger.Error("error publishing sink activity", zap.Error(err)) + cacheKey := fmt.Sprintf("active_sink-%s-%s", mfOwnerId, sinkId) + _, found := bs.inMemoryCache.Get(cacheKey) + if !found { + bs.logger.Debug("notifying active sink", zap.String("sink_id", sinkId), zap.String("owner_id", mfOwnerId), + zap.String("payload_size", size)) + event := producer.SinkActivityEvent{ + OwnerID: mfOwnerId, + SinkID: sinkId, + State: "active", + Size: size, + Timestamp: time.Now(), + } + err := bs.sinkerActivitySvc.PublishSinkActivity(ctx, event) + if err != nil { + bs.logger.Error("error publishing sink activity", zap.Error(err)) + } + bs.inMemoryCache.Set(cacheKey, true, cache.DefaultExpiration) + } else { + bs.logger.Debug("active sink already notified", zap.String("sink_id", sinkId), zap.String("owner_id", mfOwnerId), + zap.String("payload_size", size)) } + return nil } diff --git a/sinker/otel/kafkafanoutexporter/kafka_exporter.go b/sinker/otel/kafkafanoutexporter/kafka_exporter.go index d07474126..cfb252d51 100644 --- a/sinker/otel/kafkafanoutexporter/kafka_exporter.go +++ b/sinker/otel/kafkafanoutexporter/kafka_exporter.go @@ -50,7 +50,7 @@ func (ke kafkaErrors) Error() string { func (e *kafkaTracesProducer) tracesPusher(ctx context.Context, td ptrace.Traces) error { sinkId := ctx.Value("sink_id").(string) topic := e.topic + "-" + sinkId - e.logger.Info("Pushing traces to kafka topic = " + topic) + e.logger.Debug("Pushing traces to kafka topic = " + topic) messages, err := e.marshaler.Marshal(td, topic) if err != nil { return consumererror.NewPermanent(err) @@ -83,7 +83,7 @@ type kafkaMetricsProducer struct { func (e *kafkaMetricsProducer) metricsDataPusher(ctx context.Context, md pmetric.Metrics) error { sinkId := ctx.Value("sink_id").(string) topic := e.topic + "-" + sinkId - e.logger.Info("Pushing metrics to kafka topic = " + topic) + e.logger.Debug("Pushing metrics to kafka topic = " + topic) messages, err := e.marshaler.Marshal(md, topic) if err != nil { return consumererror.NewPermanent(err) @@ -116,7 +116,7 @@ type kafkaLogsProducer struct { func (e *kafkaLogsProducer) logsDataPusher(ctx context.Context, ld plog.Logs) error { sinkId := ctx.Value("sink_id").(string) topic := e.topic + "-" + sinkId - e.logger.Info("Pushing logs to kafka topic = " + topic) + e.logger.Debug("Pushing logs to kafka topic = " + topic) messages, err := e.marshaler.Marshal(ld, topic) if err != nil { return consumererror.NewPermanent(err) From 65de143dbff3efd8b2f96e521487448a2b9a89aa Mon Sep 17 00:00:00 2001 From: joaoguilherme2003 Date: Tue, 3 Oct 2023 17:42:39 -0300 Subject: [PATCH 5/5] fix(orb-ui): improvements on view headers (#2706) --- ui/src/app/@theme/styles/_overrides.scss | 23 ++++- .../agents/view/agent.view.component.html | 59 +++++++------ .../agents/view/agent.view.component.scss | 29 +++++-- .../pages/sinks/view/sink.view.component.html | 85 +++++++++---------- .../pages/sinks/view/sink.view.component.scss | 18 +++- 5 files changed, 129 insertions(+), 85 deletions(-) diff --git a/ui/src/app/@theme/styles/_overrides.scss b/ui/src/app/@theme/styles/_overrides.scss index 5636b7ee0..3ed4e8c5a 100644 --- a/ui/src/app/@theme/styles/_overrides.scss +++ b/ui/src/app/@theme/styles/_overrides.scss @@ -199,13 +199,13 @@ } .orb-service- { - &new { + &new, &unknown { color: #9b51e0; } - &online, &healthy { + &online, &healthy, &active { color: #6fcf97; } - &stale { + &stale, &idle { color: #f2994a; } &error, &failure { @@ -215,6 +215,23 @@ color: #969fb9; } } +.orb-service-background- { + &new, &unknown { + background-color: #9b51e0; + } + &online, &healthy, &active { + background-color: #6fcf97; + } + &stale, &idle { + background-color: #f2994a; + } + &error, &failure { + background-color: #df316f; + } + &offline, &none { + background-color: #969fb9; + } +} .required { color: #df316f; padding-left: 2px; diff --git a/ui/src/app/pages/fleet/agents/view/agent.view.component.html b/ui/src/app/pages/fleet/agents/view/agent.view.component.html index 4727c52b5..59da9207e 100644 --- a/ui/src/app/pages/fleet/agents/view/agent.view.component.html +++ b/ui/src/app/pages/fleet/agents/view/agent.view.component.html @@ -4,36 +4,41 @@

Agent View

-
- -
-
- -
-
-
- - - - {{ agent?.state | ngxCapitalize }} - +
+ +
+
-
- - Last activity - - today, at {{ agent?.ts_last_hb | date: 'HH:mm z' }} +
+
+ + + + {{ agent?.state | ngxCapitalize }} - - on {{ agent?.ts_last_hb | date: 'M/d/yy, HH:mm z' }} +
+
+ + Last activity + + today, at {{ agent?.ts_last_hb | date: 'HH:mm z' }} + + + on {{ agent?.ts_last_hb | date: 'M/d/yy, HH:mm z' }} + - - - This Agent has been provisioned but never connected. - + + This Agent has been provisioned but never connected. + +
diff --git a/ui/src/app/pages/fleet/agents/view/agent.view.component.scss b/ui/src/app/pages/fleet/agents/view/agent.view.component.scss index 399a8c231..682b95a3d 100644 --- a/ui/src/app/pages/fleet/agents/view/agent.view.component.scss +++ b/ui/src/app/pages/fleet/agents/view/agent.view.component.scss @@ -202,18 +202,31 @@ nb-card { color: #969fb9; font-size: 14px; } -.state { - font-size: 15px; - font-weight: 700; -} -.fa.fa-circle { - font-size: 11px; + +.state-circle { + width: 9px; + height: 9px; + border-radius: 50%; } .offline-circle { - width: 10px; - height: 10px; + width: 9px; + height: 9px; border: 2px solid #969fb9; border-radius: 50%; background-color: transparent; } +.state { + font-size: 15px; + font-weight: 700; + font-family: 'Montserrat'; +} +.state-div { + margin-bottom: 23px; +} +.date { + font-size: 14px; + font-weight: 400; + margin-top: 23px; + line-height: 1.25rem; +} diff --git a/ui/src/app/pages/sinks/view/sink.view.component.html b/ui/src/app/pages/sinks/view/sink.view.component.html index fa8897aa3..8143a2a78 100644 --- a/ui/src/app/pages/sinks/view/sink.view.component.html +++ b/ui/src/app/pages/sinks/view/sink.view.component.html @@ -12,53 +12,50 @@

{{ strings.sink.view.header }}

- -
-
- - - -
-
-
- - - {{ sink?.state | ngxCapitalize }} - + +
+ + +
-
- - Created on {{ sink?.ts_created | date: 'M/d/yy, HH:mm z' }} - +
+
+ + + {{ sink?.state | ngxCapitalize }} + +
+
+ + Created on {{ sink?.ts_created | date: 'M/d/yy, HH:mm z' }} + +
-
diff --git a/ui/src/app/pages/sinks/view/sink.view.component.scss b/ui/src/app/pages/sinks/view/sink.view.component.scss index c4a64214a..8c018adb6 100644 --- a/ui/src/app/pages/sinks/view/sink.view.component.scss +++ b/ui/src/app/pages/sinks/view/sink.view.component.scss @@ -96,12 +96,15 @@ h4 { } } } -.fa.fa-circle { - font-size: 11px; +.state-circle { + width: 9px; + height: 9px; + border-radius: 50%; } .state { - font-size: 16px; + font-size: 15px; font-weight: 700; + font-family: 'Montserrat'; } .orb-service- { &active { @@ -122,4 +125,13 @@ h4 { color: #969fb9; font-size: 14px; } +.state-div { + margin-bottom: 23px; +} +.date { + font-size: 14px; + font-weight: 400; + margin-top: 23px; + line-height: 1.25rem; +}