From 5d92e8b36933f2cb908bc43553daf3343679ca5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EC=A7=80=EC=9B=90?= Date: Mon, 28 Oct 2024 13:58:37 +0900 Subject: [PATCH] refactor : http to grpc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 기존 HTTP 로 받던 Trace 및 Metric 데이터를 GRPC로 리팩토링 --- .gitignore | 4 +- Makefile | 8 +- {es => infra/es}/Dockerfile | 0 {es => infra/es}/docker-compose.yml | 0 {kafka => infra/kafka}/common.yml | 0 {kafka => infra/kafka}/kafka_cluster.yml | 0 {kafka => infra/kafka}/zookeeper.yml | 0 infra/nginx/conf.d/default.conf | 44 +++++++++ infra/nginx/docker-compose.yml | 9 ++ tracedin-application/app-api/build.gradle | 50 +++++++++- .../api/metric/grpc/ServiceMetricsGrpc.java | 73 ++++++++++++++ .../api/span/dto/AppendSpanRequest.java | 5 +- .../univ/tracedin/api/span/grpc/SpanGrpc.java | 95 +++++++++++++++++++ .../src/main/proto/service-metrics.proto | 31 ++++++ .../app-api/src/main/proto/span.proto | 47 +++++++++ .../src/main/resources/application.yml | 3 + .../domain/anomaly/AnomalyTraceProcessor.java | 17 ++-- .../univ/tracedin/domain/span/SpanType.java | 2 +- ...SpanElasticSearchRepositoryCustomImpl.java | 2 + 19 files changed, 375 insertions(+), 15 deletions(-) rename {es => infra/es}/Dockerfile (100%) rename {es => infra/es}/docker-compose.yml (100%) rename {kafka => infra/kafka}/common.yml (100%) rename {kafka => infra/kafka}/kafka_cluster.yml (100%) rename {kafka => infra/kafka}/zookeeper.yml (100%) create mode 100644 infra/nginx/conf.d/default.conf create mode 100644 infra/nginx/docker-compose.yml create mode 100644 tracedin-application/app-api/src/main/java/com/univ/tracedin/api/metric/grpc/ServiceMetricsGrpc.java create mode 100644 tracedin-application/app-api/src/main/java/com/univ/tracedin/api/span/grpc/SpanGrpc.java create mode 100644 tracedin-application/app-api/src/main/proto/service-metrics.proto create mode 100644 tracedin-application/app-api/src/main/proto/span.proto diff --git a/.gitignore b/.gitignore index 4c5a7e6..eecacce 100644 --- a/.gitignore +++ b/.gitignore @@ -5,8 +5,8 @@ build/ !**/src/main/**/build/ !**/src/test/**/build/ -/kafka/volumes -/kafka/.env +infra/kafka/volumes +infra/kafka/.env ### STS ### .apt_generated .classpath diff --git a/Makefile b/Makefile index 05e8ecb..69ba563 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ start: - docker compose -f kafka/common.yml -f kafka/zookeeper.yml -f kafka/kafka_cluster.yml up -d - docker compose -f es/docker-compose.yml up -d + docker compose -f infra/kafka/common.yml -f infra/kafka/zookeeper.yml -f infra/kafka/kafka_cluster.yml up -d + docker compose -f infra/es/docker-compose.yml up -d stop: - docker compose -f kafka/common.yml -f kafka/zookeeper.yml -f kafka/kafka_cluster.yml down - docker compose -f es/docker-compose.yml down \ No newline at end of file + docker compose -f infra/kafka/common.yml -f infra/kafka/zookeeper.yml -f infra/kafka/kafka_cluster.yml down + docker compose -f infra/es/docker-compose.yml down \ No newline at end of file diff --git a/es/Dockerfile b/infra/es/Dockerfile similarity index 100% rename from es/Dockerfile rename to infra/es/Dockerfile diff --git a/es/docker-compose.yml b/infra/es/docker-compose.yml similarity index 100% rename from es/docker-compose.yml rename to infra/es/docker-compose.yml diff --git a/kafka/common.yml b/infra/kafka/common.yml similarity index 100% rename from kafka/common.yml rename to infra/kafka/common.yml diff --git a/kafka/kafka_cluster.yml b/infra/kafka/kafka_cluster.yml similarity index 100% rename from kafka/kafka_cluster.yml rename to infra/kafka/kafka_cluster.yml diff --git a/kafka/zookeeper.yml b/infra/kafka/zookeeper.yml similarity index 100% rename from kafka/zookeeper.yml rename to infra/kafka/zookeeper.yml diff --git a/infra/nginx/conf.d/default.conf b/infra/nginx/conf.d/default.conf new file mode 100644 index 0000000..5acef4b --- /dev/null +++ b/infra/nginx/conf.d/default.conf @@ -0,0 +1,44 @@ + listen 80; + http2 on; + server_name tracedin.p-e.kr; + server_tokens off; + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + + location / { + return 301 https://$server_name$request_uri; + } +} + +server { + listen 443 ssl; + http2 on; + server_name tracedin.p-e.kr; + server_tokens off; + + access_log /var/log/nginx/access.log main; + error_log /var/log/nginx/error.log; + + ssl_certificate /etc/letsencrypt/live/tracedin.p-e.kr/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/tracedin.p-e.kr/privkey.pem; # managed by Certbot + include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + + location / { + proxy_pass http://tracedin-green; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_set_header Connection ''; + proxy_http_version 1.1; + } + + location /SpanGrpcAppender/AppendSpans { + grpc_pass grpc://grpcserver; + } + + location /ServiceMetricsGrpcAppender/AppendServiceMetrics { + grpc_pass grpc://grpcserver; + } +} \ No newline at end of file diff --git a/infra/nginx/docker-compose.yml b/infra/nginx/docker-compose.yml new file mode 100644 index 0000000..dfeb8c0 --- /dev/null +++ b/infra/nginx/docker-compose.yml @@ -0,0 +1,9 @@ +services: + nginx: + image: nginx:latest + container_name: nginx + ports: + - "80:80" + volumes: + - ./conf.d:/etc/nginx/conf.d # Nginx 설정 파일 + - ./nginx/log:/var/log/nginx # 로그 파일 diff --git a/tracedin-application/app-api/build.gradle b/tracedin-application/app-api/build.gradle index 21c3fad..e92c9fd 100644 --- a/tracedin-application/app-api/build.gradle +++ b/tracedin-application/app-api/build.gradle @@ -1,3 +1,17 @@ +buildscript { + ext { + protobufVersion = '3.21.5' + protobufPluginVersion = '0.9.4' + grpcVersion = '1.65.1' + } +} + + +plugins { + // Protobuf 플러그인을 적용하여 .proto 파일을 컴파일할 수 있다. 여기서 버전은 ext에 정의된 protobufPluginVersion을 사용한다. + id 'com.google.protobuf' version "${protobufPluginVersion}" +} + dependencies { implementation project(':tracedin-domain') @@ -22,4 +36,38 @@ dependencies { //web implementation 'org.springframework.boot:spring-boot-starter-web' -} \ No newline at end of file + + /// grpc 서버, 클라이언트 설정 + implementation 'net.devh:grpc-spring-boot-starter:3.1.0.RELEASE' + // Spring Boot와 gRPC의 통합을 간편하게 도와주는 스타터 + implementation "io.grpc:grpc-netty-shaded:${grpcVersion}" + // Netty Shaded 사용(gRPC 서버와 클라이언트의 Netty 전송 계층을 제공) + implementation "io.grpc:grpc-protobuf:${grpcVersion}" // Protobuf 메시지와 gRPC의 통합을 지원 + implementation "io.grpc:grpc-stub:${grpcVersion}" // gRPC 클라이언트 스텁을 생성 + compileOnly 'org.apache.tomcat:annotations-api:6.0.53' + // 이걸 추가해야 gRPC 컴파일시 javax 어노테이션 오류가 발생하지 않는다. +} + +protobuf { + // Protobuf 컴파일러를 지정하여 .proto 파일을 컴파일합니다. + protoc { + artifact = "com.google.protobuf:protoc:${protobufVersion}" + } + // 생성된 파일을 정리합니다. + clean { + delete generatedFilesBaseDir + } + // gRPC 플러그인을 설정하여 Protobuf 파일로부터 gRPC 관련 코드를 생성합니다. + plugins { + grpc { + artifact = "io.grpc:protoc-gen-grpc-java:${grpcVersion}" + } + } + // 모든 프로토콜 버퍼 작업에 대해 gRPC 플러그인을 적용합니다. + generateProtoTasks { + all()*.plugins { + grpc {} + } + } +} + diff --git a/tracedin-application/app-api/src/main/java/com/univ/tracedin/api/metric/grpc/ServiceMetricsGrpc.java b/tracedin-application/app-api/src/main/java/com/univ/tracedin/api/metric/grpc/ServiceMetricsGrpc.java new file mode 100644 index 0000000..0e84378 --- /dev/null +++ b/tracedin-application/app-api/src/main/java/com/univ/tracedin/api/metric/grpc/ServiceMetricsGrpc.java @@ -0,0 +1,73 @@ +package com.univ.tracedin.api.metric.grpc; + +import java.time.LocalDateTime; +import java.util.Map; +import java.util.stream.Collectors; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import com.univ.tracedin.api.metric.grpc.ServiceMetricsGrpcAppenderGrpc.ServiceMetricsGrpcAppenderImplBase; +import com.univ.tracedin.api.metric.grpc.ServiceMetricsProto.AppendServiceMetricsRequest; +import com.univ.tracedin.api.metric.grpc.ServiceMetricsProto.AppendServiceMetricsResponse; +import com.univ.tracedin.api.metric.grpc.ServiceMetricsProto.MetricRequest; +import com.univ.tracedin.domain.metric.Metric; +import com.univ.tracedin.domain.metric.MetricType; +import com.univ.tracedin.domain.metric.ServiceMetrics; +import com.univ.tracedin.domain.metric.ServiceMetricsService; +import com.univ.tracedin.domain.project.ProjectKey; + +import io.grpc.stub.StreamObserver; +import net.devh.boot.grpc.server.service.GrpcService; + +@Slf4j +@RequiredArgsConstructor +@GrpcService +public class ServiceMetricsGrpc extends ServiceMetricsGrpcAppenderImplBase { + + private final ServiceMetricsService serviceMetricsService; + + @Override + public void appendServiceMetrics( + AppendServiceMetricsRequest request, + StreamObserver responseObserver) { + try { + log.info("appendServiceMetrics request: {}", request.toString()); + serviceMetricsService.appendMetrics(toServiceMetrics(request)); + responseObserver.onNext( + AppendServiceMetricsResponse.newBuilder().setStatusCode(200).build()); + } catch (Exception e) { + log.error("Failed to append service metrics", e); + responseObserver.onNext( + AppendServiceMetricsResponse.newBuilder().setStatusCode(500).build()); + } finally { + responseObserver.onCompleted(); + } + } + + private ServiceMetrics toServiceMetrics(AppendServiceMetricsRequest request) { + return ServiceMetrics.builder() + .projectKey(ProjectKey.from(request.getProjectKey())) + .serviceName(request.getServiceName()) + .metrics(request.getMetricsList().stream().map(this::toMetric).toList()) + .build(); + } + + private Metric toMetric(MetricRequest metricRequest) { + return Metric.builder() + .name(metricRequest.getName()) + .description(metricRequest.getDescription()) + .unit(metricRequest.getUnit()) + .type(MetricType.fromValue(metricRequest.getType())) + .value(metricRequest.getValue()) + .count(metricRequest.getCount()) + .sum(metricRequest.getSum()) + .min(metricRequest.getMin()) + .max(metricRequest.getMax()) + .attributes( + metricRequest.getAttributesMap().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))) + .timestamp(LocalDateTime.now()) + .build(); + } +} diff --git a/tracedin-application/app-api/src/main/java/com/univ/tracedin/api/span/dto/AppendSpanRequest.java b/tracedin-application/app-api/src/main/java/com/univ/tracedin/api/span/dto/AppendSpanRequest.java index 72637c9..19f9b4c 100644 --- a/tracedin-application/app-api/src/main/java/com/univ/tracedin/api/span/dto/AppendSpanRequest.java +++ b/tracedin-application/app-api/src/main/java/com/univ/tracedin/api/span/dto/AppendSpanRequest.java @@ -13,6 +13,8 @@ import com.univ.tracedin.domain.span.SpanType; import com.univ.tracedin.domain.span.TraceId; +import io.micrometer.common.util.StringUtils; + public record AppendSpanRequest( String serviceName, String projectKey, @@ -68,7 +70,8 @@ public Span toSpan() { } private SpanType getSpanType() { - SpanType type = spanType == null ? SpanType.UNKNOWN : SpanType.fromValue(spanType); + SpanType type = + StringUtils.isBlank(spanType) ? SpanType.UNKNOWN : SpanType.fromValue(spanType); if (attributes.data().containsKey("db.operation")) { type = SpanType.QUERY; } diff --git a/tracedin-application/app-api/src/main/java/com/univ/tracedin/api/span/grpc/SpanGrpc.java b/tracedin-application/app-api/src/main/java/com/univ/tracedin/api/span/grpc/SpanGrpc.java new file mode 100644 index 0000000..f347033 --- /dev/null +++ b/tracedin-application/app-api/src/main/java/com/univ/tracedin/api/span/grpc/SpanGrpc.java @@ -0,0 +1,95 @@ +package com.univ.tracedin.api.span.grpc; + +import java.util.Map; +import java.util.stream.Collectors; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import com.univ.tracedin.api.span.grpc.SpanGrpcAppenderGrpc.SpanGrpcAppenderImplBase; +import com.univ.tracedin.api.span.grpc.SpanProto.AppendSpanResponse; +import com.univ.tracedin.api.span.grpc.SpanProto.AppendSpansRequest; +import com.univ.tracedin.domain.span.Span; +import com.univ.tracedin.domain.span.SpanAttributes; +import com.univ.tracedin.domain.span.SpanEvent; +import com.univ.tracedin.domain.span.SpanId; +import com.univ.tracedin.domain.span.SpanKind; +import com.univ.tracedin.domain.span.SpanService; +import com.univ.tracedin.domain.span.SpanStatus; +import com.univ.tracedin.domain.span.SpanTiming; +import com.univ.tracedin.domain.span.SpanType; +import com.univ.tracedin.domain.span.TraceId; + +import io.grpc.stub.StreamObserver; +import net.devh.boot.grpc.server.service.GrpcService; + +@Slf4j +@RequiredArgsConstructor +@GrpcService +public class SpanGrpc extends SpanGrpcAppenderImplBase { + + private final SpanService spanService; + + @Override + public void appendSpans( + AppendSpansRequest request, StreamObserver responseObserver) { + try { + log.info("appendSpans request: {}", request.toString()); + spanService.publishSpans(request.getSpansList().stream().map(this::toSpan).toList()); + responseObserver.onNext(AppendSpanResponse.newBuilder().setStatusCode(200).build()); + } catch (Exception e) { + log.error("Failed to append spans", e); + responseObserver.onNext(AppendSpanResponse.newBuilder().setStatusCode(500).build()); + } finally { + responseObserver.onCompleted(); + } + } + + private Span toSpan(SpanProto.Span span) { + return Span.builder() + .id(SpanId.from(span.getSpanId())) + .traceId(TraceId.from(span.getTraceId())) + .parentId(SpanId.from(span.getParentSpanId())) + .name(span.getName()) + .serviceName(span.getServiceName()) + .projectKey(span.getProjectKey()) + .spanType(SpanType.fromValue(span.getSpanType())) + .kind(SpanKind.fromValue(span.getKind())) + .timing( + SpanTiming.builder() + .startEpochMillis(nanosToMillis(span.getStartEpochNanos())) + .endEpochMillis(nanosToMillis(span.getEndEpochNanos())) + .build()) + .status(SpanStatus.fromValue(span.getSpanStatus())) + .attributes( + SpanAttributes.builder() + .data( + span.getAttributes().getDataMap().entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue))) + .capacity(span.getAttributes().getCapacity()) + .totalAddedValues(span.getAttributes().getTotalAddedValues()) + .build()) + .events( + span.getEventsList().stream() + .map( + event -> + new SpanEvent( + event.getName(), + event.getAttributesMap().entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + Map.Entry + ::getValue)), + event.getEpochNanos())) + .toList()) + .build(); + } + + private long nanosToMillis(long nanos) { + return nanos / 1_000_000; + } +} diff --git a/tracedin-application/app-api/src/main/proto/service-metrics.proto b/tracedin-application/app-api/src/main/proto/service-metrics.proto new file mode 100644 index 0000000..e939222 --- /dev/null +++ b/tracedin-application/app-api/src/main/proto/service-metrics.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; + +option java_package = "com.univ.tracedin.api.metric.grpc"; +option java_outer_classname = "ServiceMetricsProto"; + +message AppendServiceMetricsRequest { + string project_key = 1; + string service_name = 2; + repeated MetricRequest metrics = 3; +} + +message MetricRequest { + string name = 1; + string description = 2; + string unit = 3; + string type = 4; + double value = 5; // GAUGE 및 SUM 타입의 단일 값 + int64 count = 6; // HISTOGRAM 타입의 카운트 + double sum = 7; // HISTOGRAM 타입의 합계 + double min = 8; // HISTOGRAM 타입의 최소값 + double max = 9; // HISTOGRAM 타입의 최대값 + map attributes = 10; // 속성 정보 +} +message AppendServiceMetricsResponse { + int64 status_code = 1; +} + + +service ServiceMetricsGrpcAppender { + rpc AppendServiceMetrics (AppendServiceMetricsRequest) returns (AppendServiceMetricsResponse); +} diff --git a/tracedin-application/app-api/src/main/proto/span.proto b/tracedin-application/app-api/src/main/proto/span.proto new file mode 100644 index 0000000..960b8e7 --- /dev/null +++ b/tracedin-application/app-api/src/main/proto/span.proto @@ -0,0 +1,47 @@ +syntax = "proto3"; + +option java_package = "com.univ.tracedin.api.span.grpc"; +option java_outer_classname = "SpanProto"; +message Span { + string service_name = 1; + string project_key = 2; + string trace_id = 3; + string span_id = 4; + string parent_span_id = 5; + string span_type = 6; + string name = 7; + string kind = 8; + int64 start_epoch_nanos = 9; + int64 end_epoch_nanos = 10; + string span_status = 11; + Attributes attributes = 12; + repeated Event events = 13; +} + +message Attributes { + map data = 1; + int32 capacity = 2; + int32 total_added_values = 3; +} + +message Event { + string name = 1; + map attributes = 2; + int64 epoch_nanos = 3; +} + +message AppendSpansRequest { + repeated Span spans = 1; +} + +message AppendSpanResponse { + int64 status_code = 1; +} + +service SpanGrpcAppender { + rpc AppendSpans (AppendSpansRequest) returns (AppendSpanResponse); +} + + + + diff --git a/tracedin-application/app-api/src/main/resources/application.yml b/tracedin-application/app-api/src/main/resources/application.yml index 5493aca..9851028 100644 --- a/tracedin-application/app-api/src/main/resources/application.yml +++ b/tracedin-application/app-api/src/main/resources/application.yml @@ -26,6 +26,9 @@ jwt: expiration: 25920000 server: port: 8089 +grpc: + server: + port: 6565 --- spring: config: diff --git a/tracedin-domain/src/main/java/com/univ/tracedin/domain/anomaly/AnomalyTraceProcessor.java b/tracedin-domain/src/main/java/com/univ/tracedin/domain/anomaly/AnomalyTraceProcessor.java index 5426ddd..bac723a 100644 --- a/tracedin-domain/src/main/java/com/univ/tracedin/domain/anomaly/AnomalyTraceProcessor.java +++ b/tracedin-domain/src/main/java/com/univ/tracedin/domain/anomaly/AnomalyTraceProcessor.java @@ -2,7 +2,6 @@ import java.util.HashMap; import java.util.List; -import java.util.Map; import org.springframework.stereotype.Component; @@ -40,11 +39,13 @@ private void sendAlerts(List anomalyTraces) { private Alert createAnomalyAlert(AnomalyTrace anomalyTrace) { Project project = projectReader.readByKey(anomalyTrace.projectKey()); - Map details = new HashMap<>(); - details.put("traceId", anomalyTrace.traceId().getValue()); - details.put( - "anomalySpanIds", - anomalyTrace.anomalySpanIds().stream().map(SpanId::getValue).toList().toString()); + HashMap details = + new HashMap<>() { + { + put("traceId", anomalyTrace.traceId().getValue()); + put("anomalySpanIds", getSpanIdToString(anomalyTrace)); + } + }; return Alert.create("트랜잭션에 이상치가 탐지되었습니다!", project.getId(), details); } @@ -56,4 +57,8 @@ private void updateAnomalySpans(List anomalyTraces) { .forEach(spanUpdater::update); } } + + private String getSpanIdToString(AnomalyTrace anomalyTrace) { + return anomalyTrace.anomalySpanIds().stream().map(SpanId::getValue).toList().toString(); + } } diff --git a/tracedin-domain/src/main/java/com/univ/tracedin/domain/span/SpanType.java b/tracedin-domain/src/main/java/com/univ/tracedin/domain/span/SpanType.java index 53f084c..d95d84f 100644 --- a/tracedin-domain/src/main/java/com/univ/tracedin/domain/span/SpanType.java +++ b/tracedin-domain/src/main/java/com/univ/tracedin/domain/span/SpanType.java @@ -22,6 +22,6 @@ public static SpanType fromValue(String value) { return type; } } - throw new IllegalArgumentException("Unknown span type: " + value); + return UNKNOWN; } } diff --git a/tracedin-infra/src/main/java/com/univ/tracedin/infra/span/repository/SpanElasticSearchRepositoryCustomImpl.java b/tracedin-infra/src/main/java/com/univ/tracedin/infra/span/repository/SpanElasticSearchRepositoryCustomImpl.java index 20914ee..68fe2a1 100644 --- a/tracedin-infra/src/main/java/com/univ/tracedin/infra/span/repository/SpanElasticSearchRepositoryCustomImpl.java +++ b/tracedin-infra/src/main/java/com/univ/tracedin/infra/span/repository/SpanElasticSearchRepositoryCustomImpl.java @@ -3,6 +3,7 @@ import static com.univ.tracedin.infra.elasticsearch.ESUtils.executeESQuery; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -254,6 +255,7 @@ private List parseHttpTpsResponse(SearchResponse response) { double tps = bucket.aggregations().get("tps").simpleValue().value(); httpTpsList.add(HttpTps.of(startEpochMillis, tps)); }); + httpTpsList.sort(Comparator.comparing(HttpTps::timestamp).reversed()); return httpTpsList; }