From 9370d0a6afa378e657a8cf810cf80c7c12b674a2 Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Thu, 26 Dec 2024 12:29:17 +0700 Subject: [PATCH 01/15] Add new dependencies --- logzio-sender/pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/logzio-sender/pom.xml b/logzio-sender/pom.xml index 2f72709..ba5cb0b 100644 --- a/logzio-sender/pom.xml +++ b/logzio-sender/pom.xml @@ -84,6 +84,16 @@ + + io.opentelemetry + opentelemetry-api + 1.10.0 + + + io.opentelemetry + opentelemetry-sdk + 1.10.0 + org.ikasan bigqueue From 134325f21e2cab7554634084f0016c3d495930c7 Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Thu, 26 Dec 2024 12:30:32 +0700 Subject: [PATCH 02/15] Inject otel context working version --- .../src/main/java/io/logz/sender/LogzioSender.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/logzio-sender/src/main/java/io/logz/sender/LogzioSender.java b/logzio-sender/src/main/java/io/logz/sender/LogzioSender.java index 504ea8c..0131964 100644 --- a/logzio-sender/src/main/java/io/logz/sender/LogzioSender.java +++ b/logzio-sender/src/main/java/io/logz/sender/LogzioSender.java @@ -18,6 +18,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; public class LogzioSender { private static final int MAX_SIZE_IN_BYTES = 3 * 1024 * 1024; // 3 MB @@ -145,7 +147,16 @@ public void clearQueue() throws IOException { } public void send(JsonObject jsonMessage) { - + // Extract OpenTelemetry context + Span currentSpan = Span.current(); + if (currentSpan != null) { + SpanContext spanContext = currentSpan.getSpanContext(); + if (spanContext.isValid()) { + jsonMessage.addProperty("trace_id", spanContext.getTraceId()); + jsonMessage.addProperty("span_id", spanContext.getSpanId()); + jsonMessage.addProperty("service_name", "your-service-name"); // Replace with your actual service name + } + } // check for oversized message int jsonByteLength = jsonMessage.toString().getBytes(StandardCharsets.UTF_8).length; String jsonMessageField = jsonMessage.get("message").getAsString(); From 48d22efdf62d48bac8419ab69eb25ade4369aa27 Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Thu, 26 Dec 2024 12:30:47 +0700 Subject: [PATCH 03/15] Add unit tests --- .../java/io/logz/sender/LogzioSenderTest.java | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java b/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java index 71bf237..16542a2 100644 --- a/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java +++ b/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java @@ -4,6 +4,12 @@ import io.logz.sender.exceptions.LogzioParameterErrorException; import io.logz.test.MockLogzioBulkListener; import io.logz.test.TestEnvironment; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.trace.SdkTracerProvider; + + import org.junit.jupiter.api.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -17,8 +23,16 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; + import static io.logz.sender.LogzioTestSenderUtil.LOGLEVEL; import static io.logz.sender.LogzioTestSenderUtil.createJsonMessage; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public abstract class LogzioSenderTest { protected MockLogzioBulkListener mockListener; @@ -92,6 +106,57 @@ protected String random(int numberOfChars) { return UUID.randomUUID().toString().substring(0, numberOfChars - 1); } + /** + * Initializes the OpenTelemetry SDK with a logging span exporter and the W3C Trace Context + * propagator. + * + * @return A ready-to-use {@link OpenTelemetry} instance. + */ + protected OpenTelemetry initOpenTelemetry() { + SdkTracerProvider sdkTracerProvider = + SdkTracerProvider.builder() + .build(); + + OpenTelemetrySdk sdk = + OpenTelemetrySdk.builder() + .setTracerProvider(sdkTracerProvider) + .build(); + + Runtime.getRuntime().addShutdownHook(new Thread(sdkTracerProvider::close)); + return sdk; + } + @Test + public void testOpenTelemetryContextInjection() throws Exception { + OpenTelemetry openTelemetry = initOpenTelemetry(); + // Set up OpenTelemetry tracer + Tracer tracer = openTelemetry.getTracer("test"); + // Start a new span + Span span = tracer.spanBuilder("test").setSpanKind(SpanKind.CLIENT).startSpan(); + // Activate the span + try (Scope scope = span.makeCurrent()) { + String token = "testToken"; + String type = "testType"; + int drainTimeout = 2; + LogzioSender.Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, + 10 * 1000, 10 * 1000, tasks, false); + LogzioSender testSender = createLogzioSender(testSenderBuilder); + + JsonObject jsonMessage = createJsonMessage("testLogger", "Test message"); + + testSender.send(jsonMessage); + + assertTrue(jsonMessage.has("trace_id")); + assertTrue(jsonMessage.has("span_id")); + assertTrue(jsonMessage.has("service_name")); + assertEquals(span.getSpanContext().getTraceId(), jsonMessage.get("trace_id").getAsString()); + assertEquals(span.getSpanContext().getSpanId(), jsonMessage.get("span_id").getAsString()); + assertEquals("your-service-name", jsonMessage.get("service_name").getAsString()); // Replace with your actual service name + } finally { + // End the span + span.end(); + } + } + @Test public void simpleAppending() throws Exception { String token = "aBcDeFgHiJkLmNoPqRsT"; From 01da566cc103a628990266045f24d428df61a0a4 Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Thu, 26 Dec 2024 13:29:15 +0700 Subject: [PATCH 04/15] Add `opentelemetry-semconv` --- logzio-sender/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/logzio-sender/pom.xml b/logzio-sender/pom.xml index ba5cb0b..cfa86b8 100644 --- a/logzio-sender/pom.xml +++ b/logzio-sender/pom.xml @@ -89,6 +89,11 @@ opentelemetry-api 1.10.0 + + io.opentelemetry + opentelemetry-semconv + 1.26.0-alpha + io.opentelemetry opentelemetry-sdk From 219b487c500f01f11952961f64b9c99a0c7f37b7 Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Thu, 26 Dec 2024 13:29:40 +0700 Subject: [PATCH 05/15] Add `addOpenTelemetryContext` method --- .../java/io/logz/sender/LogzioSender.java | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/logzio-sender/src/main/java/io/logz/sender/LogzioSender.java b/logzio-sender/src/main/java/io/logz/sender/LogzioSender.java index 0131964..34ce488 100644 --- a/logzio-sender/src/main/java/io/logz/sender/LogzioSender.java +++ b/logzio-sender/src/main/java/io/logz/sender/LogzioSender.java @@ -18,8 +18,10 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.sdk.resources.Resource; public class LogzioSender { private static final int MAX_SIZE_IN_BYTES = 3 * 1024 * 1024; // 3 MB @@ -104,6 +106,20 @@ private static LogzioSender getLogzioSender(HttpsRequestConfiguration httpsReque } } + private void addOpenTelemetryContext(JsonObject jsonMessage) { + Span currentSpan = Span.current(); + if (currentSpan != null) { + SpanContext spanContext = currentSpan.getSpanContext(); + if (spanContext.isValid()) { + jsonMessage.addProperty("trace_id", spanContext.getTraceId()); + jsonMessage.addProperty("span_id", spanContext.getSpanId()); + Resource resource = Resource.getDefault(); + Attributes attributes = resource.getAttributes(); + String serviceName = attributes.get(io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME); + jsonMessage.addProperty("service_name", serviceName); + } + } + } public void start() { tasksExecutor.scheduleWithFixedDelay(this::drainQueueAndSend, 0, drainTimeout, TimeUnit.SECONDS); } @@ -147,16 +163,7 @@ public void clearQueue() throws IOException { } public void send(JsonObject jsonMessage) { - // Extract OpenTelemetry context - Span currentSpan = Span.current(); - if (currentSpan != null) { - SpanContext spanContext = currentSpan.getSpanContext(); - if (spanContext.isValid()) { - jsonMessage.addProperty("trace_id", spanContext.getTraceId()); - jsonMessage.addProperty("span_id", spanContext.getSpanId()); - jsonMessage.addProperty("service_name", "your-service-name"); // Replace with your actual service name - } - } + addOpenTelemetryContext(jsonMessage); // check for oversized message int jsonByteLength = jsonMessage.toString().getBytes(StandardCharsets.UTF_8).length; String jsonMessageField = jsonMessage.get("message").getAsString(); From 8bbbd274c51f6b8f996cb4ffc2083aac0f69f1bb Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Thu, 26 Dec 2024 13:29:57 +0700 Subject: [PATCH 06/15] unit tests --- .../src/test/java/io/logz/sender/LogzioSenderTest.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java b/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java index 16542a2..55ae63e 100644 --- a/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java +++ b/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java @@ -23,9 +23,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; -import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.SpanBuilder; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Scope; @@ -128,11 +126,8 @@ protected OpenTelemetry initOpenTelemetry() { @Test public void testOpenTelemetryContextInjection() throws Exception { OpenTelemetry openTelemetry = initOpenTelemetry(); - // Set up OpenTelemetry tracer Tracer tracer = openTelemetry.getTracer("test"); - // Start a new span Span span = tracer.spanBuilder("test").setSpanKind(SpanKind.CLIENT).startSpan(); - // Activate the span try (Scope scope = span.makeCurrent()) { String token = "testToken"; String type = "testType"; @@ -150,9 +145,7 @@ public void testOpenTelemetryContextInjection() throws Exception { assertTrue(jsonMessage.has("service_name")); assertEquals(span.getSpanContext().getTraceId(), jsonMessage.get("trace_id").getAsString()); assertEquals(span.getSpanContext().getSpanId(), jsonMessage.get("span_id").getAsString()); - assertEquals("your-service-name", jsonMessage.get("service_name").getAsString()); // Replace with your actual service name } finally { - // End the span span.end(); } } From 5b2edc3c17f5d69bbd3b644152f060031e8a6313 Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Sun, 29 Dec 2024 17:28:43 +0700 Subject: [PATCH 07/15] Add `withOpentelemetryContext` option --- .../java/io/logz/sender/LogzioSender.java | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/logzio-sender/src/main/java/io/logz/sender/LogzioSender.java b/logzio-sender/src/main/java/io/logz/sender/LogzioSender.java index 34ce488..9ecbbcd 100644 --- a/logzio-sender/src/main/java/io/logz/sender/LogzioSender.java +++ b/logzio-sender/src/main/java/io/logz/sender/LogzioSender.java @@ -41,10 +41,11 @@ public class LogzioSender { private ScheduledExecutorService tasksExecutor; private final AtomicBoolean drainRunning = new AtomicBoolean(false); private final HttpsSyncSender httpsSyncSender; + private final boolean withOpentelemetryContext; private LogzioSender(HttpsRequestConfiguration httpsRequestConfiguration, int drainTimeout, boolean debug, SenderStatusReporter reporter, ScheduledExecutorService tasksExecutor, - LogsQueue logsQueue, String exceedMaxSizeAction) throws LogzioParameterErrorException { + LogsQueue logsQueue, String exceedMaxSizeAction, boolean withOpentelemetryContext) throws LogzioParameterErrorException { if (logsQueue == null || reporter == null || httpsRequestConfiguration == null) { throw new LogzioParameterErrorException("logsQueue=" + logsQueue + " reporter=" + reporter @@ -59,6 +60,7 @@ private LogzioSender(HttpsRequestConfiguration httpsRequestConfiguration, int dr this.reporter = reporter; httpsSyncSender = new HttpsSyncSender(httpsRequestConfiguration, reporter); this.tasksExecutor = tasksExecutor; + this.withOpentelemetryContext = withOpentelemetryContext; debug("Created new LogzioSender class"); } @@ -71,7 +73,7 @@ private String validateAndGetExceedMaxSizeAction(String exceedMaxSizeAction) thr } private static LogzioSender getLogzioSender(HttpsRequestConfiguration httpsRequestConfiguration, int drainTimeout, boolean debug, SenderStatusReporter reporter, - ScheduledExecutorService tasksExecutor, LogsQueue logsQueue, String exceedMaxSizeAction) + ScheduledExecutorService tasksExecutor, LogsQueue logsQueue, String exceedMaxSizeAction, boolean withOpentelemetryContext) throws LogzioParameterErrorException { String tokenHash = Hashing.sha256() .hashString(httpsRequestConfiguration.getLogzioToken(), StandardCharsets.UTF_8) @@ -89,7 +91,7 @@ private static LogzioSender getLogzioSender(HttpsRequestConfiguration httpsReque } LogzioSender logzioSender = new LogzioSender(httpsRequestConfiguration, drainTimeout, debug, reporter, - tasksExecutor, logsQueue, exceedMaxSizeAction); + tasksExecutor, logsQueue, exceedMaxSizeAction, withOpentelemetryContext); logzioSenderInstances.put(tokenAndTypePair, logzioSender); return logzioSender; } else { @@ -163,7 +165,9 @@ public void clearQueue() throws IOException { } public void send(JsonObject jsonMessage) { - addOpenTelemetryContext(jsonMessage); + if (this.withOpentelemetryContext) { + addOpenTelemetryContext(jsonMessage); + } // check for oversized message int jsonByteLength = jsonMessage.toString().getBytes(StandardCharsets.UTF_8).length; String jsonMessageField = jsonMessage.get("message").getAsString(); @@ -298,6 +302,12 @@ public static class Builder { private DiskQueue.Builder diskQueueBuilder; private HttpsRequestConfiguration httpsRequestConfiguration; private String exceedMaxSizeAction = "cut"; + private boolean withOpentelemetryContext = true; + + public Builder setWithOpentelemetryContext(boolean withOpentelemetryContext) { + this.withOpentelemetryContext = withOpentelemetryContext; + return this; + } public Builder setExceedMaxSizeAction(String exceedMaxSizeAction) { this.exceedMaxSizeAction = exceedMaxSizeAction; @@ -360,7 +370,8 @@ public LogzioSender build() throws LogzioParameterErrorException, IOException { reporter, tasksExecutor, getLogsQueue(), - exceedMaxSizeAction + exceedMaxSizeAction, + withOpentelemetryContext ); } From 3fada4d2fcd2ac1766fb790cdcdeb6ee1983d77c Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Sun, 29 Dec 2024 17:41:18 +0700 Subject: [PATCH 08/15] Add new option for tests --- .../java/io/logz/sender/DiskQueueTest.java | 8 +++--- .../io/logz/sender/InMemoryQueueTest.java | 8 +++--- .../java/io/logz/sender/LogzioSenderTest.java | 27 ++++++++++--------- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/logzio-sender/src/test/java/io/logz/sender/DiskQueueTest.java b/logzio-sender/src/test/java/io/logz/sender/DiskQueueTest.java index d0b2f79..03d3621 100644 --- a/logzio-sender/src/test/java/io/logz/sender/DiskQueueTest.java +++ b/logzio-sender/src/test/java/io/logz/sender/DiskQueueTest.java @@ -27,9 +27,9 @@ public class DiskQueueTest extends LogzioSenderTest { protected Builder getLogzioSenderBuilder(String token, String type, Integer drainTimeout, Integer socketTimeout, Integer serverTimeout, ScheduledExecutorService tasks, - boolean compressRequests) throws LogzioParameterErrorException { + boolean compressRequests, boolean withOpentelemetryContext) throws LogzioParameterErrorException { Builder logzioSenderBuilder = super.getLogzioSenderBuilder(token, type, drainTimeout, - socketTimeout, serverTimeout, tasks, compressRequests); + socketTimeout, serverTimeout, tasks, compressRequests, withOpentelemetryContext); if (queueDir == null) { queueDir = TestEnvironment.createTempDirectory(); @@ -70,7 +70,7 @@ public void testSenderCantWriteToEmptyDirectory() { try { setQueueDir(tempDirectory); Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, 10 * 1000, - 10 * 1000, tasks, false); + 10 * 1000, tasks, false, false); LogzioSender testSender = createLogzioSender(testSenderBuilder); throw new LogzioParameterErrorException("Should not reach here", "fail"); } catch (LogzioParameterErrorException | IOException e) { @@ -95,7 +95,7 @@ public void testSenderCreatesDirectoryWhichDoesNotExists() throws Exception { assertFalse(queueDir.exists()); setQueueDir(queueDir); Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, - 10 * 1000, 10 * 1000, tasks, false); + 10 * 1000, 10 * 1000, tasks, false, false); LogzioSender testSender = createLogzioSender(testSenderBuilder); testSender.send(createJsonMessage(loggerName, message1)); assertTrue(queueDir.exists()); diff --git a/logzio-sender/src/test/java/io/logz/sender/InMemoryQueueTest.java b/logzio-sender/src/test/java/io/logz/sender/InMemoryQueueTest.java index 67369d2..ffe4b7e 100644 --- a/logzio-sender/src/test/java/io/logz/sender/InMemoryQueueTest.java +++ b/logzio-sender/src/test/java/io/logz/sender/InMemoryQueueTest.java @@ -20,10 +20,10 @@ public class InMemoryQueueTest extends LogzioSenderTest { @Override protected Builder getLogzioSenderBuilder(String token, String type, Integer drainTimeout, Integer socketTimeout, Integer serverTimeout, - ScheduledExecutorService tasks, boolean compressRequests) + ScheduledExecutorService tasks, boolean compressRequests, boolean withOpentelemetryContext) throws LogzioParameterErrorException { Builder logzioSenderBuilder = super.getLogzioSenderBuilder(token, type, drainTimeout, - socketTimeout, serverTimeout, tasks, compressRequests); + socketTimeout, serverTimeout, tasks, compressRequests, withOpentelemetryContext); setCapacityInBytes(logzioSenderBuilder, defaultCapacityInBytes); return logzioSenderBuilder; } @@ -59,7 +59,7 @@ public void checkCapacityReachedToSizeBelowCapacity() throws LogzioParameterErro int logSize = log.toString().getBytes(StandardCharsets.UTF_8).length; ScheduledExecutorService tasks = Executors.newScheduledThreadPool(3); Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, 10 * 1000, - 10 * 1000, tasks, false); + 10 * 1000, tasks, false, false); setCapacityInBytes(testSenderBuilder, logSize * successfulLogs); LogzioSender testSender = createLogzioSender(testSenderBuilder); sleepSeconds(drainTimeout - 1); @@ -87,7 +87,7 @@ public void checkLogMessageCountLimitWithCapacityInBytes() throws LogzioParamete JsonObject log = createJsonMessage(loggerName, message); ScheduledExecutorService tasks = Executors.newScheduledThreadPool(3); Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, 10 * 1000, - 10 * 1000, tasks, false); + 10 * 1000, tasks, false, false); setLogsCountLimit(testSenderBuilder, successfulLogs); LogzioSender testSender = createLogzioSender(testSenderBuilder); sleepSeconds(drainTimeout - 1); diff --git a/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java b/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java index 55ae63e..1bc67ce 100644 --- a/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java +++ b/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java @@ -71,7 +71,7 @@ protected void sleepSeconds(int seconds) { protected LogzioSender.Builder getLogzioSenderBuilder(String token, String type, Integer drainTimeout, Integer socketTimeout, Integer serverTimeout, ScheduledExecutorService tasks, - boolean compressRequests) + boolean compressRequests, boolean withOpentelemetryContext) throws LogzioParameterErrorException { LogzioTestStatusReporter logy = new LogzioTestStatusReporter(logger); HttpsRequestConfiguration httpsRequestConfiguration = HttpsRequestConfiguration @@ -86,6 +86,7 @@ protected LogzioSender.Builder getLogzioSenderBuilder(String token, String type, return LogzioSender .builder() .setDebug(false) + .setWithOpentelemetryContext(withOpentelemetryContext) .setTasksExecutor(tasks) .setDrainTimeoutSec(drainTimeout) .setReporter(logy) @@ -133,7 +134,7 @@ public void testOpenTelemetryContextInjection() throws Exception { String type = "testType"; int drainTimeout = 2; LogzioSender.Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, - 10 * 1000, 10 * 1000, tasks, false); + 10 * 1000, 10 * 1000, tasks, false, true); LogzioSender testSender = createLogzioSender(testSenderBuilder); JsonObject jsonMessage = createJsonMessage("testLogger", "Test message"); @@ -159,7 +160,7 @@ public void simpleAppending() throws Exception { String message1 = "Testing.." + random(5); String message2 = "Warning test.." + random(5); LogzioSender.Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, - 10 * 1000, 10 * 1000, tasks, false); + 10 * 1000, 10 * 1000, tasks, false,false); LogzioSender testSender = createLogzioSender(testSenderBuilder); testSender.send(createJsonMessage(loggerName, message1)); testSender.send(createJsonMessage(loggerName, message2)); @@ -178,7 +179,7 @@ public void malformedBulk() throws Exception { String message1 = "Testing.." + random(5); String message2 = "Warning test.." + random(5); LogzioSender.Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, - 10 * 1000, 10 * 1000, tasks, false); + 10 * 1000, 10 * 1000, tasks, false,false); LogzioSender testSender = createLogzioSender(testSenderBuilder); testSender.send(createJsonMessage(loggerName, message1)); testSender.send(createJsonMessage(loggerName, message2)); @@ -199,7 +200,7 @@ public void simpleByteArrayAppending() throws Exception { String message1 = "Testing.." + random(5); String message2 = "Warning test.." + random(5); LogzioSender.Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, - 10 * 1000, 10 * 1000, tasks, false); + 10 * 1000, 10 * 1000, tasks, false,false); LogzioSender testSender = createLogzioSender(testSenderBuilder); testSender.send(createJsonMessage(loggerName, message1).toString().getBytes(StandardCharsets.UTF_8)); testSender.send(createJsonMessage(loggerName, message2).toString().getBytes(StandardCharsets.UTF_8)); @@ -218,7 +219,7 @@ public void simpleGzipAppending() throws Exception { String message1 = "Testing.." + random(5); String message2 = "Warning test.." + random(5); LogzioSender.Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, 10 * 1000, - 10 * 1000, tasks, true); + 10 * 1000, tasks, true,false); LogzioSender testSender = createLogzioSender(testSenderBuilder); testSender.send(createJsonMessage(loggerName, message1)); testSender.send(createJsonMessage(loggerName, message2)); @@ -237,7 +238,7 @@ public void multipleQueueDrains() throws Exception { String message1 = "Testing first drain - " + random(5); String message2 = "And the second drain" + random(5); LogzioSender.Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, 10 * 1000, - 10 * 1000, tasks, false); + 10 * 1000, tasks, false, false); LogzioSender testSender = createLogzioSender(testSenderBuilder); testSender.send(createJsonMessage(loggerName, message1)); sleepSeconds(2 * drainTimeout); @@ -258,7 +259,7 @@ public void longDrainTimeout() throws Exception { String message1 = "Sending one log - " + random(5); String message2 = "And one more important one - " + random(5); LogzioSender.Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, 10 * 1000, - 10 * 1000, tasks, false); + 10 * 1000, tasks, false, false); LogzioSender testSender = createLogzioSender(testSenderBuilder); testSender.send(createJsonMessage(loggerName, message1)); testSender.send(createJsonMessage(loggerName, message2)); @@ -280,7 +281,7 @@ public void fsPercentDrop() throws Exception { String message1 = "First log that will be dropped - " + random(5); String message2 = "And a second drop - " + random(5); LogzioSender.Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeoutSec, 10 * 1000, - 10 * 1000, tasks, false); + 10 * 1000, tasks, false, false); setZeroThresholdQueue(testSenderBuilder); LogzioSender testSender = createLogzioSender(testSenderBuilder); // verify the thread that checks for space made at least one check @@ -302,7 +303,7 @@ public void serverCrash() throws Exception { String message2 = "Log during drop - " + random(5); String message3 = "Log after drop - " + random(5); LogzioSender.Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, 10 * 1000, - 10 * 1000, tasks, false); + 10 * 1000, tasks, false, false); LogzioSender testSender = createLogzioSender(testSenderBuilder); testSender.send(createJsonMessage(loggerName, message1)); sleepSeconds(2 * drainTimeout); @@ -331,7 +332,7 @@ public void getTimeoutFromServer() throws Exception { String message2 = "Log that would timeout and then being re-sent - " + random(5); int socketTimeout = serverTimeout / 2; LogzioSender.Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, socketTimeout, - serverTimeout, tasks, false); + serverTimeout, tasks, false, false); LogzioSender testSender = createLogzioSender(testSenderBuilder); testSender.send(createJsonMessage(loggerName, message1)); sleepSeconds(2 * drainTimeout); @@ -367,7 +368,7 @@ public void getExceptionFromServer() throws Exception { String message1 = "Log that will be sent - " + random(5); String message2 = "Log that would get exception and be sent again - " + random(5); LogzioSender.Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, 10 * 1000, - 10 * 1000, tasks, false); + 10 * 1000, tasks, false, false); LogzioSender testSender = createLogzioSender(testSenderBuilder); testSender.send(createJsonMessage(loggerName, message1)); sleepSeconds(2 * drainTimeout); @@ -471,7 +472,7 @@ public void checkExceedingMaxSizeBytesLogWithDrop() throws LogzioParameterErrorE private LogzioSender getLogzioSenderWithAndExceedMaxSizeAction(String token, String type, int drainTimeout, int logSize, ScheduledExecutorService tasks, String exceedMaxSizeAction) throws LogzioParameterErrorException, IOException { LogzioSender.Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, 10 * 1000, - 10 * 1000, tasks, false); + 10 * 1000, tasks, false, false); testSenderBuilder.setExceedMaxSizeAction(exceedMaxSizeAction); return createLogzioSender(testSenderBuilder); } From 9727ed4f40304326d585adb771f784f69d30399d Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Sun, 29 Dec 2024 17:59:20 +0700 Subject: [PATCH 09/15] Add more test cases --- .../java/io/logz/sender/LogzioSenderTest.java | 45 ++++++++++++++++++- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java b/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java index 1bc67ce..b361f58 100644 --- a/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java +++ b/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java @@ -29,8 +29,7 @@ import static io.logz.sender.LogzioTestSenderUtil.LOGLEVEL; import static io.logz.sender.LogzioTestSenderUtil.createJsonMessage; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; public abstract class LogzioSenderTest { protected MockLogzioBulkListener mockListener; @@ -151,6 +150,48 @@ public void testOpenTelemetryContextInjection() throws Exception { } } + @Test + public void testOpenTelemetryContextInjectionDisabled() throws Exception { + OpenTelemetry openTelemetry = initOpenTelemetry(); + Tracer tracer = openTelemetry.getTracer("test"); + Span span = tracer.spanBuilder("test").setSpanKind(SpanKind.CLIENT).startSpan(); + try (Scope scope = span.makeCurrent()) { + String token = "testToken"; + String type = "testType"; + int drainTimeout = 2; + LogzioSender.Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, + 10 * 1000, 10 * 1000, tasks, false, false); + LogzioSender testSender = createLogzioSender(testSenderBuilder); + + JsonObject jsonMessage = createJsonMessage("testLogger", "Test message"); + + testSender.send(jsonMessage); + + assertFalse(jsonMessage.has("trace_id")); + assertFalse(jsonMessage.has("span_id")); + assertFalse(jsonMessage.has("service_name")); + } finally { + span.end(); + } + } + @Test + public void testOpenTelemetryContextInjectionEnabledNoContext() throws Exception { + String token = "testToken"; + String type = "testType"; + int drainTimeout = 2; + LogzioSender.Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, + 10 * 1000, 10 * 1000, tasks, false, true); + LogzioSender testSender = createLogzioSender(testSenderBuilder); + + JsonObject jsonMessage = createJsonMessage("testLogger", "Test message"); + + testSender.send(jsonMessage); + + assertFalse(jsonMessage.has("trace_id")); + assertFalse(jsonMessage.has("span_id")); + assertFalse(jsonMessage.has("service_name")); + } + @Test public void simpleAppending() throws Exception { String token = "aBcDeFgHiJkLmNoPqRsT"; From 84fee92378a07d7e4176ad78abc6f00c061d270b Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Sun, 29 Dec 2024 17:59:35 +0700 Subject: [PATCH 10/15] Update pom.xml --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d2c7d62..8238e3a 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ io.logz.sender logzio-java-sender pom - 2.1.0 + 2.2.0 Logz.io Logs Sender Send your log messages to your logz.io account in an encrypted, non-blocking manner. From e5903ebdb548b8c286a1774ec685e265b331c7ee Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Sun, 29 Dec 2024 18:01:32 +0700 Subject: [PATCH 11/15] docs --- README.md | 51 +++++++++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 819a13a..84a33e3 100644 --- a/README.md +++ b/README.md @@ -21,36 +21,37 @@ This appender uses [BigQueue](https://github.com/bulldog2011/bigqueue) implement If you use Gradle, add the dependency to your project as follows: ```java -implementation 'io.logz.sender:logzio-java-sender:2.0.0' +implementation 'io.logz.sender:logzio-java-sender:${logzio-sender-version}' ``` ### Parameters -| Parameter | Default | Explained | -| ------------------ | ------------------------------------ | ----- | -| **token** | *None* | Your Logz.io token, which can be found under "settings" in your account. -| **logzioType** | *java* | The [log type](http://support.logz.io/support/solutions/articles/6000103063-what-is-type-) for that sender | -| **drainTimeoutSec** | *5* | How often the sender should drain the queue (in seconds) | -| **logzioUrl** | *https://listener.logz.io:8071* | Logz.io URL, that can be found under "Log Shipping -> Libraries" in your account. -| **socketTimeout** | *10 * 1000* | The socket timeout during log shipment | -| **connectTimeout** | *10 * 1000* | The connection timeout during log shipment | -| **debug** | *false* | Print some debug messages to stdout to help to diagnose issues | -| **compressRequests** | *false* | Boolean. `true` if logs are compressed in gzip format before sending. `false` if logs are sent uncompressed. | -| **exceedMaxSizeAction** | `cut` | String. `cut` to truncate the message field or `drop` to drop log that exceed the allowed maximum size for logzio. If the log size exceeding the maximum size allowed after truncating the message field, the log will be dropped.| +| Parameter | Default | Explained | +|------------------------------|---------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **token** | *None* | Your Logz.io token, which can be found under "settings" in your account. | +| **logzioType** | *java* | The [log type](http://support.logz.io/support/solutions/articles/6000103063-what-is-type-) for that sender | +| **drainTimeoutSec** | *5* | How often the sender should drain the queue (in seconds) | +| **logzioUrl** | *https://listener.logz.io:8071* | Logz.io URL, that can be found under "Log Shipping -> Libraries" in your account. | +| **socketTimeout** | *10 * 1000* | The socket timeout during log shipment | +| **connectTimeout** | *10 * 1000* | The connection timeout during log shipment | +| **debug** | *false* | Print some debug messages to stdout to help to diagnose issues | +| **compressRequests** | *false* | Boolean. `true` if logs are compressed in gzip format before sending. `false` if logs are sent uncompressed. | +| **exceedMaxSizeAction** | `cut` | String. `cut` to truncate the message field or `drop` to drop log that exceed the allowed maximum size for logzio. If the log size exceeding the maximum size allowed after truncating the message field, the log will be dropped. | +| **withOpentelemetryContext** | `true` | Boolean. Add trace_id, span_id, service_name fields to logs when opentelemetry context is available. | #### Parameters for in-memory queue -| Parameter | Default | Explained | -| ------------------ | ------------------------------------ | ----- | -| **inMemoryQueueCapacityInBytes** | *1024 * 1024 * 100* | The amount of memory(bytes) we are allowed to use for the memory queue. If the value is -1 the sender will not limit the queue size.| -| **logsCountLimit** | *-1* | The number of logs in the memory queue before dropping new logs. Default value is -1 (the sender will not limit the queue by logs count)| +| Parameter | Default | Explained | +|----------------------------------|---------------------|------------------------------------------------------------------------------------------------------------------------------------------| +| **inMemoryQueueCapacityInBytes** | *1024 * 1024 * 100* | The amount of memory(bytes) we are allowed to use for the memory queue. If the value is -1 the sender will not limit the queue size. | +| **logsCountLimit** | *-1* | The number of logs in the memory queue before dropping new logs. Default value is -1 (the sender will not limit the queue by logs count) | #### Parameters for disk queue -| Parameter | Default | Explained | -| ------------------ | ------------------------------------ | ----- | -| **queueDir** | *None* | Where the sender should store the queue. It should be at least one folder in path.| -| **fileSystemFullPercentThreshold** | *98* | The percent of used file system space at which the sender will stop queueing. When we will reach that percentage, the file system in which the queue is stored will drop all new logs until the percentage of used space drops below that threshold. Set to -1 to never stop processing new logs | -| **gcPersistedQueueFilesIntervalSeconds** | *30* | How often the disk queue should clean sent logs from disk | -| **checkDiskSpaceInterval** | *1000* | How often the should disk queue check for space (in milliseconds) | +| Parameter | Default | Explained | +|------------------------------------------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **queueDir** | *None* | Where the sender should store the queue. It should be at least one folder in path. | +| **fileSystemFullPercentThreshold** | *98* | The percent of used file system space at which the sender will stop queueing. When we will reach that percentage, the file system in which the queue is stored will drop all new logs until the percentage of used space drops below that threshold. Set to -1 to never stop processing new logs | +| **gcPersistedQueueFilesIntervalSeconds** | *30* | How often the disk queue should clean sent logs from disk | +| **checkDiskSpaceInterval** | *1000* | How often the should disk queue check for space (in milliseconds) | @@ -135,8 +136,10 @@ public class LogzioSenderExample { ## Release notes -- 2.1.0 - - Upgrade packages versions +- 2.2.0 + - Add `WithOpentelemetryContext` parameter to add `trace_id`, `span_id`, `service_name` fields to logs when opentelemetry context is available. + - 2.1.0 + - Upgrade packages versions - 2.0.1 - Add `User-Agent` header with logz.io information - 2.0.0 - **THIS IS A SNAPSHOT RELEASE - SUPPORTED WITH JDK 11 AND ABOVE** From f21bb4dcfd00a04a53f62dba06553be2e91a02ff Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Sun, 29 Dec 2024 18:02:44 +0700 Subject: [PATCH 12/15] docs --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 84a33e3..66584d8 100644 --- a/README.md +++ b/README.md @@ -136,8 +136,8 @@ public class LogzioSenderExample { ## Release notes -- 2.2.0 - - Add `WithOpentelemetryContext` parameter to add `trace_id`, `span_id`, `service_name` fields to logs when opentelemetry context is available. + - 2.2.0 + - Add `WithOpentelemetryContext` parameter to add `trace_id`, `span_id`, `service_name` fields to logs when opentelemetry context is available. - 2.1.0 - Upgrade packages versions - 2.0.1 From eb26870e70cafd717c72e9bbb3b1df44745964d2 Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Mon, 30 Dec 2024 11:15:41 +0700 Subject: [PATCH 13/15] Change otel context tests order --- .../java/io/logz/sender/LogzioSenderTest.java | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java b/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java index b361f58..7f5e46f 100644 --- a/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java +++ b/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java @@ -123,6 +123,24 @@ protected OpenTelemetry initOpenTelemetry() { Runtime.getRuntime().addShutdownHook(new Thread(sdkTracerProvider::close)); return sdk; } + @Test + public void testOpenTelemetryContextInjectionEnabledNoContext() throws Exception { + String token = "testToken"; + String type = "testType"; + int drainTimeout = 2; + LogzioSender.Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, + 10 * 1000, 10 * 1000, tasks, false, true); + LogzioSender testSender = createLogzioSender(testSenderBuilder); + + JsonObject jsonMessage = createJsonMessage("testLogger", "Test message"); + + testSender.send(jsonMessage); + + assertFalse(jsonMessage.has("trace_id")); + assertFalse(jsonMessage.has("span_id")); + assertFalse(jsonMessage.has("service_name")); + } + @Test public void testOpenTelemetryContextInjection() throws Exception { OpenTelemetry openTelemetry = initOpenTelemetry(); @@ -174,23 +192,6 @@ public void testOpenTelemetryContextInjectionDisabled() throws Exception { span.end(); } } - @Test - public void testOpenTelemetryContextInjectionEnabledNoContext() throws Exception { - String token = "testToken"; - String type = "testType"; - int drainTimeout = 2; - LogzioSender.Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, - 10 * 1000, 10 * 1000, tasks, false, true); - LogzioSender testSender = createLogzioSender(testSenderBuilder); - - JsonObject jsonMessage = createJsonMessage("testLogger", "Test message"); - - testSender.send(jsonMessage); - - assertFalse(jsonMessage.has("trace_id")); - assertFalse(jsonMessage.has("span_id")); - assertFalse(jsonMessage.has("service_name")); - } @Test public void simpleAppending() throws Exception { From 20a183e81a53e3ee55ff773b527f3b8d1c8292d6 Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Mon, 30 Dec 2024 11:42:04 +0700 Subject: [PATCH 14/15] Change otel context tests order --- .../java/io/logz/sender/LogzioSenderTest.java | 30 ++----------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java b/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java index 7f5e46f..1071189 100644 --- a/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java +++ b/logzio-sender/src/test/java/io/logz/sender/LogzioSenderTest.java @@ -123,8 +123,9 @@ protected OpenTelemetry initOpenTelemetry() { Runtime.getRuntime().addShutdownHook(new Thread(sdkTracerProvider::close)); return sdk; } + @Test - public void testOpenTelemetryContextInjectionEnabledNoContext() throws Exception { + public void openTelemetryContextInjectionEnabledNoContext() throws Exception { String token = "testToken"; String type = "testType"; int drainTimeout = 2; @@ -142,7 +143,7 @@ public void testOpenTelemetryContextInjectionEnabledNoContext() throws Exception } @Test - public void testOpenTelemetryContextInjection() throws Exception { + public void openTelemetryContextInjection() throws Exception { OpenTelemetry openTelemetry = initOpenTelemetry(); Tracer tracer = openTelemetry.getTracer("test"); Span span = tracer.spanBuilder("test").setSpanKind(SpanKind.CLIENT).startSpan(); @@ -168,31 +169,6 @@ public void testOpenTelemetryContextInjection() throws Exception { } } - @Test - public void testOpenTelemetryContextInjectionDisabled() throws Exception { - OpenTelemetry openTelemetry = initOpenTelemetry(); - Tracer tracer = openTelemetry.getTracer("test"); - Span span = tracer.spanBuilder("test").setSpanKind(SpanKind.CLIENT).startSpan(); - try (Scope scope = span.makeCurrent()) { - String token = "testToken"; - String type = "testType"; - int drainTimeout = 2; - LogzioSender.Builder testSenderBuilder = getLogzioSenderBuilder(token, type, drainTimeout, - 10 * 1000, 10 * 1000, tasks, false, false); - LogzioSender testSender = createLogzioSender(testSenderBuilder); - - JsonObject jsonMessage = createJsonMessage("testLogger", "Test message"); - - testSender.send(jsonMessage); - - assertFalse(jsonMessage.has("trace_id")); - assertFalse(jsonMessage.has("span_id")); - assertFalse(jsonMessage.has("service_name")); - } finally { - span.end(); - } - } - @Test public void simpleAppending() throws Exception { String token = "aBcDeFgHiJkLmNoPqRsT"; From 0189748a760ac32fbbefc8876d9e5b608e98af49 Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Mon, 30 Dec 2024 15:10:59 +0700 Subject: [PATCH 15/15] update version --- logzio-sender-test/pom.xml | 2 +- logzio-sender/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/logzio-sender-test/pom.xml b/logzio-sender-test/pom.xml index 5c9606c..0b5efbe 100644 --- a/logzio-sender-test/pom.xml +++ b/logzio-sender-test/pom.xml @@ -5,7 +5,7 @@ logzio-java-sender io.logz.sender - 2.1.0 + 2.2.0 4.0.0 diff --git a/logzio-sender/pom.xml b/logzio-sender/pom.xml index cfa86b8..c9662ac 100644 --- a/logzio-sender/pom.xml +++ b/logzio-sender/pom.xml @@ -5,7 +5,7 @@ logzio-java-sender io.logz.sender - 2.1.0 + 2.2.0 4.0.0