Skip to content

Commit

Permalink
test(flake): enable flakey test reruns, disable problematic tests in …
Browse files Browse the repository at this point in the history 
…CI (#151)

* test(flake): enable flakey test reruns

* add container-specific log message startup checks

* just log cleanup failures rather than throwing. throwing them hides the true test failure causes

* disable suspect test case

* remove framework example starter resource

* better check if self custom target is defined

* disable problematic tests only in github CI environment
  • Loading branch information
@andrewazores
andrewazores authored Nov 8, 2023
1 parent edb04d9 commit 167184b
Show file tree
Hide file tree
Showing 19 changed files with 201 additions and 172 deletions.
4 changes: 3 additions & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,9 @@
<com.google.java-format.version>1.18.1</com.google.java-format.version>
<com.mycila.license.maven.plugin.version>4.3</com.mycila.license.maven.plugin.version>
<surefire-plugin.version>3.2.2</surefire-plugin.version>
<failsafe-plugin.version>3.2.2</failsafe-plugin.version>
<surefire.rerunFailingTestsCount>5</surefire.rerunFailingTestsCount>
<failsafe-plugin.version>${surefire-plugin.version}</failsafe-plugin.version>
<failsafe.rerunFailingTestsCount>${surefire.rerunFailingTestsCount}</failsafe.rerunFailingTestsCount>
<com.google.code.gson.version>2.10.1</com.google.code.gson.version><!-- TODO drop this -->
</properties>
<dependencyManagement>
Expand Down
6 changes: 6 additions & 0 deletions smoketest/compose/cryostat-grafana.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,9 @@ services:
- "3000:3000"
expose:
- "3000"
healthcheck:
test: curl --fail http://localhost:3000/ || exit 1
retries: 3
interval: 30s
start_period: 10s
timeout: 1s
6 changes: 6 additions & 0 deletions smoketest/compose/jfr-datasource.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,9 @@ services:
io.cryostat.jmxPort: "11223"
environment:
JAVA_OPTS_APPEND: "-Dcom.sun.management.jmxremote.autodiscovery=true -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=11223 -Dcom.sun.management.jmxremote.rmi.port=11223 -Djava.rmi.server.hostname=jfr-datasource -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false"
healthcheck:
test: curl --fail http://localhost:8080/ || exit 1
retries: 3
interval: 30s
start_period: 10s
timeout: 1s
31 changes: 0 additions & 31 deletions src/main/java/io/cryostat/GreetingResource.java
View file

This file was deleted.

23 changes: 0 additions & 23 deletions src/test/java/io/cryostat/GreetingResourceIT.java
View file

This file was deleted.

37 changes: 0 additions & 37 deletions src/test/java/io/cryostat/GreetingResourceTest.java
View file

This file was deleted.

6 changes: 5 additions & 1 deletion src/test/java/io/cryostat/resources/GrafanaResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import io.quarkus.test.common.DevServicesContext;
import io.quarkus.test.common.QuarkusTestResourceLifecycleManager;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.utility.DockerImageName;

public class GrafanaResource
Expand All @@ -43,7 +44,9 @@ public Map<String, String> start() {
new GenericContainer<>(DockerImageName.parse(IMAGE_NAME))
.withExposedPorts(GRAFANA_PORT)
.withEnv(envMap)
.withLogConsumer(outputFrame -> {});
.waitingFor(
Wait.forLogMessage(
".*inserting datasource from configuration.*", 1));
containerNetworkId.ifPresent(container::withNetworkMode);

container.start();
Expand All @@ -57,6 +60,7 @@ public Map<String, String> start() {
@Override
public void stop() {
container.stop();
container.close();
}

@Override
Expand Down
4 changes: 3 additions & 1 deletion src/test/java/io/cryostat/resources/JFRDatasourceResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import io.quarkus.test.common.DevServicesContext;
import io.quarkus.test.common.QuarkusTestResourceLifecycleManager;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.utility.DockerImageName;

public class JFRDatasourceResource
Expand All @@ -39,7 +40,7 @@ public Map<String, String> start() {
new GenericContainer<>(DockerImageName.parse(IMAGE_NAME))
.withExposedPorts(JFR_DATASOURCE_PORT)
.withEnv(envMap)
.withLogConsumer(outputFrame -> {});
.waitingFor(Wait.forLogMessage(".*Listening on:.*", 1));
containerNetworkId.ifPresent(container::withNetworkMode);

container.start();
Expand All @@ -56,6 +57,7 @@ public Map<String, String> start() {
@Override
public void stop() {
container.stop();
container.close();
}

@Override
Expand Down
2 changes: 2 additions & 0 deletions src/test/java/itest/CryostatTemplateIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable;

@DisabledIfEnvironmentVariable(named = "CI", matches = "true")
@QuarkusIntegrationTest
public class CryostatTemplateIT extends StandardSelfTest {

Expand Down
9 changes: 6 additions & 3 deletions src/test/java/itest/CustomTargetsIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,9 +92,12 @@ static void cleanup() throws Exception {
deleteResponse.get(REQUEST_TIMEOUT_SECONDS, TimeUnit.SECONDS),
Matchers.equalTo(expectedDeleteResponse));
} catch (Exception e) {
throw new ITestCleanupFailedException(
String.format("Failed to clean up credential with ID %d", storedCredential.id),
e);
logger.error(
new ITestCleanupFailedException(
String.format(
"Failed to clean up credential with ID %d",
storedCredential.id),
e));
}
}

Expand Down
23 changes: 15 additions & 8 deletions src/test/java/itest/RecordingWorkflowIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -275,9 +275,12 @@ public void testWorkflow() throws Exception {
try {
deleteRespFuture1.get(REQUEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
} catch (InterruptedException | ExecutionException | TimeoutException e) {
throw new ITestCleanupFailedException(
String.format("Failed to delete target recording %s", TEST_RECORDING_NAME),
e);
logger.error(
new ITestCleanupFailedException(
String.format(
"Failed to delete target recording %s",
TEST_RECORDING_NAME),
e));
}

CompletableFuture<JsonArray> savedRecordingsFuture = new CompletableFuture<>();
Expand All @@ -296,7 +299,9 @@ public void testWorkflow() throws Exception {
savedRecordings =
savedRecordingsFuture.get(REQUEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
} catch (InterruptedException | ExecutionException | TimeoutException e) {
throw new ITestCleanupFailedException("Failed to retrieve archived recordings", e);
logger.error(
new ITestCleanupFailedException(
"Failed to retrieve archived recordings", e));
}

for (Object savedRecording : savedRecordings) {
Expand All @@ -319,10 +324,12 @@ public void testWorkflow() throws Exception {
try {
deleteRespFuture2.get(REQUEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
} catch (InterruptedException | ExecutionException | TimeoutException e) {
throw new ITestCleanupFailedException(
String.format(
"Failed to delete archived recording %s", recordingName),
e);
logger.error(
new ITestCleanupFailedException(
String.format(
"Failed to delete archived recording %s",
recordingName),
e));
}
}
}
Expand Down
18 changes: 12 additions & 6 deletions src/test/java/itest/ReportIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,9 +235,12 @@ void testGetReportShouldSendFile() throws Exception {
deleteActiveRecResponse.get(REQUEST_TIMEOUT_SECONDS, TimeUnit.SECONDS),
Matchers.equalTo(null));
} catch (ExecutionException | InterruptedException e) {
throw new ITestCleanupFailedException(
String.format("Failed to delete target recording %s", TEST_RECORDING_NAME),
e);
logger.error(
new ITestCleanupFailedException(
String.format(
"Failed to delete target recording %s",
TEST_RECORDING_NAME),
e));
}

CompletableFuture<JsonObject> deleteArchivedRecResp = new CompletableFuture<>();
Expand All @@ -254,9 +257,12 @@ void testGetReportShouldSendFile() throws Exception {
try {
deleteArchivedRecResp.get(REQUEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
} catch (InterruptedException | ExecutionException e) {
throw new ITestCleanupFailedException(
String.format("Failed to delete archived recording %s", savedRecordingName),
e);
logger.error(
new ITestCleanupFailedException(
String.format(
"Failed to delete archived recording %s",
savedRecordingName),
e));
}
}
}
Expand Down
5 changes: 3 additions & 2 deletions src/test/java/itest/RulesPostFormIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,8 +163,9 @@ void testAddRuleThrowsWhenRuleNameAlreadyExists() throws Exception {
deleteResponse.get(10, TimeUnit.SECONDS),
Matchers.equalTo(expectedDeleteResponse));
} catch (InterruptedException | ExecutionException e) {
throw new ITestCleanupFailedException(
String.format("Failed to delete rule %s", TEST_RULE_NAME), e);
logger.error(
new ITestCleanupFailedException(
String.format("Failed to delete rule %s", TEST_RULE_NAME), e));
}
}
}
Expand Down
14 changes: 10 additions & 4 deletions src/test/java/itest/RulesPostJsonIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
import org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable;

@QuarkusIntegrationTest
@TestMethodOrder(OrderAnnotation.class)
Expand Down Expand Up @@ -106,6 +107,12 @@ void testAddRuleThrowsWhenMimeUnsupported() throws Exception {
ex.getCause().getMessage(), Matchers.equalTo("Unsupported Media Type"));
}

@DisabledIfEnvironmentVariable(
named = "CI",
matches = "true",
disabledReason =
"The server 500 seems to cause issues for the next test in the suite, ex. HTTP"
+ " connection closed when attempting to POST the next rule definition")
@Test
@Order(3)
void testAddRuleThrowsWhenMimeInvalid() throws Exception {
Expand Down Expand Up @@ -179,8 +186,6 @@ void testAddRuleThrowsWhenRuleNameAlreadyExists() throws Exception {
((HttpException) ex.getCause()).getStatusCode(), Matchers.equalTo(409));
MatcherAssert.assertThat(ex.getCause().getMessage(), Matchers.equalTo("Conflict"));

} catch (Exception e) {
logger.error(e);
} finally {
// clean up rule before running next test
CompletableFuture<JsonObject> deleteResponse = new CompletableFuture<>();
Expand All @@ -207,8 +212,9 @@ void testAddRuleThrowsWhenRuleNameAlreadyExists() throws Exception {
deleteResponse.get(10, TimeUnit.SECONDS),
Matchers.equalTo(expectedDeleteResponse));
} catch (InterruptedException | ExecutionException e) {
throw new ITestCleanupFailedException(
String.format("Failed to delete rule %s", TEST_RULE_NAME), e);
logger.error(
new ITestCleanupFailedException(
String.format("Failed to delete rule %s", TEST_RULE_NAME), e));
}
}
}
Expand Down
30 changes: 20 additions & 10 deletions src/test/java/itest/SnapshotIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,12 @@ void testPostV1ShouldCreateSnapshot() throws Exception {
try {
deleteRecordingResponse.get();
} catch (InterruptedException | ExecutionException e) {
throw new ITestCleanupFailedException(
String.format("Failed to delete target recording %s", TEST_RECORDING_NAME),
e);
logger.error(
new ITestCleanupFailedException(
String.format(
"Failed to delete target recording %s",
TEST_RECORDING_NAME),
e));
}

CompletableFuture<JsonObject> deleteSnapshotResponse = new CompletableFuture<>();
Expand All @@ -146,8 +149,10 @@ void testPostV1ShouldCreateSnapshot() throws Exception {
try {
deleteSnapshotResponse.get();
} catch (InterruptedException | ExecutionException e) {
throw new ITestCleanupFailedException(
String.format("Failed to delete snapshot %s", snapshotName.get()), e);
logger.error(
new ITestCleanupFailedException(
String.format("Failed to delete snapshot %s", snapshotName.get()),
e));
}
}
}
Expand Down Expand Up @@ -343,9 +348,12 @@ void testPostV2ShouldCreateSnapshot() throws Exception {
try {
deleteRecordingResponse.get();
} catch (InterruptedException | ExecutionException e) {
throw new ITestCleanupFailedException(
String.format("Failed to delete target recording %s", TEST_RECORDING_NAME),
e);
logger.error(
new ITestCleanupFailedException(
String.format(
"Failed to delete target recording %s",
TEST_RECORDING_NAME),
e));
}

CompletableFuture<JsonObject> deleteSnapshotResponse = new CompletableFuture<>();
Expand All @@ -362,8 +370,10 @@ void testPostV2ShouldCreateSnapshot() throws Exception {
try {
deleteSnapshotResponse.get();
} catch (InterruptedException | ExecutionException e) {
throw new ITestCleanupFailedException(
String.format("Failed to delete snapshot %s", snapshotName.get()), e);
logger.error(
new ITestCleanupFailedException(
String.format("Failed to delete snapshot %s", snapshotName.get()),
e));
}
}
}
Expand Down
Loading

0 comments on commit 167184b

Please sign in to comment.