Feature/integration testing workflow (#275)

* updates to allow integration tests to run in GitHub Actions on feature/task_cwms_support branch

* disable CWMS for now

.github/workflows/build.yml

@@ -0,0 +1,51 @@
+name: CI/CD
+on:
+  push:
+    branches:
+      - main
+      - feature/task_cwms_support
+  pull_request:
+    branches:
+      - main
+      - feature/task_cwms_support
+permissions:
+  contents: write
+jobs:
+  build:
+    name: Build, Test, and Analyze
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/[email protected]
+        with:
+          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+      - name: Set up JDK 17
+        uses: actions/[email protected]
+        with:
+          java-version: 17
+          distribution: 'temurin'
+      - name: Cache SonarCloud packages
+        uses: actions/[email protected]
+        with:
+          path: ~/.sonar/cache
+          key: ${{ runner.os }}-sonar
+          restore-keys: ${{ runner.os }}-sonar
+      - name: Setup Gradle
+        uses: gradle/[email protected]
+        with:
+          dependency-graph: generate-and-submit
+      - name: Build
+        run: ./gradlew build
+      - name: Integration Tests
+        run: ./gradlew integrationTest --stacktrace --info
+      - name: Analyze
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+        run: ./gradlew sonar
+      - name: Upload WAR files
+        uses: actions/[email protected]
+        with:
+          name: warfile
+          path: ./**/build/libs/*.war
+          retention-days: 1
+          if-no-files-found: error

.github/workflows/default.yml

@@ -6,6 +6,7 @@ on:
   pull_request:
     branches:
       - main
+      -
 permissions:
   contents: write
 jobs:

buildSrc/src/main/groovy/opendcs-rest-api.deps-conventions.gradle

@@ -1,3 +1,43 @@
+/*
+ *  Copyright 2024 OpenDCS Consortium and its Contributors
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License")
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
 repositories {
-    mavenCentral()
+    maven
+            {
+                url "https://www.hec.usace.army.mil/nexus/repository/maven-public"
+                content
+                        {
+                            includeGroupByRegex "mil.army.usace.hec.*"
+                            includeGroupByRegex "org.jooq.pro"
+                            excludeGroup "com.oracle"
+                        }
+                metadataSources
+                        {
+                            mavenPom()
+                            artifact()
+                        }
+            }
+    mavenCentral() {
+        content
+                {
+                    excludeGroupByRegex "mil.army.usace.hec.*"
+                }
+        metadataSources
+                {
+                    mavenPom()
+                    artifact()
+                }
+    }
 }

gradle/libs.versions.toml

@@ -1,5 +1,5 @@
 [versions]
-opendcs = "7.0.12"
+opendcs = "7.0.13"
 
 jetty = "9.4.53.v20231009"
 jetty-jsp = "9.2.30.v20200428"
@@ -8,6 +8,7 @@ servlet-api = "4.0.1" #Updating this further will require a change to the jakart
 slf4j = "2.0.12"
 jersey = "2.40"
 postgresql = "42.7.4"
+
 swagger = "2.2.24"
 swagger-ui = "5.17.14"
 glassfish-jaxb = "2.3.3"
@@ -25,7 +26,13 @@ byte-buddy = "1.15.3"
 rest-assured = "5.5.0"
 hamcrest = "3.0"
 tomcat = "9.0.91"
-testcontainers = "1.20.3"
+testcontainers = "1.18.1"
+hec-testcontainers-cwms = "1.0.7"
+commons-io = "2.11.0"
+derby = "10.14.2.0"
+webcompere = "2.0.2"
+jdbi = "3.38.2"
+flywaydb = "9.8.1"
 
 [libraries]
 opendcs = { module = "org.opendcs:opendcs", version.ref = "opendcs" }
@@ -75,7 +82,16 @@ tomcat-embedded-core = { module = "org.apache.tomcat.embed:tomcat-embed-core", v
 tomcat-embedded-jasper = { module = "org.apache.tomcat.embed:tomcat-embed-jasper", version.ref = "tomcat" }
 tomcat-jdbc = { module = "org.apache.tomcat:tomcat-jdbc", version.ref = "tomcat" }
 testcontainers-core = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" }
-testcontainers-jdbc = { module = "org.testcontainers:jdbc", version.ref = "testcontainers" }
+commons-io = { module = "commons-io:commons-io", version.ref = "commons-io" }
+apache-derby = { module = "org.apache.derby:derby", version.ref = "derby" }
+webcompere-system-stubs-jupiter = { module = "uk.org.webcompere:system-stubs-jupiter", version.ref = "webcompere" }
+testcontainers-postgresql = { module = "org.testcontainers:postgresql", version.ref="testcontainers" }
+hec-testcontainers-cwms = { module = "mil.army.usace.hec:testcontainers-cwms", version.ref = "hec-testcontainers-cwms" }
+org-jdbi3-core = { module = "org.jdbi:jdbi3-core", version.ref = "jdbi" }
+org-jdbi3-sqlobject = { module = "org.jdbi:jdbi3-sqlobject", version.ref = "jdbi" }
+org-jdbi3-postgres = { module = "org.jdbi:jdbi3-postgres", version.ref = "jdbi" }
+org-flywaydb-flyway-core = { module = "org.flywaydb:flyway-core", version.ref = "flywaydb" }
 
 [bundles]
-tomcat = ["tomcat-embedded-core", "tomcat-embedded-jasper", "tomcat-jdbc"]
+tomcat = ["tomcat-embedded-core", "tomcat-embedded-jasper", "tomcat-jdbc"]
+jdbi = [ "org-jdbi3-core", "org-jdbi3-sqlobject", "org-jdbi3-postgres" ]

opendcs-rest-api/build.gradle

@@ -31,7 +31,22 @@ dependencies {
 
 
     //TODO - remove after https://github.com/opendcs/opendcs/pull/768 is merged
-//    api(libs.opendcs)
+    //Hacky workaround to still get transitive dependencies for runtime, but use
+    //branched code
+    api(libs.opendcs) {
+        exclude group: "org.opendcs", module: "opendcs"
+        exclude group: "org.opendcs", module: "opendcs-annotations"
+        exclude group: "javax.media", module: "jai_core"
+        exclude group: "javax.media", module: "jai_codec"
+        exclude group: "org.jooq.pro", module: "jooq-meta"
+        exclude group: "org.jooq.pro", module: "jooq-codegen"
+        exclude group: "com.oracle.database.jdbc", module: "jmscommon"
+        exclude group: "com.oracle.database.jdbc", module: "aqapi"
+        exclude group: "com.oracle.database.jdbc", module: "xmlparserv2"
+        exclude group: "com.oracle.database.jdbc", module: "xdb"
+        exclude group: "com.sun.media", module: "jai_imageio"
+        exclude group: "com.mrsid", module: "mrsidreader"
+    }
 //    api(libs.opendcs.api)
 
     implementation(libs.servlet.api)
@@ -52,13 +67,20 @@ dependencies {
     testImplementation(libs.rest.assured)
     testImplementation(libs.hamcrest)
     testImplementation(libs.bundles.tomcat)
+    testImplementation(libs.commons.io)
+    testImplementation(libs.apache.derby)
+    testImplementation(libs.webcompere.system.stubs.jupiter)
+    testImplementation(libs.testcontainers.core)
+    testImplementation(libs.testcontainers.postgresql)
+    testImplementation(libs.hec.testcontainers.cwms)
+    testImplementation(libs.bundles.jdbi)
+    implementation(libs.org.flywaydb.flyway.core)
 
     //TODO - remove after https://github.com/opendcs/opendcs/pull/768 is merged
     implementation(files("temp-deps/opendcs-integrationtesting-fixtures.jar"))
-    implementation("uk.org.webcompere:system-stubs-jupiter:2.0.2")
 
 
-//    implementation(libs.opendcs.integrationtesting)
+//    implementation(libs.opendcs.integrationtesting.fixtures)
     compileOnly(libs.auto.service)
     annotationProcessor(libs.auto.service)
 }
@@ -130,6 +152,11 @@ tasks.register('integrationTest', Test) {
     useJUnitPlatform {
         includeTags "integration"
     }
+    doFirst {
+        classpath = classpath.filter {
+            !it.name.endsWith('opendcs-7.0.13.jar')
+        }
+    }
     shouldRunAfter test
     dependsOn war
     dependsOn generateConfig
@@ -138,15 +165,9 @@ tasks.register('integrationTest', Test) {
     jvmArgs += "-Djava.util.logging.config.file=$projectDir/test/resources/logging.properties"
     jvmArgs += "-Dorg.apache.tomcat.util.digester.PROPERTY_SOURCE=org.apache.tomcat.util.digester.EnvironmentPropertySource"
     jvmArgs += "-Dcatalina.base=${layout.buildDirectory.file("tomcat")}"
-    systemProperties += project.properties.findAll { k, v -> k.startsWith("opendcs") }
-    systemProperties += project.properties.findAll { k, v -> k.startsWith("postgresdb") }
+    jvmArgs += "-Dopendcs.test.integration.CWMS-Oracle.disabled=true"
     def outputDir = downloadAndUnzipInstaller.outputs.files.singleFile
     systemProperties.put("DCSTOOL_HOME", outputDir)
-    if (!project.hasProperty("postgresdb.url")) {
-        systemProperties.put("postgresdb.url", "jdbc:postgresql://localhost:5432/open_tsdb")
-        systemProperties.put("opendcs.db.username", "tsdbadm")
-        systemProperties.put("opendcs.db.password", "postgres_pass")
-    }
 }
 
 

opendcs-rest-api/src/test/java/org/opendcs/odcsapi/fixtures/DatabaseContextProvider.java

@@ -74,7 +74,7 @@ public List<Extension> getAdditionalExtensions()
 				{
 					File tmp = Files.createTempDirectory("configs-" + provider.getImplementation()).toFile();
 					Configuration config = provider.getConfig(tmp);
-					EXTENSIONS.add(new DatabaseSetupExtension(config));
+					EXTENSIONS.add(new DatabaseSetupExtension(config, dbType));
 				}
 				catch(Exception ex)
 				{

opendcs-rest-api/src/test/java/org/opendcs/odcsapi/fixtures/DatabaseSetupExtension.java

@@ -34,12 +34,26 @@
 public class DatabaseSetupExtension implements BeforeEachCallback
 {
 	private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseSetupExtension.class);
+	private static DbType currentDbType;
+	private static TomcatServer currentTomcat;
 	private final Configuration config;
+	private final DbType dbType;
 	private TomcatServer tomcatServer;
 
-	public DatabaseSetupExtension(Configuration config)
+	public DatabaseSetupExtension(Configuration config, DbType dbType)
 	{
 		this.config = config;
+		this.dbType = dbType;
+	}
+
+	public static DbType getCurrentDbType()
+	{
+		return currentDbType;
+	}
+
+	public static TomcatServer getCurrentTomcat()
+	{
+		return currentTomcat;
 	}
 
 	@Override
@@ -53,6 +67,8 @@ public void beforeEach(ExtensionContext context) throws Exception
 		RestAssured.baseURI = "http://localhost";
 		RestAssured.port = tomcatServer.getPort();
 		RestAssured.basePath = warContext;
+		currentDbType = dbType;
+		currentTomcat = tomcatServer;
 	}
 
 	private TomcatServer startTomcat(String warContext) throws Exception

opendcs-rest-api/src/test/java/org/opendcs/odcsapi/sec/basicauth/BasicAuthIT.java

@@ -21,17 +21,17 @@
 import io.restassured.filter.session.SessionFilter;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Tag;
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestTemplate;
 import org.junit.jupiter.api.extension.ExtendWith;
-import org.opendcs.odcsapi.fixtures.EmbeddedTomcatExtension;
+import org.opendcs.odcsapi.fixtures.DatabaseContextProvider;
 import org.opendcs.odcsapi.hydrojson.DbInterface;
 
 import static io.restassured.RestAssured.given;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
-@ExtendWith(EmbeddedTomcatExtension.class)
 @Tag("integration")
+@ExtendWith(DatabaseContextProvider.class)
 final class BasicAuthIT
 {
 
@@ -43,7 +43,7 @@ void setup()
 		sessionFilter = new SessionFilter();
 	}
 
-	@Test
+	@TestTemplate
 	void testBasicAuthFlow()
 	{
 		DbInterface.decodesProperties.setProperty("opendcs.rest.api.authorization.type", "basic");
@@ -61,8 +61,8 @@ void testBasicAuthFlow()
 			.statusCode(is(HttpServletResponse.SC_UNAUTHORIZED))
 		;
 		Credentials credentials = new Credentials();
-		credentials.setUsername(System.getProperty("opendcs.db.username"));
-		credentials.setPassword(System.getProperty("opendcs.db.password"));
+		credentials.setUsername(System.getProperty("DB_USERNAME"));
+		credentials.setPassword(System.getProperty("DB_PASSWORD"));
 		given()
 			.log().ifValidationFails(LogDetail.ALL, true)
 			.accept("application/json")

opendcs-rest-api/src/test/java/org/opendcs/odcsapi/sec/cwms/ServletSsoAuthIT.java

@@ -23,9 +23,11 @@
 import org.apache.catalina.session.StandardSession;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Tag;
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestTemplate;
 import org.junit.jupiter.api.extension.ExtendWith;
-import org.opendcs.odcsapi.fixtures.EmbeddedTomcatExtension;
+import org.opendcs.odcsapi.fixtures.DatabaseContextProvider;
+import org.opendcs.odcsapi.fixtures.DatabaseSetupExtension;
+import org.opendcs.odcsapi.fixtures.DbType;
 import org.opendcs.odcsapi.fixtures.TomcatServer;
 import org.opendcs.odcsapi.hydrojson.DbInterface;
 import org.opendcs.odcsapi.sec.OpenDcsApiRoles;
@@ -35,8 +37,9 @@
 
 import static io.restassured.RestAssured.given;
 import static org.hamcrest.Matchers.is;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
-@ExtendWith(EmbeddedTomcatExtension.class)
+@ExtendWith(DatabaseContextProvider.class)
 @Tag("integration")
 final class ServletSsoAuthIT
 {
@@ -53,7 +56,7 @@ void setup()
 	private static void setupSession()
 	{
 		String username = System.getProperty("opendcs.db.username");
-		TomcatServer tomcat = EmbeddedTomcatExtension.getOpendcsInstance();
+		TomcatServer tomcat = DatabaseSetupExtension.getCurrentTomcat();
 		StandardSession session = (StandardSession) tomcat.getTestSessionManager()
 				.createSession(COOKIE);
 		if(session == null) {
@@ -72,9 +75,10 @@ private static void setupSession()
 				.wrappedRegister(COOKIE, mcup, "CLIENT-CERT", null,null);
 	}
 
-	@Test
+	@TestTemplate
 	void testSsoAuthFlow()
 	{
+		assumeTrue(DatabaseSetupExtension.getCurrentDbType() == DbType.CWMS);
 		DbInterface.decodesProperties.setProperty("opendcs.rest.api.authorization.type", "sso");
 		//Initial session should be unauthorized
 		given()