Devops/build automation (#965)

Use the new "ready-database" to run the integration tests from github
actions.

Updates versioning in build.gradle, calendar versioning, but also
supporting tag based so we have some control.


Added an additional Annotation that uses the full schema version
(versions, it's a list) to decided if a test should run.

.github/workflows/build.yml

@@ -1,10 +1,9 @@
 ---
-name: Build CWMS Data API
+name: Build And Test CWMS Data API
 on:
   push:
     branches:
       - develop
-      - 'release/**'
   pull_request:
     branches:
       - develop
@@ -27,6 +26,8 @@ jobs:
       - name: build and test
         id: thebuild
         run: ./gradlew build --info --init-script init.gradle
+      - name: integration tests
+        run: ./gradlew integrationtest --info --init-script init.gradle
       - name: Upload WAR
         uses: actions/[email protected]
         with:
@@ -36,55 +37,3 @@ jobs:
           if-no-files-found: error
       - name: Build docker image
         run: docker build -t cda:build-latest .
-
-  release:
-    if: github.event_name == 'push' && startsWith(github.ref,'refs/heads/release')
-    name: Create and push releases
-    needs: build
-    runs-on: ubuntu-latest
-    steps:
-      - name: checkout code
-        uses: actions/[email protected]
-      - name: setup java
-        uses: actions/[email protected]
-        with:
-          distribution: 'temurin'
-          java-version: '8'
-          cache: 'gradle'
-      - name: Download all workflow run artifacts from build
-        id: artifacts
-        uses: actions/[email protected]
-        with:
-          path: ./
-      - name: get version
-        id: get_version
-        run: .github/workflows/get_version.sh
-      - name: show version
-        run: echo ${VERSION}
-      - name: Create Release
-        id: create_release
-        uses: softprops/[email protected]
-        with:
-          files: warfile/${{ needs.build.outputs.thewar}}
-          tag_name: ${{env.VERSION}}
-          generate_release_notes: true
-      - name: Login to Alt Registry
-        uses: docker/[email protected]
-        id: login-alt
-        with:
-          registry: ${{ secrets.ALT_REGISTRY }}
-          username: ${{ secrets.ALT_REG_USER }}
-          password: ${{ secrets.ALT_REG_PASSWORD }}
-      - name: Build docker image
-        env:
-          IMAGE_TAG: ${{env.VERSION}}
-          ALT_REGISTRY: ${{secrets.ALT_REGISTRY}}
-        run: |
-          docker build -t cda:build-latest .
-          docker tag cda:build-latest $ALT_REGISTRY/cwms/data-api:$IMAGE_TAG
-          docker tag cda:build-latest $ALT_REGISTRY/cwms/data-api:latest
-          docker push $ALT_REGISTRY/cwms/data-api:$IMAGE_TAG
-          docker push $ALT_REGISTRY/cwms/data-api:latest
-      - name: Logout of ALT registry
-        if: ${{ always() }}
-        run: docker logout ${{ steps.login-alt.outputs.registry }}

.github/workflows/deploy.yml

@@ -0,0 +1,75 @@
+---
+name: Release images and war
+on:
+  workflow_run:
+    workflows: ["Build And Test CWMS Data API"]
+    branches: ["develop"]
+    types:
+      - completed
+
+jobs:
+  release:
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    name: Create and push releases
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout code
+        uses: actions/[email protected]
+      - name: setup java
+        uses: actions/[email protected]
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+          cache: 'gradle'
+      - name: Download all workflow run artifacts from build
+        id: artifacts
+        uses: actions/[email protected]
+        with:
+          path: ./
+      - name: get version
+        id: get_version
+        run: .github/workflows/get_version.sh
+      - name: show version
+        run: echo ${VERSION}
+      - name: Create Release
+        id: create_release
+        uses: softprops/[email protected]
+        with:
+          files: warfile/${{ needs.build.outputs.thewar}}
+          tag_name: ${{env.VERSION}}
+          generate_release_notes: true
+      - name: Login to Alt Registry
+        uses: docker/[email protected]
+        id: login-alt
+        with:
+          registry: ${{ secrets.ALT_REGISTRY }}
+          username: ${{ secrets.ALT_REG_USER }}
+          password: ${{ secrets.ALT_REG_PASSWORD }}
+      - name: Login to Alt Public Registry
+        uses: docker/[email protected]
+        id: login-alt2
+        with:
+          registry: ${{ secrets.HEC_PUB_REGISTRY }}
+          username: ${{ secrets.ALT_REG_USER }}
+          password: ${{ secrets.ALT_REG_PASSWORD }}
+      - name: Build docker image
+        env:
+          IMAGE_TAG: ${{env.VERSION}}
+          ALT_REGISTRY: ${{secrets.ALT_REGISTRY}}
+          HEC_PUB_REGISTRY: ${{secrets.HEC_PUB_REGISTRY}}
+        run: |
+          docker build -t cda:build-latest .
+          docker tag cda:build-latest $ALT_REGISTRY/cwms/data-api:$IMAGE_TAG
+          docker tag cda:build-latest $ALT_REGISTRY/cwms/data-api:latest
+          docker tag cda:build-latest $HEC_PUB_REGISTRY/cwms/data-api:$IMAGE_TAG
+          docker tag cda:build-latest $HEC_PUB_REGISTRY/cwms/data-api:latest
+          docker push $ALT_REGISTRY/cwms/data-api:$IMAGE_TAG
+          docker push $ALT_REGISTRY/cwms/data-api:latest
+          docker push $HEC_PUB_REGISTRY/cwms/data-api:$IMAGE_TAG
+          docker push $HEC_PUB_REGISTRY/cwms/data-api:latest
+      - name: Logout of ALT registry
+        if: ${{ always() }}
+        run: |
+          docker logout ${{ steps.login-alt.outputs.registry }}
+          docker logout ${{ steps.login-alt2.outputs.registry }}

.gitignore

@@ -9,4 +9,5 @@ gradle.properties
 .idea
 kdata
 pki/certs
-compose.env
+compose.env
+.vscode

Dockerfile

@@ -15,11 +15,11 @@ RUN apk --no-cache upgrade && \
 
 RUN mkdir /download && \
     cd /download && \
-    wget https://archive.apache.org/dist/tomcat/tomcat-9/v9.0.93/bin/apache-tomcat-9.0.93.tar.gz && \
-    echo "3069924eb7041ccc0f2aeceb7d8626793a1a073a5b739a840d7974a18ebeb26cc3374cc5f4a3ffc74d3b019c0cb33e3d1fe96296e6663ac75a73c1171811726d *apache-tomcat-9.0.93.tar.gz" > checksum.txt && \
+    wget https://archive.apache.org/dist/tomcat/tomcat-9/v9.0.97/bin/apache-tomcat-9.0.97.tar.gz && \
+    echo "537dbbfc03b37312c2ec282c6906828298cb74e42aca6e3e6835d44bf6923fd8c5db77e98bf6ce9ef19e1922729de53b20546149176e07ac04087df786a62fd9 *apache-tomcat-9.0.97.tar.gz" > checksum.txt && \
     sha512sum -c checksum.txt && \
     tar xzf apache-tomcat-*tar.gz && \
-    mv apache-tomcat-9.0.93 /usr/local/tomcat/ && \
+    mv apache-tomcat-9.0.97 /usr/local/tomcat/ && \
     cd / && \
     rm -rf /download
 CMD ["/usr/local/tomcat/bin/catalina.sh","run"]

build.gradle

@@ -1,10 +1,26 @@
+buildscript {
+    repositories {
+        gradlePluginPortal()
+    }
+}
+
 plugins {
     id 'base'
     id 'eclipse'
     id 'idea'
+    id "com.palantir.git-version" version "3.1.0"
+}
+
+def static versionLabel(gitInfo) {
+    def branch = gitInfo.branchName // all branches are snapshots, only tags get released
+    def tag = gitInfo.lastTag
+    // tag is returned as is. Branch may need cleanup
+    // our primary versioning is branch+date, the tag will be used for the final "release"
+    return branch == null ? tag : new Date().format("yyyy.MM.dd-") + branch.replace("/", "-")
 }
+
 allprojects {
     apply plugin: 'cda.java-conventions'
     group = 'mil.army.usace.hec.cwms'
-    version = '3.1.6-SNAPSHOT' // ApiServlet.VERSION should be updated to match MAJOR.MINOR changes.
+    version = versionLabel(versionDetails())
 }

cwms-data-api/src/main/java/cwms/cda/ApiServlet.java

@@ -260,7 +260,8 @@ public class ApiServlet extends HttpServlet {
 
     // The VERSION should match the gradle version but not contain the patch version.
     // For example 2.4 not 2.4.13
-    public static final String VERSION = "3.0";
+    private static String VERSION;
+
     public static final String APPLICATION_TITLE = "CWMS Data API";
     public static final String PROVIDER_KEY_OLD = "radar.access.provider";
     public static final String PROVIDER_KEY = "cwms.dataapi.access.provider";
@@ -277,6 +278,9 @@ public class ApiServlet extends HttpServlet {
     @Resource(name = "jdbc/CWMS3")
     DataSource cwms;
 
+    public static String getApiVersion() {
+        return VERSION != null ? VERSION : "Not Yet Known";
+    }
 
 
     @Override
@@ -286,16 +290,21 @@ public void destroy() {
 
     @Override
     public void init(ServletConfig config) throws ServletException {
-        logger.atInfo().log("Initializing CWMS Data API Version:  " + obtainFullVersion(config));
+        if (VERSION == null) {
+            ApiServlet.VERSION = obtainFullVersion(config);
+        }
+        logger.atInfo().log("Initializing CWMS Data API Version:  " + VERSION);
         metrics = (MetricRegistry)config.getServletContext()
                 .getAttribute(MetricsServlet.METRICS_REGISTRY);
         totalRequests = metrics.meter("cwms.dataapi.total_requests");
+
         super.init(config);
     }
 
     @SuppressWarnings({"java:S125","java:S2095"}) // closed in destroy handler
     @Override
     public void init() {
+        logger.atInfo().log("Initializing Javalin.");
         JavalinValidation.register(UnitSystem.class, UnitSystem::systemFor);
         JavalinValidation.register(JooqDao.DeleteMethod.class, Controllers::getDeleteMethod);
 
@@ -843,7 +852,7 @@ public static String getResourceId(String fullPath) {
     }
 
     private void getOpenApiOptions(JavalinConfig config) {
-        Info applicationInfo = new Info().title(APPLICATION_TITLE).version(VERSION)
+        Info applicationInfo = new Info().title(APPLICATION_TITLE).version(ApiServlet.getApiVersion())
                 .description("CWMS REST API for Data Retrieval");
 
         String provider = getAccessManagerName();

cwms-data-api/src/main/java/cwms/cda/data/dao/Dao.java

@@ -84,7 +84,7 @@ public int getDbVersion() {
         }
     }
 
-    private static int versionAsInteger(String version) {
+    public static int versionAsInteger(String version) {
         String[] parts = version.split("\\.");
 
         return Integer.parseInt(parts[0]) * 10000

cwms-data-api/src/main/java/cwms/cda/data/dao/JooqDao.java

@@ -146,7 +146,7 @@ public static DSLContext getDslContext(Connection connection, String officeId) {
 
     private static Connection setClientInfo(Context ctx, Connection connection) {
         try {
-            connection.setClientInfo("OCSID.ECID", ApiServlet.APPLICATION_TITLE + " " + ApiServlet.VERSION);
+            connection.setClientInfo("OCSID.ECID", ApiServlet.APPLICATION_TITLE + " " + ApiServlet.getApiVersion());
             connection.setClientInfo("OCSID.MODULE", ctx.endpointHandlerPath());
             connection.setClientInfo("OCSID.ACTION", ctx.method());
             connection.setClientInfo("OCSID.CLIENTID", ctx.url().replace(ctx.path(), "") + ctx.contextPath());

cwms-data-api/src/main/java/cwms/cda/data/dao/location/kind/OutletDao.java

@@ -90,7 +90,6 @@ public Outlet retrieveOutlet(String officeId, String locationId) {
             LOCATION_REF_T locRef = LocationUtil.getLocationRef(locationId, officeId);
             Configuration config = DSL.using(conn).configuration();
             PROJECT_STRUCTURE_OBJ_T outletStruct = CWMS_OUTLET_PACKAGE.call_RETRIEVE_OUTLET(config, locRef);
-
             LocationGroupDao locGroupDao = new LocationGroupDao(dsl);
             List<LocationGroup> groups = locGroupDao.getLocationGroups(config, null, officeId, null,
                     Outlet.RATING_LOC_GROUP_CATEGORY, null);

cwms-data-api/src/test/java/cwms/cda/api/LocationCategoryControllerTestIT.java

@@ -30,6 +30,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
+import fixtures.FunctionalSchemas;
 import fixtures.TestAccounts;
 import io.restassured.filter.log.LogDetail;
 
@@ -144,6 +145,7 @@ void test_create_already_existing_CWMS_category() {
 	}
 
 	@Test
+	@FunctionalSchemas(values = {"99.99.99.9-CDA_STAGING"})
 	void test_create_read_delete_same_category_different_office() {
 		String officeId = user.getOperatingOffice();
 		String officeId2 = user2.getOperatingOffice();

cwms-data-api/src/test/java/cwms/cda/api/LocationGroupControllerTestIT.java

@@ -33,6 +33,7 @@
 import cwms.cda.formatters.ContentType;
 import cwms.cda.formatters.Formats;
 import fixtures.CwmsDataApiSetupCallback;
+import fixtures.FunctionalSchemas;
 import fixtures.TestAccounts;
 import io.restassured.filter.log.LogDetail;
 import io.restassured.response.ExtractableResponse;
@@ -465,6 +466,7 @@ void test_create_read_delete_agency_aliases_same_name() throws Exception {
     }
 
     @Test
+    @FunctionalSchemas(values = {"99.99.99.9-CDA_STAGING"})
     void test_create_read_delete_same_names_different_offices() throws Exception {
         // Create two location groups of the same name with an agency alias category
         String officeId = user.getOperatingOffice();