Merge pull request #725 from gregschohn/SplitRfsPhasesIntoSeparateGra…

…dleProjects Move the 3 main classes out of RFS into their own projects and put them onto the migration console
opensearch-project · Jun 18, 2024 · 5c9e7f1 · 5c9e7f1
2 parents 2fa83e2 + 6381c73
commit 5c9e7f1
Show file tree

Hide file tree

Showing 10 changed files with 200 additions and 39 deletions.
diff --git a/.gitignore b/.gitignore
@@ -24,3 +24,4 @@ TrafficCapture/**/out/
 # CDK files from end-to-end testing
 opensearch-cluster-cdk/
 test/opensearch-cluster-cdk/
+TrafficCapture/dockerSolution/src/main/docker/migrationConsole/staging
diff --git a/CreateSnapshot/build.gradle b/CreateSnapshot/build.gradle
@@ -0,0 +1,49 @@
+plugins {
+    id 'application'
+    id 'java'
+    id 'jacoco'
+    id 'io.freefair.lombok' version '8.6'
+}
+
+import org.opensearch.migrations.common.CommonUtils
+
+java.sourceCompatibility = JavaVersion.VERSION_11
+java.targetCompatibility = JavaVersion.VERSION_11
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    implementation project(":commonDependencyVersionConstraints")
+
+    implementation project(":RFS")
+    implementation group: 'com.beust', name: 'jcommander'
+    implementation group: 'org.slf4j', name: 'slf4j-api'
+    implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl'
+}
+
+application {
+    mainClassName = 'com.rfs.CreateSnapshot'
+}
+
+// Utility task to allow copying required libraries into a 'dependencies' folder for security scanning
+tasks.register('copyDependencies', Sync) {
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+
+    from configurations.runtimeClasspath
+    into "${buildDir}/dependencies"
+}
+
+jacocoTestReport {
+    reports {
+        xml.required = true
+        xml.destination file("${buildDir}/reports/jacoco/test/jacocoTestReport.xml")
+        html.required = true
+        html.destination file("${buildDir}/reports/jacoco/test/html")
+    }
+}
+
+test {
+    useJUnitPlatform()
+}
diff --git a/.../main/java/com/rfs/RfsCreateSnapshot.java → ...src/main/java/com/rfs/CreateSnapshot.java b/.../main/java/com/rfs/RfsCreateSnapshot.java → ...src/main/java/com/rfs/CreateSnapshot.java
@@ -3,6 +3,7 @@
 import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
 
+import lombok.extern.slf4j.Slf4j;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
@@ -18,9 +19,8 @@
 import com.rfs.worker.GlobalState;
 import com.rfs.worker.SnapshotRunner;
 
-public class RfsCreateSnapshot {
-    private static final Logger logger = LogManager.getLogger(RfsCreateSnapshot.class);
-
+@Slf4j
+public class CreateSnapshot {
     public static class Args {
         @Parameter(names = {"--snapshot-name"}, description = "The name of the snapshot to migrate", required = true)
         public String snapshotName;
@@ -48,9 +48,6 @@ public static class Args {
 
         @Parameter(names = {"--target-password"}, description = "Optional.  The target password; if not provided, will assume no auth on target", required = false)
         public String targetPass = null;
-
-        @Parameter(names = {"--log-level"}, description = "What log level you want.  Default: 'info'", required = false, converter = Logging.ArgsConverter.class)
-        public Level logLevel = Level.INFO;
     }
 
     public static void main(String[] args) throws Exception {
@@ -70,15 +67,12 @@ public static void main(String[] args) throws Exception {
         final String targetHost = arguments.targetHost;
         final String targetUser = arguments.targetUser;
         final String targetPass = arguments.targetPass;
-        final Level logLevel = arguments.logLevel;
-
-        Logging.setLevel(logLevel);
 
         final ConnectionDetails sourceConnection = new ConnectionDetails(sourceHost, sourceUser, sourcePass);
         final ConnectionDetails targetConnection = new ConnectionDetails(targetHost, targetUser, targetPass);
 
         TryHandlePhaseFailure.executeWithTryCatch(() -> {
-            logger.info("Running RfsWorker");
+            log.info("Running RfsWorker");
             GlobalState globalState = GlobalState.getInstance();
             OpenSearchClient sourceClient = new OpenSearchClient(sourceConnection);
             OpenSearchClient targetClient = new OpenSearchClient(targetConnection);

diff --git a/DocumentsFromSnapshotMigration/build.gradle b/DocumentsFromSnapshotMigration/build.gradle
@@ -0,0 +1,49 @@
+plugins {
+    id 'application'
+    id 'java'
+    id 'jacoco'
+    id 'io.freefair.lombok' version '8.6'
+}
+
+import org.opensearch.migrations.common.CommonUtils
+
+java.sourceCompatibility = JavaVersion.VERSION_11
+java.targetCompatibility = JavaVersion.VERSION_11
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    implementation project(":commonDependencyVersionConstraints")
+
+    implementation project(":RFS")
+    implementation group: 'com.beust', name: 'jcommander'
+    implementation group: 'org.slf4j', name: 'slf4j-api'
+    implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl'
+}
+
+application {
+    mainClassName = 'com.rfs.RfsMigrateDocuments'
+}
+
+// Utility task to allow copying required libraries into a 'dependencies' folder for security scanning
+tasks.register('copyDependencies', Sync) {
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+
+    from configurations.runtimeClasspath
+    into "${buildDir}/dependencies"
+}
+
+jacocoTestReport {
+    reports {
+        xml.required = true
+        xml.destination file("${buildDir}/reports/jacoco/test/jacocoTestReport.xml")
+        html.required = true
+        html.destination file("${buildDir}/reports/jacoco/test/html")
+    }
+}
+
+test {
+    useJUnitPlatform()
+}
diff --git a/...ain/java/com/rfs/RfsMigrateDocuments.java → ...ain/java/com/rfs/RfsMigrateDocuments.java b/...ain/java/com/rfs/RfsMigrateDocuments.java → ...ain/java/com/rfs/RfsMigrateDocuments.java
@@ -6,6 +6,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
+import lombok.extern.slf4j.Slf4j;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
@@ -34,9 +35,8 @@
 import com.rfs.worker.DocumentsRunner;
 import com.rfs.worker.GlobalState;
 
+@Slf4j
 public class RfsMigrateDocuments {
-    private static final Logger logger = LogManager.getLogger(RfsMigrateDocuments.class);
-
     public static class Args {
         @Parameter(names = {"--snapshot-name"}, description = "The name of the snapshot to migrate", required = true)
         public String snapshotName;
@@ -65,9 +65,6 @@ public static class Args {
         @Parameter(names = {"--max-shard-size-bytes"}, description = ("Optional. The maximum shard size, in bytes, to allow when"
             + " performing the document migration.  Useful for preventing disk overflow.  Default: 50 * 1024 * 1024 * 1024 (50 GB)"), required = false)
         public long maxShardSizeBytes = 50 * 1024 * 1024 * 1024L;
-
-        @Parameter(names = {"--log-level"}, description = "What log level you want.  Default: 'info'", required = false, converter = Logging.ArgsConverter.class)
-        public Level logLevel = Level.INFO;
     }
 
     public static void main(String[] args) throws Exception {
@@ -87,14 +84,11 @@ public static void main(String[] args) throws Exception {
         final String targetUser = arguments.targetUser;
         final String targetPass = arguments.targetPass;
         final long maxShardSizeBytes = arguments.maxShardSizeBytes;
-        final Level logLevel = arguments.logLevel;
-
-        Logging.setLevel(logLevel);
 
         final ConnectionDetails targetConnection = new ConnectionDetails(targetHost, targetUser, targetPass);
 
         TryHandlePhaseFailure.executeWithTryCatch(() -> {
-            logger.info("Running RfsWorker");
+            log.info("Running RfsWorker");
 
             GlobalState globalState = GlobalState.getInstance();
             OpenSearchClient targetClient = new OpenSearchClient(targetConnection);

diff --git a/MetadataMigration/build.gradle b/MetadataMigration/build.gradle
@@ -0,0 +1,49 @@
+plugins {
+    id 'application'
+    id 'java'
+    id 'jacoco'
+    id 'io.freefair.lombok' version '8.6'
+}
+
+import org.opensearch.migrations.common.CommonUtils
+
+java.sourceCompatibility = JavaVersion.VERSION_11
+java.targetCompatibility = JavaVersion.VERSION_11
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    implementation project(":commonDependencyVersionConstraints")
+
+    implementation project(":RFS")
+    implementation group: 'com.beust', name: 'jcommander'
+    implementation group: 'org.slf4j', name: 'slf4j-api'
+    implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl'
+}
+
+application {
+    mainClassName = 'com.rfs.MetadataMigration'
+}
+
+// Utility task to allow copying required libraries into a 'dependencies' folder for security scanning
+tasks.register('copyDependencies', Sync) {
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+
+    from configurations.runtimeClasspath
+    into "${buildDir}/dependencies"
+}
+
+jacocoTestReport {
+    reports {
+        xml.required = true
+        xml.destination file("${buildDir}/reports/jacoco/test/jacocoTestReport.xml")
+        html.required = true
+        html.destination file("${buildDir}/reports/jacoco/test/html")
+    }
+}
+
+test {
+    useJUnitPlatform()
+}
diff --git a/...main/java/com/rfs/RfsMigrateMetadata.java → .../main/java/com/rfs/MetadataMigration.java b/...main/java/com/rfs/RfsMigrateMetadata.java → .../main/java/com/rfs/MetadataMigration.java
@@ -7,18 +7,12 @@
 import java.nio.file.Paths;
 import java.util.List;
 
-import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.LogManager;
-
-
 import com.rfs.cms.CmsClient;
 import com.rfs.cms.OpenSearchCmsClient;
 import com.rfs.common.ClusterVersion;
 import com.rfs.common.ConnectionDetails;
 import com.rfs.common.GlobalMetadata;
 import com.rfs.common.IndexMetadata;
-import com.rfs.common.Logging;
 import com.rfs.common.OpenSearchClient;
 import com.rfs.common.S3Uri;
 import com.rfs.common.S3Repo;
@@ -35,10 +29,11 @@
 import com.rfs.worker.GlobalState;
 import com.rfs.worker.IndexRunner;
 import com.rfs.worker.MetadataRunner;
+import lombok.extern.slf4j.Slf4j;
 
-public class RfsMigrateMetadata {
-    private static final Logger logger = LogManager.getLogger(RfsMigrateMetadata.class);
-
+@Slf4j
+public class MetadataMigration {
+    
     public static class Args {
         @Parameter(names = {"--snapshot-name"}, description = "The name of the snapshot to migrate", required = true)
         public String snapshotName;
@@ -78,9 +73,6 @@ public static class Args {
             + " This can be useful for migrating to targets which use zonal deployments and require additional replicas to meet zone requirements.  Default: 0")
             , required = false)
         public int minNumberOfReplicas = 0;
-
-        @Parameter(names = {"--log-level"}, description = "What log level you want.  Default: 'info'", required = false, converter = Logging.ArgsConverter.class)
-        public Level logLevel = Level.INFO;
     }
 
     public static void main(String[] args) throws Exception {
@@ -101,14 +93,11 @@ public static void main(String[] args) throws Exception {
         final List<String> indexTemplateAllowlist = arguments.indexTemplateAllowlist;
         final List<String> componentTemplateAllowlist = arguments.componentTemplateAllowlist;
         final int awarenessDimensionality = arguments.minNumberOfReplicas + 1;
-        final Level logLevel = arguments.logLevel;
-
-        Logging.setLevel(logLevel);
 
         final ConnectionDetails targetConnection = new ConnectionDetails(targetHost, targetUser, targetPass);
 
         TryHandlePhaseFailure.executeWithTryCatch(() -> {
-            logger.info("Running RfsWorker");
+            log.info("Running RfsWorker");
             GlobalState globalState = GlobalState.getInstance();
             OpenSearchClient targetClient = new OpenSearchClient(targetConnection);
             final CmsClient cmsClient = new OpenSearchCmsClient(targetClient);

diff --git a/TrafficCapture/dockerSolution/build.gradle b/TrafficCapture/dockerSolution/build.gradle
@@ -37,9 +37,12 @@ dockerFilesForExternalServices.each { projectName, dockerImageName ->
     def escapedProjectName = projectName;
     task("buildDockerImage_${escapedProjectName}", type: DockerBuildImage) {
         if (escapedProjectName == "migrationConsole") {
-            def destDir = "src/main/docker/${escapedProjectName}/build/jars"
-            CommonUtils.copyArtifactFromProjectToProjectsDockerStaging(project as Project,
-                    project(":TrafficCapture:trafficReplayer"), projectName, destDir)
+            copyMigrationConsoleArtifactsFromProjectsToProjectsDockerStaging(project as Project,
+                    List.of(project(":TrafficCapture:trafficReplayer"),
+                            project(":CreateSnapshot"),
+                            project(":MetadataMigration"),
+                    ), projectName, "src/main/docker/${escapedProjectName}")
+
             dependsOn "copyArtifact_${escapedProjectName}"
             dependsOn "buildDockerImage_elasticsearchTestConsole"
         }
@@ -50,6 +53,33 @@ dockerFilesForExternalServices.each { projectName, dockerImageName ->
     }
 }
 
+
+static def copyMigrationConsoleArtifactsFromProjectsToProjectsDockerStaging(Project dockerBuildProject,
+                                                                            List<Project> sourceArtifactProjects,
+                                                                            String destProjectName, String destDir) {
+    // Create the overarching task
+    def parentCopyTask = dockerBuildProject.tasks.create("copyArtifact_${destProjectName}")
+
+    sourceArtifactProjects.each { sourceArtifactProject ->
+        def applicationDestDir = "${destDir}/staging/${sourceArtifactProject.name}/";
+        def libCopyTask = dockerBuildProject.tasks.create("copyLibArtifacts_${destProjectName}_${sourceArtifactProject.name}", Sync) {
+            from { sourceArtifactProject.configurations.findByName("runtimeClasspath").files }
+            from { sourceArtifactProject.tasks.getByName('jar') }
+            into "${applicationDestDir}/lib"
+        }
+        def binCopyTask = dockerBuildProject.tasks.create("copyBinArtifacts_${destProjectName}_${sourceArtifactProject.name}", Sync) {
+            from { sourceArtifactProject.tasks.getByName('startScripts').outputs.files }
+            into "${applicationDestDir}/bin"
+        }
+        libCopyTask.dependsOn(sourceArtifactProject.tasks.named("assemble"))
+        binCopyTask.dependsOn(sourceArtifactProject.tasks.named("assemble"))
+
+        // Make the parent task depend on this individual task
+        parentCopyTask.dependsOn(libCopyTask)
+        parentCopyTask.dependsOn(binCopyTask)
+    }
+}
+
 def javaContainerServices = [
         ":TrafficCapture:trafficCaptureProxyServer": "capture_proxy",
         ":TrafficCapture:trafficReplayer": "traffic_replayer"

diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile
@@ -10,10 +10,13 @@ RUN wget -O kafka/libs/msk-iam-auth.jar https://github.com/aws/aws-msk-iam-auth/
 WORKDIR /root
 
 # Add Traffic Replayer jars for running KafkaPrinter from this container
-COPY build/jars /root/kafka-tools/replayer-jars
+COPY staging/trafficReplayer/lib /root/kafka-tools/replayer-jars
 RUN printf "#!/bin/sh\njava -cp `echo /root/kafka-tools/replayer-jars/*.jar | tr \   :` \"\$@\" " > /root/kafka-tools/runJavaWithClasspath.sh
 RUN chmod +x /root/kafka-tools/runJavaWithClasspath.sh
 
+COPY staging/CreateSnapshot /root/createSnapshot
+COPY staging/MetadataMigration /root/metadataMigration
+
 COPY osiPipelineTemplate.yaml /root/
 COPY msk-iam-auth.properties /root/kafka-tools/aws
 COPY kafkaCmdRef.md /root/kafka-tools

diff --git a/settings.gradle b/settings.gradle
@@ -42,6 +42,9 @@ include 'commonDependencyVersionConstraints'
 include 'coreUtilities'
 include 'testHelperFixtures'
 include 'RFS'
+include 'CreateSnapshot'
+include 'MetadataMigration'
+include 'DocumentsFromSnapshotMigration'
 include 'TrafficCapture:captureKafkaOffloader'
 include 'TrafficCapture:captureOffloader'
 include 'TrafficCapture:captureProtobufs'