From 67bdd194bca220923de8c326f7041ea7c187a14c Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Mon, 17 Jun 2024 19:58:05 -0400 Subject: [PATCH] Move the 3 main classes out of RFS into their own projects and updated the migration console to pull them onto its image. I dropped the "RFS" prefix when not appropriate and placed the work 'migration' at the end of docs migration and metadata migration to make command completion more unique. Signed-off-by: Greg Schohn --- .gitignore | 1 + CreateSnapshot/build.gradle | 49 +++++++++++++++++++ .../src/main/java/com/rfs/CreateSnapshot.java | 14 ++---- DocumentsFromSnapshotMigration/build.gradle | 49 +++++++++++++++++++ .../java/com/rfs/RfsMigrateDocuments.java | 12 ++--- MetadataMigration/build.gradle | 49 +++++++++++++++++++ .../main/java/com/rfs/MetadataMigration.java | 21 ++------ TrafficCapture/dockerSolution/build.gradle | 36 ++++++++++++-- .../main/docker/migrationConsole/Dockerfile | 5 +- settings.gradle | 3 ++ 10 files changed, 200 insertions(+), 39 deletions(-) create mode 100644 CreateSnapshot/build.gradle rename RFS/src/main/java/com/rfs/RfsCreateSnapshot.java => CreateSnapshot/src/main/java/com/rfs/CreateSnapshot.java (89%) create mode 100644 DocumentsFromSnapshotMigration/build.gradle rename {RFS => DocumentsFromSnapshotMigration}/src/main/java/com/rfs/RfsMigrateDocuments.java (92%) create mode 100644 MetadataMigration/build.gradle rename RFS/src/main/java/com/rfs/RfsMigrateMetadata.java => MetadataMigration/src/main/java/com/rfs/MetadataMigration.java (91%) diff --git a/.gitignore b/.gitignore index 979e297fa..cbaed2c10 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ TrafficCapture/**/out/ # CDK files from end-to-end testing opensearch-cluster-cdk/ test/opensearch-cluster-cdk/ +TrafficCapture/dockerSolution/src/main/docker/migrationConsole/staging diff --git a/CreateSnapshot/build.gradle b/CreateSnapshot/build.gradle new file mode 100644 index 000000000..d01e8275d --- /dev/null +++ b/CreateSnapshot/build.gradle @@ -0,0 +1,49 @@ +plugins { + id 'application' + id 'java' + id 'jacoco' + id 'io.freefair.lombok' version '8.6' +} + +import org.opensearch.migrations.common.CommonUtils + +java.sourceCompatibility = JavaVersion.VERSION_11 +java.targetCompatibility = JavaVersion.VERSION_11 + +repositories { + mavenCentral() +} + +dependencies { + implementation project(":commonDependencyVersionConstraints") + + implementation project(":RFS") + implementation group: 'com.beust', name: 'jcommander' + implementation group: 'org.slf4j', name: 'slf4j-api' + implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl' +} + +application { + mainClassName = 'com.rfs.CreateSnapshot' +} + +// Utility task to allow copying required libraries into a 'dependencies' folder for security scanning +tasks.register('copyDependencies', Sync) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + + from configurations.runtimeClasspath + into "${buildDir}/dependencies" +} + +jacocoTestReport { + reports { + xml.required = true + xml.destination file("${buildDir}/reports/jacoco/test/jacocoTestReport.xml") + html.required = true + html.destination file("${buildDir}/reports/jacoco/test/html") + } +} + +test { + useJUnitPlatform() +} \ No newline at end of file diff --git a/RFS/src/main/java/com/rfs/RfsCreateSnapshot.java b/CreateSnapshot/src/main/java/com/rfs/CreateSnapshot.java similarity index 89% rename from RFS/src/main/java/com/rfs/RfsCreateSnapshot.java rename to CreateSnapshot/src/main/java/com/rfs/CreateSnapshot.java index fd16d70cd..5b2b72330 100644 --- a/RFS/src/main/java/com/rfs/RfsCreateSnapshot.java +++ b/CreateSnapshot/src/main/java/com/rfs/CreateSnapshot.java @@ -3,6 +3,7 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; +import lombok.extern.slf4j.Slf4j; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -18,9 +19,8 @@ import com.rfs.worker.GlobalState; import com.rfs.worker.SnapshotRunner; -public class RfsCreateSnapshot { - private static final Logger logger = LogManager.getLogger(RfsCreateSnapshot.class); - +@Slf4j +public class CreateSnapshot { public static class Args { @Parameter(names = {"--snapshot-name"}, description = "The name of the snapshot to migrate", required = true) public String snapshotName; @@ -48,9 +48,6 @@ public static class Args { @Parameter(names = {"--target-password"}, description = "Optional. The target password; if not provided, will assume no auth on target", required = false) public String targetPass = null; - - @Parameter(names = {"--log-level"}, description = "What log level you want. Default: 'info'", required = false, converter = Logging.ArgsConverter.class) - public Level logLevel = Level.INFO; } public static void main(String[] args) throws Exception { @@ -70,15 +67,12 @@ public static void main(String[] args) throws Exception { final String targetHost = arguments.targetHost; final String targetUser = arguments.targetUser; final String targetPass = arguments.targetPass; - final Level logLevel = arguments.logLevel; - - Logging.setLevel(logLevel); final ConnectionDetails sourceConnection = new ConnectionDetails(sourceHost, sourceUser, sourcePass); final ConnectionDetails targetConnection = new ConnectionDetails(targetHost, targetUser, targetPass); TryHandlePhaseFailure.executeWithTryCatch(() -> { - logger.info("Running RfsWorker"); + log.info("Running RfsWorker"); GlobalState globalState = GlobalState.getInstance(); OpenSearchClient sourceClient = new OpenSearchClient(sourceConnection); OpenSearchClient targetClient = new OpenSearchClient(targetConnection); diff --git a/DocumentsFromSnapshotMigration/build.gradle b/DocumentsFromSnapshotMigration/build.gradle new file mode 100644 index 000000000..06ad89c24 --- /dev/null +++ b/DocumentsFromSnapshotMigration/build.gradle @@ -0,0 +1,49 @@ +plugins { + id 'application' + id 'java' + id 'jacoco' + id 'io.freefair.lombok' version '8.6' +} + +import org.opensearch.migrations.common.CommonUtils + +java.sourceCompatibility = JavaVersion.VERSION_11 +java.targetCompatibility = JavaVersion.VERSION_11 + +repositories { + mavenCentral() +} + +dependencies { + implementation project(":commonDependencyVersionConstraints") + + implementation project(":RFS") + implementation group: 'com.beust', name: 'jcommander' + implementation group: 'org.slf4j', name: 'slf4j-api' + implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl' +} + +application { + mainClassName = 'com.rfs.RfsMigrateDocuments' +} + +// Utility task to allow copying required libraries into a 'dependencies' folder for security scanning +tasks.register('copyDependencies', Sync) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + + from configurations.runtimeClasspath + into "${buildDir}/dependencies" +} + +jacocoTestReport { + reports { + xml.required = true + xml.destination file("${buildDir}/reports/jacoco/test/jacocoTestReport.xml") + html.required = true + html.destination file("${buildDir}/reports/jacoco/test/html") + } +} + +test { + useJUnitPlatform() +} \ No newline at end of file diff --git a/RFS/src/main/java/com/rfs/RfsMigrateDocuments.java b/DocumentsFromSnapshotMigration/src/main/java/com/rfs/RfsMigrateDocuments.java similarity index 92% rename from RFS/src/main/java/com/rfs/RfsMigrateDocuments.java rename to DocumentsFromSnapshotMigration/src/main/java/com/rfs/RfsMigrateDocuments.java index b60052222..4d5b716e5 100644 --- a/RFS/src/main/java/com/rfs/RfsMigrateDocuments.java +++ b/DocumentsFromSnapshotMigration/src/main/java/com/rfs/RfsMigrateDocuments.java @@ -6,6 +6,7 @@ import java.nio.file.Path; import java.nio.file.Paths; +import lombok.extern.slf4j.Slf4j; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -34,9 +35,8 @@ import com.rfs.worker.DocumentsRunner; import com.rfs.worker.GlobalState; +@Slf4j public class RfsMigrateDocuments { - private static final Logger logger = LogManager.getLogger(RfsMigrateDocuments.class); - public static class Args { @Parameter(names = {"--snapshot-name"}, description = "The name of the snapshot to migrate", required = true) public String snapshotName; @@ -65,9 +65,6 @@ public static class Args { @Parameter(names = {"--max-shard-size-bytes"}, description = ("Optional. The maximum shard size, in bytes, to allow when" + " performing the document migration. Useful for preventing disk overflow. Default: 50 * 1024 * 1024 * 1024 (50 GB)"), required = false) public long maxShardSizeBytes = 50 * 1024 * 1024 * 1024L; - - @Parameter(names = {"--log-level"}, description = "What log level you want. Default: 'info'", required = false, converter = Logging.ArgsConverter.class) - public Level logLevel = Level.INFO; } public static void main(String[] args) throws Exception { @@ -87,14 +84,11 @@ public static void main(String[] args) throws Exception { final String targetUser = arguments.targetUser; final String targetPass = arguments.targetPass; final long maxShardSizeBytes = arguments.maxShardSizeBytes; - final Level logLevel = arguments.logLevel; - - Logging.setLevel(logLevel); final ConnectionDetails targetConnection = new ConnectionDetails(targetHost, targetUser, targetPass); TryHandlePhaseFailure.executeWithTryCatch(() -> { - logger.info("Running RfsWorker"); + log.info("Running RfsWorker"); GlobalState globalState = GlobalState.getInstance(); OpenSearchClient targetClient = new OpenSearchClient(targetConnection); diff --git a/MetadataMigration/build.gradle b/MetadataMigration/build.gradle new file mode 100644 index 000000000..2cb5f3778 --- /dev/null +++ b/MetadataMigration/build.gradle @@ -0,0 +1,49 @@ +plugins { + id 'application' + id 'java' + id 'jacoco' + id 'io.freefair.lombok' version '8.6' +} + +import org.opensearch.migrations.common.CommonUtils + +java.sourceCompatibility = JavaVersion.VERSION_11 +java.targetCompatibility = JavaVersion.VERSION_11 + +repositories { + mavenCentral() +} + +dependencies { + implementation project(":commonDependencyVersionConstraints") + + implementation project(":RFS") + implementation group: 'com.beust', name: 'jcommander' + implementation group: 'org.slf4j', name: 'slf4j-api' + implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl' +} + +application { + mainClassName = 'com.rfs.MetadataMigration' +} + +// Utility task to allow copying required libraries into a 'dependencies' folder for security scanning +tasks.register('copyDependencies', Sync) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + + from configurations.runtimeClasspath + into "${buildDir}/dependencies" +} + +jacocoTestReport { + reports { + xml.required = true + xml.destination file("${buildDir}/reports/jacoco/test/jacocoTestReport.xml") + html.required = true + html.destination file("${buildDir}/reports/jacoco/test/html") + } +} + +test { + useJUnitPlatform() +} \ No newline at end of file diff --git a/RFS/src/main/java/com/rfs/RfsMigrateMetadata.java b/MetadataMigration/src/main/java/com/rfs/MetadataMigration.java similarity index 91% rename from RFS/src/main/java/com/rfs/RfsMigrateMetadata.java rename to MetadataMigration/src/main/java/com/rfs/MetadataMigration.java index 41ae31bd3..35cc8bbca 100644 --- a/RFS/src/main/java/com/rfs/RfsMigrateMetadata.java +++ b/MetadataMigration/src/main/java/com/rfs/MetadataMigration.java @@ -7,18 +7,12 @@ import java.nio.file.Paths; import java.util.List; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; - - import com.rfs.cms.CmsClient; import com.rfs.cms.OpenSearchCmsClient; import com.rfs.common.ClusterVersion; import com.rfs.common.ConnectionDetails; import com.rfs.common.GlobalMetadata; import com.rfs.common.IndexMetadata; -import com.rfs.common.Logging; import com.rfs.common.OpenSearchClient; import com.rfs.common.S3Uri; import com.rfs.common.S3Repo; @@ -35,10 +29,11 @@ import com.rfs.worker.GlobalState; import com.rfs.worker.IndexRunner; import com.rfs.worker.MetadataRunner; +import lombok.extern.slf4j.Slf4j; -public class RfsMigrateMetadata { - private static final Logger logger = LogManager.getLogger(RfsMigrateMetadata.class); - +@Slf4j +public class MetadataMigration { + public static class Args { @Parameter(names = {"--snapshot-name"}, description = "The name of the snapshot to migrate", required = true) public String snapshotName; @@ -78,9 +73,6 @@ public static class Args { + " This can be useful for migrating to targets which use zonal deployments and require additional replicas to meet zone requirements. Default: 0") , required = false) public int minNumberOfReplicas = 0; - - @Parameter(names = {"--log-level"}, description = "What log level you want. Default: 'info'", required = false, converter = Logging.ArgsConverter.class) - public Level logLevel = Level.INFO; } public static void main(String[] args) throws Exception { @@ -101,14 +93,11 @@ public static void main(String[] args) throws Exception { final List indexTemplateAllowlist = arguments.indexTemplateAllowlist; final List componentTemplateAllowlist = arguments.componentTemplateAllowlist; final int awarenessDimensionality = arguments.minNumberOfReplicas + 1; - final Level logLevel = arguments.logLevel; - - Logging.setLevel(logLevel); final ConnectionDetails targetConnection = new ConnectionDetails(targetHost, targetUser, targetPass); TryHandlePhaseFailure.executeWithTryCatch(() -> { - logger.info("Running RfsWorker"); + log.info("Running RfsWorker"); GlobalState globalState = GlobalState.getInstance(); OpenSearchClient targetClient = new OpenSearchClient(targetConnection); final CmsClient cmsClient = new OpenSearchCmsClient(targetClient); diff --git a/TrafficCapture/dockerSolution/build.gradle b/TrafficCapture/dockerSolution/build.gradle index bbdf3eb20..f867fd253 100644 --- a/TrafficCapture/dockerSolution/build.gradle +++ b/TrafficCapture/dockerSolution/build.gradle @@ -37,9 +37,12 @@ dockerFilesForExternalServices.each { projectName, dockerImageName -> def escapedProjectName = projectName; task("buildDockerImage_${escapedProjectName}", type: DockerBuildImage) { if (escapedProjectName == "migrationConsole") { - def destDir = "src/main/docker/${escapedProjectName}/build/jars" - CommonUtils.copyArtifactFromProjectToProjectsDockerStaging(project as Project, - project(":TrafficCapture:trafficReplayer"), projectName, destDir) + copyMigrationConsoleArtifactsFromProjectsToProjectsDockerStaging(project as Project, + List.of(project(":TrafficCapture:trafficReplayer"), + project(":CreateSnapshot"), + project(":MetadataMigration"), + ), projectName, "src/main/docker/${escapedProjectName}") + dependsOn "copyArtifact_${escapedProjectName}" dependsOn "buildDockerImage_elasticsearchTestConsole" } @@ -50,6 +53,33 @@ dockerFilesForExternalServices.each { projectName, dockerImageName -> } } + +static def copyMigrationConsoleArtifactsFromProjectsToProjectsDockerStaging(Project dockerBuildProject, + List sourceArtifactProjects, + String destProjectName, String destDir) { + // Create the overarching task + def parentCopyTask = dockerBuildProject.tasks.create("copyArtifact_${destProjectName}") + + sourceArtifactProjects.each { sourceArtifactProject -> + def applicationDestDir = "${destDir}/staging/${sourceArtifactProject.name}/"; + def libCopyTask = dockerBuildProject.tasks.create("copyLibArtifacts_${destProjectName}_${sourceArtifactProject.name}", Sync) { + from { sourceArtifactProject.configurations.findByName("runtimeClasspath").files } + from { sourceArtifactProject.tasks.getByName('jar') } + into "${applicationDestDir}/lib" + } + def binCopyTask = dockerBuildProject.tasks.create("copyBinArtifacts_${destProjectName}_${sourceArtifactProject.name}", Sync) { + from { sourceArtifactProject.tasks.getByName('startScripts').outputs.files } + into "${applicationDestDir}/bin" + } + libCopyTask.dependsOn(sourceArtifactProject.tasks.named("assemble")) + binCopyTask.dependsOn(sourceArtifactProject.tasks.named("assemble")) + + // Make the parent task depend on this individual task + parentCopyTask.dependsOn(libCopyTask) + parentCopyTask.dependsOn(binCopyTask) + } +} + def javaContainerServices = [ ":TrafficCapture:trafficCaptureProxyServer": "capture_proxy", ":TrafficCapture:trafficReplayer": "traffic_replayer" diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile index 153e146c0..71b8f3833 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile @@ -10,10 +10,13 @@ RUN wget -O kafka/libs/msk-iam-auth.jar https://github.com/aws/aws-msk-iam-auth/ WORKDIR /root # Add Traffic Replayer jars for running KafkaPrinter from this container -COPY build/jars /root/kafka-tools/replayer-jars +COPY staging/trafficReplayer/lib /root/kafka-tools/replayer-jars RUN printf "#!/bin/sh\njava -cp `echo /root/kafka-tools/replayer-jars/*.jar | tr \ :` \"\$@\" " > /root/kafka-tools/runJavaWithClasspath.sh RUN chmod +x /root/kafka-tools/runJavaWithClasspath.sh +COPY staging/CreateSnapshot /root/createSnapshot +COPY staging/MetadataMigration /root/metadataMigration + COPY osiPipelineTemplate.yaml /root/ COPY msk-iam-auth.properties /root/kafka-tools/aws COPY kafkaCmdRef.md /root/kafka-tools diff --git a/settings.gradle b/settings.gradle index 687f007bf..db34d00a7 100644 --- a/settings.gradle +++ b/settings.gradle @@ -42,6 +42,9 @@ include 'commonDependencyVersionConstraints' include 'coreUtilities' include 'testHelperFixtures' include 'RFS' +include 'CreateSnapshot' +include 'MetadataMigration' +include 'DocumentsFromSnapshotMigration' include 'TrafficCapture:captureKafkaOffloader' include 'TrafficCapture:captureOffloader' include 'TrafficCapture:captureProtobufs'