Jenkins/RFS bug fixes for working pipeline (opensearch-project#830)

Required fixes to get RFS flow working as well as fix minor Jenkins issues

---------

Signed-off-by: Tanner Lewis <[email protected]>

MetadataMigration/src/main/java/com/rfs/MetadataMigration.java

@@ -60,7 +60,7 @@ public static class Args {
         public String s3Region;
 
         @ParametersDelegate
-        public ConnectionDetails.TargetArgs targetArgs;
+        public ConnectionDetails.TargetArgs targetArgs = new ConnectionDetails.TargetArgs();
 
         @Parameter(names = { "--index-allowlist" }, description = ("Optional.  List of index names to migrate"
             + " (e.g. 'logs_2024_01, logs_2024_02').  Default: all non-system indices (e.g. those not starting with '.')"), required = false)

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/metadata.py

@@ -175,9 +175,9 @@ def migrate(self, detached_log=None) -> CommandResult:
 
         if self._component_template_allowlist:
             command.extend(["--component-template-allowlist", ",".join(self._component_template_allowlist)])
-        
+
         if self._otel_endpoint:
-            command.extend(["--otelCollectorEndpoint", self._otel_endpoint])
+            command.extend(["--otel-collector-endpoint", self._otel_endpoint])
 
         if password_field_index:
             display_command = command[:password_field_index] + ["********"] + command[password_field_index:]

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/snapshot.py

@@ -105,7 +105,7 @@ def create(self, *args, **kwargs) -> CommandResult:
             command.extend(["--max-snapshot-rate-mb-per-node",
                             str(max_snapshot_rate_mb_per_node)])
         if self.otel_endpoint:
-            command.extend(["--otelCollectorEndpoint", self.otel_endpoint])
+            command.extend(["--otel-collector-endpoint", self.otel_endpoint])
 
         logger.info(f"Creating snapshot with command: {' '.join(command)}")
         try:
@@ -149,7 +149,7 @@ def create(self, *args, **kwargs) -> CommandResult:
         if self.source_cluster.allow_insecure:
             command.append("--source-insecure")
         if self.otel_endpoint:
-            command.extend(["--otelCollectorEndpoint", self.otel_endpoint])
+            command.extend(["--otel-collector-endpoint", self.otel_endpoint])
 
         logger.info(f"Creating snapshot with command: {' '.join(command)}")
         try:

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_metadata.py

@@ -168,7 +168,7 @@ def test_metadata_with_s3_snapshot_makes_correct_subprocess_call(mocker):
         "--s3-repo-uri", config["from_snapshot"]["s3"]["repo_uri"],
         "--s3-region", config["from_snapshot"]["s3"]["aws_region"],
         "--target-insecure",
-        "--otelCollectorEndpoint", config["otel_endpoint"],
+        "--otel-collector-endpoint", config["otel_endpoint"],
     ], stdout=None, stderr=None, text=True, check=True
     )
 
@@ -196,7 +196,7 @@ def test_metadata_with_fs_snapshot_makes_correct_subprocess_call(mocker):
         "--min-replicas", '0',
         "--file-system-repo-path", config["from_snapshot"]["fs"]["repo_path"],
         "--target-insecure",
-        "--otelCollectorEndpoint", config["otel_endpoint"],
+        "--otel-collector-endpoint", config["otel_endpoint"],
     ], stdout=None, stderr=None, text=True, check=True)
 
 
@@ -256,7 +256,7 @@ def test_metadata_with_allowlists_makes_correct_subprocess_call(mocker):
         "--index-allowlist", "index1,index2",
         "--index-template-allowlist", "index_template1,index_template2",
         "--component-template-allowlist", "component_template1,component_template2",
-        "--otelCollectorEndpoint", config["otel_endpoint"],
+        "--otel-collector-endpoint", config["otel_endpoint"],
     ], stdout=None, stderr=None, text=True, check=True
     )
 

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_snapshot.py

@@ -214,7 +214,7 @@ def test_fs_snapshot_create_calls_subprocess_run_with_correct_args(mocker):
                                   "--file-system-repo-path", config["snapshot"]["fs"]["repo_path"],
                                   "--source-host", source.endpoint,
                                   "--source-insecure",
-                                  "--otelCollectorEndpoint", config["snapshot"]["otel_endpoint"],
+                                  "--otel-collector-endpoint", config["snapshot"]["otel_endpoint"],
                                   ], stdout=None, stderr=None, text=True, check=True)
 
 
@@ -241,7 +241,7 @@ def test_s3_snapshot_create_calls_subprocess_run_with_correct_args(mocker):
                                   "--s3-region", config["snapshot"]["s3"]["aws_region"],
                                   "--source-host", source.endpoint,
                                   "--source-insecure", "--no-wait",
-                                  "--otelCollectorEndpoint", config["snapshot"]["otel_endpoint"],
+                                  "--otel-collector-endpoint", config["snapshot"]["otel_endpoint"],
                                   ], stdout=None, stderr=None, text=True, check=True)
 
 

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/backfill_tests.py

@@ -56,10 +56,14 @@ def setup_backfill(request):
     status_result: CommandResult = snapshot.status()
     if status_result.success:
         snapshot.delete()
-    snapshot.create(wait=True)
-    metadata.migrate()
-    backfill.start()
-    backfill.scale(units=10)
+    snapshot_result: CommandResult = snapshot.create(wait=True)
+    assert snapshot_result.success
+    metadata_result: CommandResult = metadata.migrate()
+    assert metadata_result.success
+    backfill_start_result: CommandResult = backfill.start()
+    assert backfill_start_result.success
+    backfill_scale_result: CommandResult = backfill.scale(units=10)
+    assert backfill_scale_result.success
 
 
 @pytest.fixture(scope="session", autouse=True)
@@ -104,4 +108,4 @@ def test_backfill_0002_sample_benchmarks(self):
 
         # Confirm documents on target after backfill
         check_doc_counts_match(cluster=target_cluster, expected_index_details=EXPECTED_BENCHMARK_DOCS,
-                               max_attempts=40, delay=30.0, test_case=self)
+                               max_attempts=30, delay=30.0, test_case=self)

jenkins/migrationIntegPipelines/ec2SourceE2EPipeline.groovy

@@ -1,5 +1,5 @@
 // Note:
-// 1. We are using an existing common VPC that we provide through a 'vpcId' parameter on the pipeline for now until we move
+// 1. We are using an existing common VPC that we provide through a manually created 'VPC_ID' parameter on the pipeline for now until we move
 //    to a proper Jenkins accounts and can create a setup without public subnets as well as request an extension to allow more than 5 VPCs per region
 // 2. There is a still a manual step needed on the EC2 source load balancer to replace its security group rule which allows all traffic (0.0.0.0/0) to
 //    allow traffic for the relevant service security group. This needs a better story around accepting user security groups in our Migration CDK.
@@ -10,7 +10,7 @@ def migrationContextId = 'migration-default'
 // all future runs should use the specified parameters
 def gitBranch = params.GIT_BRANCH ?: 'main'
 def gitUrl = params.GIT_REPO_URL ?: 'https://github.com/opensearch-project/opensearch-migrations.git'
-def vpcId = params.VPC_ID ?: 'vpc-00000000'
+def vpcId = params.VPC_ID
 def source_cdk_context = """
     {
       "source-single-node-ec2": {

jenkins/migrationIntegPipelines/rfsBackfillE2EPipeline.groovy

@@ -1,5 +1,5 @@
 // Note:
-// 1. We are using an existing common VPC that we provide through a 'vpcId' parameter on the pipeline for now until we move
+// 1. We are using an existing common VPC that we provide through a manually created 'VPC_ID' parameter on the pipeline for now until we move
 //    to a proper Jenkins accounts and can create a setup without public subnets as well as request an extension to allow more than 5 VPCs per region
 // 2. There is a still a manual step needed on the EC2 source load balancer to replace its security group rule which allows all traffic (0.0.0.0/0) to
 //    allow traffic for the relevant service security group. This needs a better story around accepting user security groups in our Migration CDK.
@@ -10,7 +10,7 @@ def migrationContextId = 'migration-rfs'
 // all future runs should use the specified parameters
 def gitBranch = params.GIT_BRANCH ?: 'main'
 def gitUrl = params.GIT_REPO_URL ?: 'https://github.com/opensearch-project/opensearch-migrations.git'
-def vpcId = params.VPC_ID ?: 'vpc-00000000'
+def vpcId = params.VPC_ID
 def source_cdk_context = """
     {
       "source-single-node-ec2": {

vars/defaultIntegPipeline.groovy

@@ -23,7 +23,6 @@ def call(Map config = [:]) {
             string(name: 'GIT_REPO_URL', defaultValue: 'https://github.com/opensearch-project/opensearch-migrations.git', description: 'Git repository url')
             string(name: 'GIT_BRANCH', defaultValue: 'main', description: 'Git branch to use for repository')
             string(name: 'STAGE', defaultValue: "${defaultStageId}", description: 'Stage name for deployment environment')
-            string(name: 'VPC_ID', description: 'VPC to use for deployments')
         }
 
         stages {