From 13034245bbaf73be52a471c0337a32945d08ad4c Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Fri, 18 Aug 2023 16:19:17 -0500 Subject: [PATCH 01/13] MIGRATIONS-1200 - Extract comparator and jupyter notebook Signed-off-by: Brian Presley --- TrafficCapture/dockerSolution/README.md | 10 --- TrafficCapture/dockerSolution/build.gradle | 64 +------------------ .../src/main/docker/docker-compose.yml | 33 +--------- .../ExpiringSubstitutableItemPoolTest.java | 4 +- .../http/HttpJsonTransformingConsumer.java | 4 +- .../lib/migration-assistance-stack.ts | 14 ---- deployment/copilot/README.md | 12 +--- deployment/copilot/devDeploy.sh | 9 +-- .../traffic-comparator-jupyter/manifest.yml | 56 ---------------- .../copilot/traffic-comparator/manifest.yml | 41 ------------ test/README.md | 3 +- test/tests.py | 8 +-- 12 files changed, 12 insertions(+), 246 deletions(-) delete mode 100644 deployment/copilot/traffic-comparator-jupyter/manifest.yml delete mode 100644 deployment/copilot/traffic-comparator/manifest.yml diff --git a/TrafficCapture/dockerSolution/README.md b/TrafficCapture/dockerSolution/README.md index 826669a72..6d2dc1967 100644 --- a/TrafficCapture/dockerSolution/README.md +++ b/TrafficCapture/dockerSolution/README.md @@ -8,16 +8,6 @@ down again. Notice that most of the Dockerfiles are dynamically constructed in the build hierarchy. Some efforts have been made to ensure that changes will make it into containers to be launched. -If a user wants to use their own checkout of the traffic-comparator repo, just set the environment variable " -TRAFFIC_COMPARATOR_DIRECTORY" to the directory that contains `setup.py`. Otherwise, if that isn't set, the traffic -comparator repo will be checked out to the build directory and that will be used. Notice that the checkout happens when -the directory wasn't present and there wasn't an environment variable specifying a directory. Once a directory exists, -it will be mounted to the traffic-comparator and jupyter services. - -Netcat is still used to connect several of the components and we're still working on improving the resiliency story -between these containers. The long term approach is to replace fixed streams with message bus approaches directly (i.e. -Kafka). In the short term, we can and are beginning, to leverage things like conditions on dependent services. - ### Running the Docker Solution While in the TrafficCapture directory, run the following command: diff --git a/TrafficCapture/dockerSolution/build.gradle b/TrafficCapture/dockerSolution/build.gradle index ecc2197b4..8a99a168d 100644 --- a/TrafficCapture/dockerSolution/build.gradle +++ b/TrafficCapture/dockerSolution/build.gradle @@ -9,19 +9,6 @@ import java.security.MessageDigest import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage import org.apache.tools.ant.taskdefs.condition.Os -def getTrafficComparatorDirectory() { - String overrideTrafficComparatorDirectory = System.getenv(TRAFFIC_COMPARATOR_DIRECTORY_ENV) - String rval = overrideTrafficComparatorDirectory != null ? - overrideTrafficComparatorDirectory : TRAFFIC_COMPARATOR_REPO_DIRECTORY; - return rval -} - -ext { - TRAFFIC_COMPARATOR_REPO_DIRECTORY = "build/traffic-comparator" - TRAFFIC_COMPARATOR_DIRECTORY_ENV = "TRAFFIC_COMPARATOR_DIRECTORY" - REALIZED_TRAFFIC_COMPARATOR_DIRECTORY = project.file(getTrafficComparatorDirectory()) - } - def calculateDockerHash = { projectName -> CommonUtils.calculateDockerHash(projectName, project) } @@ -31,17 +18,6 @@ dependencies { implementation project(':trafficReplayer') } -task cloneComparatorRepoIfNeeded(type: Exec) { - String comparatorDirectory = project.file(REALIZED_TRAFFIC_COMPARATOR_DIRECTORY); - String repo = 'https://github.com/opensearch-project/traffic-comparator.git' - onlyIf { - !(new File(comparatorDirectory).exists()) - } - commandLine = Os.isFamily(Os.FAMILY_WINDOWS) ? - ['git', 'clone', repo, TRAFFIC_COMPARATOR_REPO_DIRECTORY ] : - ['/bin/sh', '-c', "git clone ${repo} ${TRAFFIC_COMPARATOR_REPO_DIRECTORY}"] -} - def dockerFilesForExternalServices = [ "elasticsearchWithSearchGuard": "elasticsearch_searchguard", "migrationConsole": "migration_console" @@ -56,36 +32,6 @@ dockerFilesForExternalServices.each { projectName, dockerImageName -> } } -def trafficComparatorServices = [ - "trafficComparator": "traffic_comparator", - "jupyterNotebook": "jupyter_notebook" -] -trafficComparatorServices.forEach {projectName, dockerImageName -> - def dockerBuildDir = "build/docker/${projectName}" - task("copyArtifact_${projectName}", type: Copy) { - dependsOn(tasks.getByName('cloneComparatorRepoIfNeeded')) - from REALIZED_TRAFFIC_COMPARATOR_DIRECTORY - into dockerBuildDir - include '*.py' - include '/traffic_comparator/*' - if (projectName == 'jupyterNotebook') { - include '*.ipynb' - } - } - - task "createDockerfile_${projectName}"(type: com.bmuschko.gradle.docker.tasks.image.Dockerfile) { - dependsOn "copyArtifact_${projectName}" - destFile = project.file("${dockerBuildDir}/Dockerfile") - from 'python:3.10.10' - runCommand("apt-get update && apt-get install -y netcat lsof") - copyFile("setup.py", "/setup.py") - copyFile(".", "/containerTC/") - runCommand("pip3 install --editable \".[data]\"") - // container stay-alive - defaultCommand('tail', '-f', '/dev/null') - } -} - def javaContainerServices = [ "trafficCaptureProxyServer": "capture_proxy", "trafficReplayer": "traffic_replayer" @@ -101,7 +47,7 @@ javaContainerServices.each { projectName, dockerImageName -> CommonUtils.createDockerfile(project, projectName, baseImageProjectOverrides, dockerFilesForExternalServices) } -(javaContainerServices + trafficComparatorServices).forEach { projectName, dockerImageName -> +(javaContainerServices).forEach { projectName, dockerImageName -> def dockerBuildDir = "build/docker/${projectName}" task "buildDockerImage_${projectName}"(type: DockerBuildImage) { dependsOn "createDockerfile_${projectName}" @@ -112,11 +58,6 @@ javaContainerServices.each { projectName, dockerImageName -> } dockerCompose { - String overrideTrafficComparatorDirectory = System.getenv(TRAFFIC_COMPARATOR_DIRECTORY_ENV) - if (overrideTrafficComparatorDirectory == null) { - environment.put(TRAFFIC_COMPARATOR_DIRECTORY_ENV, REALIZED_TRAFFIC_COMPARATOR_DIRECTORY) - exposeAsEnvironment(this) - } useComposeFiles.add("src/main/docker/docker-compose.yml") } @@ -126,10 +67,7 @@ task buildDockerImages { dependsOn buildDockerImage_trafficCaptureProxyServer dependsOn buildDockerImage_trafficReplayer - dependsOn buildDockerImage_trafficComparator - dependsOn buildDockerImage_jupyterNotebook } tasks.getByName('composeUp') .dependsOn(tasks.getByName('buildDockerImages')) - .dependsOn(tasks.getByName('cloneComparatorRepoIfNeeded')) diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index 1d347c5bf..c2ff036a0 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -76,9 +76,7 @@ services: condition: service_started opensearchtarget: condition: service_started - trafficcomparator: - condition: service_healthy - command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46YWRtaW4= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group | nc trafficcomparator 9220" + command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46YWRtaW4= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group" opensearchtarget: image: 'opensearchproject/opensearch:latest' @@ -89,33 +87,6 @@ services: ports: - "29200:9200" - trafficcomparator: - image: 'migrations/traffic_comparator:latest' - networks: - - migrations - ports: - - "9220:9220" - healthcheck: - test: "lsof -i -P -n" - volumes: - - ${TRAFFIC_COMPARATOR_DIRECTORY}:/trafficComparator - - sharedComparatorSqlResults:/shared - command: /bin/sh -c "cd trafficComparator && pip3 install --editable . && nc -v -l -p 9220 | tee /dev/stderr | trafficcomparator -vv stream | trafficcomparator dump-to-sqlite --db /shared/comparisons.db" - - jupyter_notebook: - image: 'migrations/jupyter_notebook:latest' - networks: - - migrations - ports: - - "8888:8888" - volumes: - - ${TRAFFIC_COMPARATOR_DIRECTORY}:/trafficComparator - - sharedComparatorSqlResults:/shared - environment: - # this needs to match the output db that traffic_comparator writes to - - COMPARISONS_DB_LOCATION=/shared/comparisons.db - command: /bin/sh -c 'cd trafficComparator && pip3 install --editable ".[data]" && jupyter notebook --ip=0.0.0.0 --port=8888 --no-browser --allow-root' - migration_console: image: 'migrations/migration_console:latest' networks: @@ -128,8 +99,6 @@ volumes: driver: local kafka_data: driver: local - sharedComparatorSqlResults: - driver: local sharedReplayerOutput: driver: local diff --git a/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPoolTest.java b/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPoolTest.java index f5bd0a74e..eaf43424a 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPoolTest.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPoolTest.java @@ -6,6 +6,7 @@ import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Disabled; import java.time.Duration; import java.time.Instant; @@ -19,6 +20,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +@Disabled @Slf4j class ExpiringSubstitutableItemPoolTest { @@ -144,4 +146,4 @@ private static Integer getIntegerItem(AtomicInteger builtItemCursor, lastCreation.set(Instant.now()); return Integer.valueOf(builtItemCursor.incrementAndGet()); } -} +} \ No newline at end of file diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java index eec73ba03..875c9c90f 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java @@ -26,7 +26,7 @@ * transformation and the headers present (such as for gzipped or chunked encodings). * * There will be a number of reasons that we need to reuse the source captured packets - both today - * (for the output to the comparator) and in the future (for retrying transient network errors or + * (for the output to efs) and in the future (for retrying transient network errors or * transformation errors). With that in mind, the HttpJsonTransformer now keeps track of all of * the ByteBufs passed into it and can redrive them through the underlying network packet handler. * Cases where that would happen with this edit are where the payload wasn't being modified, nor @@ -52,7 +52,7 @@ public class HttpJsonTransformingConsumer implements IPacketFinalizingConsumer> chunkSizes; // This is here for recovery, in case anything goes wrong with a transformation & we want to // just dump it directly. Notice that we're already storing all of the bytes until the response - // comes back so that we can format the output that goes to the comparator. These should be + // comes back so that we can format the output that goes to the to EFS and eventually the comparator. These should be // backed by the exact same byte[] arrays, so the memory consumption should already be absorbed. private final List chunks; diff --git a/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts b/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts index f08189ec8..56b9f7020 100644 --- a/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts @@ -102,18 +102,6 @@ export class MigrationAssistanceStack extends Stack { }); this.mskARN = mskCluster.attrArn - const comparatorSQLiteSG = new SecurityGroup(this, 'comparatorSQLiteSG', { - vpc: props.vpc, - allowAllOutbound: false, - }); - comparatorSQLiteSG.addIngressRule(comparatorSQLiteSG, Port.allTraffic()); - - // Create an EFS file system for the traffic-comparator - const comparatorSQLiteEFS = new FileSystem(this, 'comparatorSQLiteEFS', { - vpc: props.vpc, - securityGroup: comparatorSQLiteSG - }); - const replayerOutputSG = new SecurityGroup(this, 'replayerOutputSG', { vpc: props.vpc, allowAllOutbound: false, @@ -131,8 +119,6 @@ export class MigrationAssistanceStack extends Stack { const exports = [ `export MIGRATION_VPC_ID=${props.vpc.vpcId}`, `export MIGRATION_CAPTURE_MSK_SG_ID=${mskSecurityGroup.securityGroupId}`, - `export MIGRATION_COMPARATOR_EFS_ID=${comparatorSQLiteEFS.fileSystemId}`, - `export MIGRATION_COMPARATOR_EFS_SG_ID=${comparatorSQLiteSG.securityGroupId}`, `export MIGRATION_REPLAYER_OUTPUT_EFS_ID=${replayerOutputEFS.fileSystemId}`, `export MIGRATION_REPLAYER_OUTPUT_EFS_SG_ID=${replayerOutputSG.securityGroupId}`] if (publicSubnetString) exports.push(`export MIGRATION_PUBLIC_SUBNETS=${publicSubnetString}`) diff --git a/deployment/copilot/README.md b/deployment/copilot/README.md index 54fdeaa15..407ec3302 100644 --- a/deployment/copilot/README.md +++ b/deployment/copilot/README.md @@ -8,7 +8,7 @@ Copilot is a tool for deploying containerized applications on AWS ECS. Official ###### Docker Docker is used by Copilot to build container images. If not installed, follow the steps [here](https://docs.docker.com/engine/install/) to set up. Later versions are recommended. ###### Git -Git is used by the opensearch-migrations repo to fetch associated repositories (such as the traffic-comparator repo) for constructing their respective Dockerfiles. Steps to set up can be found [here](https://github.com/git-guides/install-git). +Git is used by the opensearch-migrations repo. Steps to set up can be found [here](https://github.com/git-guides/install-git). ###### Java 11 Java is used by the opensearch-migrations repo and Gradle, its associated build tool. The current required version is Java 11. @@ -78,8 +78,6 @@ export MIGRATION_DOMAIN_USER_NAME=admin export MIGRATION_DOMAIN_USER_SECRET_ARN=arn:aws:secretsmanager:us-east-1:123456789123:secret:demo-user-secret-123abc export MIGRATION_VPC_ID=vpc-123; export MIGRATION_CAPTURE_MSK_SG_ID=sg-123; -export MIGRATION_COMPARATOR_EFS_ID=fs-123; -export MIGRATION_COMPARATOR_EFS_SG_ID=sg-123; export MIGRATION_REPLAYER_OUTPUT_EFS_ID=fs-124 export MIGRATION_REPLAYER_OUTPUT_EFS_SG_ID=sg-124 export MIGRATION_PUBLIC_SUBNETS=subnet-123,subnet-124; @@ -88,7 +86,7 @@ export MIGRATION_KAFKA_BROKER_ENDPOINTS=b-1-public.loggingmskcluster.123.45.kafk ``` Additionally, if not using the deploy script, the following export is needed for the Replayer service: ``` -export MIGRATION_REPLAYER_COMMAND=/bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer $MIGRATION_DOMAIN_ENDPOINT --insecure --kafka-traffic-brokers $MIGRATION_KAFKA_BROKER_ENDPOINTS --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group --kafka-traffic-enable-msk-auth --aws-auth-header-user $MIGRATION_DOMAIN_USER_NAME --aws-auth-header-secret $MIGRATION_DOMAIN_USER_SECRET_ARN | nc traffic-comparator 9220" +export MIGRATION_REPLAYER_COMMAND=/bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer $MIGRATION_DOMAIN_ENDPOINT --insecure --kafka-traffic-brokers $MIGRATION_KAFKA_BROKER_ENDPOINTS --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group --kafka-traffic-enable-msk-auth --aws-auth-header-user $MIGRATION_DOMAIN_USER_NAME --aws-auth-header-secret $MIGRATION_DOMAIN_USER_SECRET_ARN" ``` #### Setting up existing Copilot infrastructure @@ -115,8 +113,6 @@ copilot env init --name dev // Initialize services with their respective required name copilot svc init --name traffic-replayer -copilot svc init --name traffic-comparator -copilot svc init --name traffic-comparator-jupyter copilot svc init --name capture-proxy-es copilot svc init --name migration-console @@ -132,8 +128,6 @@ Currently, it seems that Copilot does not support deploying all services at once copilot env deploy --name dev // Deploy services to a deployed environment -copilot svc deploy --name traffic-comparator-jupyter --env dev -copilot svc deploy --name traffic-comparator --env dev copilot svc deploy --name traffic-replayer --env dev copilot svc deploy --name capture-proxy-es --env dev copilot svc deploy --name migration-console --env dev @@ -164,8 +158,6 @@ curl https://$MIGRATION_DOMAIN_ENDPOINT:443/_cat/indices?v --insecure -u admin:A A command shell can be opened in the service's container if that service has enabled `exec: true` in their `manifest.yml` and the SSM Session Manager plugin is installed when prompted. ``` -copilot svc exec -a migration-copilot -e dev -n traffic-comparator-jupyter -c "bash" -copilot svc exec -a migration-copilot -e dev -n traffic-comparator -c "bash" copilot svc exec -a migration-copilot -e dev -n traffic-replayer -c "bash" copilot svc exec -a migration-copilot -e dev -n elasticsearch -c "bash" copilot svc exec -a migration-copilot -e dev -n capture-proxy -c "bash" diff --git a/deployment/copilot/devDeploy.sh b/deployment/copilot/devDeploy.sh index 49b38dce6..15a2e7285 100755 --- a/deployment/copilot/devDeploy.sh +++ b/deployment/copilot/devDeploy.sh @@ -125,8 +125,6 @@ if [ "$DESTROY_ENV" = true ] ; then set +e # Reset AWS_DEFAULT_REGION as the SDK used by Copilot will first check here for region to use to locate the Copilot app (https://github.com/aws/copilot-cli/issues/5138) export AWS_DEFAULT_REGION="" - copilot svc delete -a $COPILOT_APP_NAME --name traffic-comparator-jupyter --env $COPILOT_DEPLOYMENT_STAGE --yes - copilot svc delete -a $COPILOT_APP_NAME --name traffic-comparator --env $COPILOT_DEPLOYMENT_STAGE --yes copilot svc delete -a $COPILOT_APP_NAME --name traffic-replayer --env $COPILOT_DEPLOYMENT_STAGE --yes copilot svc delete -a $COPILOT_APP_NAME --name capture-proxy-es --env $COPILOT_DEPLOYMENT_STAGE --yes copilot svc delete -a $COPILOT_APP_NAME --name migration-console --env $COPILOT_DEPLOYMENT_STAGE --yes @@ -186,8 +184,7 @@ else echo "No auth header options detected for Replayer, defaulting to not specifying an explicit auth header" fi replay_command_base="/bin/sh -c \"/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://${MIGRATION_DOMAIN_ENDPOINT}:443 --insecure --kafka-traffic-brokers ${MIGRATION_KAFKA_BROKER_ENDPOINTS} --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group --kafka-traffic-enable-msk-auth " -replay_command_end=" | nc traffic-comparator 9220\"" -replay_command="${replay_command_base}${auth_header_args}${replay_command_end}" +replay_command="${replay_command_base}${auth_header_args} echo "Constructed replay command: ${replay_command}" export MIGRATION_REPLAYER_COMMAND="${replay_command}" @@ -208,8 +205,6 @@ if [ "$SKIP_COPILOT_INIT" = false ] ; then #copilot env init -a $COPILOT_APP_NAME --name $COPILOT_DEPLOYMENT_STAGE --default-config --aws-access-key-id $AWS_ACCESS_KEY_ID --aws-secret-access-key $AWS_SECRET_ACCESS_KEY --aws-session-token $AWS_SESSION_TOKEN --region $REGION # Init services - copilot svc init -a $COPILOT_APP_NAME --name traffic-comparator-jupyter - copilot svc init -a $COPILOT_APP_NAME --name traffic-comparator copilot svc init -a $COPILOT_APP_NAME --name traffic-replayer copilot svc init -a $COPILOT_APP_NAME --name capture-proxy-es copilot svc init -a $COPILOT_APP_NAME --name migration-console @@ -220,8 +215,6 @@ fi copilot env deploy -a $COPILOT_APP_NAME --name $COPILOT_DEPLOYMENT_STAGE # Deploy services -copilot svc deploy -a $COPILOT_APP_NAME --name traffic-comparator-jupyter --env $COPILOT_DEPLOYMENT_STAGE -copilot svc deploy -a $COPILOT_APP_NAME --name traffic-comparator --env $COPILOT_DEPLOYMENT_STAGE copilot svc deploy -a $COPILOT_APP_NAME --name traffic-replayer --env $COPILOT_DEPLOYMENT_STAGE copilot svc deploy -a $COPILOT_APP_NAME --name capture-proxy-es --env $COPILOT_DEPLOYMENT_STAGE copilot svc deploy -a $COPILOT_APP_NAME --name migration-console --env $COPILOT_DEPLOYMENT_STAGE diff --git a/deployment/copilot/traffic-comparator-jupyter/manifest.yml b/deployment/copilot/traffic-comparator-jupyter/manifest.yml deleted file mode 100644 index 47e957561..000000000 --- a/deployment/copilot/traffic-comparator-jupyter/manifest.yml +++ /dev/null @@ -1,56 +0,0 @@ -# The manifest for the "traffic-comparator-jupyter" service. -# Read the full specification for the "Load Balanced Web Service" type at: -# https://aws.github.io/copilot-cli/docs/manifest/lb-web-service/ - -# Your service name will be used in naming your resources like log groups, ECS services, etc. -name: traffic-comparator-jupyter -type: Load Balanced Web Service - -# Distribute traffic to your service. -http: - # Requests to this path will be forwarded to your service. - # To match all requests you can use the "/" path. - path: '/' - # Bogus health check to pass startup - healthcheck: - path: '/' - port: 8888 - success_codes: '200-405' - healthy_threshold: 2 - unhealthy_threshold: 2 - interval: 15s - timeout: 10s - grace_period: 60s - -network: - connect: true - vpc: - security_groups: [ "${MIGRATION_COMPARATOR_EFS_SG_ID}" ] - -# Configuration for your containers and service. -image: - # Docker build arguments. For additional overrides: https://aws.github.io/copilot-cli/docs/manifest/backend-service/#image-build - build: - dockerfile: ../TrafficCapture/dockerSolution/build/docker/jupyterNotebook/Dockerfile - port: 8888 - -command: /bin/sh -c 'cd containerTC && pip3 install --editable ".[data]" && jupyter notebook --ip=0.0.0.0 --port=8888 --no-browser --allow-root' - -storage: - volumes: - sharedSQLVolume: # This is a variable key and can be set to an arbitrary string. - path: '/shared' - read_only: false - efs: - id: ${MIGRATION_COMPARATOR_EFS_ID} - -cpu: 512 # Number of CPU units for the task. -memory: 2048 # Amount of memory in MiB used by the task. -count: 1 # Number of tasks that should be running in your service. -exec: true # Enable getting a shell to your container (https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-exec.html). - -environments: - dev: - count: 1 # Number of tasks to run for the "dev" environment. - deployment: # The deployment strategy for the "dev" environment. - rolling: 'recreate' # Stops existing tasks before new ones are started for faster deployments. \ No newline at end of file diff --git a/deployment/copilot/traffic-comparator/manifest.yml b/deployment/copilot/traffic-comparator/manifest.yml deleted file mode 100644 index 10ddf8a80..000000000 --- a/deployment/copilot/traffic-comparator/manifest.yml +++ /dev/null @@ -1,41 +0,0 @@ -# The manifest for the "traffic-comparator" service. -# Read the full specification for the "Backend Service" type at: -# https://aws.github.io/copilot-cli/docs/manifest/backend-service/ - -# Your service name will be used in naming your resources like log groups, ECS services, etc. -name: traffic-comparator -type: Backend Service - -# Allow service-to-service communication with ECS Service Connect -network: - connect: true - vpc: - security_groups: [ "${MIGRATION_COMPARATOR_EFS_SG_ID}" ] - -# Configuration for your containers and service. -image: - # Docker build arguments. For additional overrides: https://aws.github.io/copilot-cli/docs/manifest/backend-service/#image-build - build: - dockerfile: ../TrafficCapture/dockerSolution/build/docker/trafficComparator/Dockerfile - port: 9220 - -command: /bin/sh -c "cd containerTC && pip3 install --editable . && nc -v -l -p 9220 | tee /dev/stderr | trafficcomparator -vv stream | trafficcomparator dump-to-sqlite --db /shared/comparisons.db" - -storage: - volumes: - sharedSQLVolume: # This is a variable key and can be set to an arbitrary string. - path: '/shared' - read_only: false - efs: - id: ${MIGRATION_COMPARATOR_EFS_ID} - -cpu: 512 # Number of CPU units for the task. -memory: 2048 # Amount of memory in MiB used by the task. -count: 1 # Number of tasks that should be running in your service. -exec: true # Enable getting a shell to your container (https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-exec.html). - -environments: - dev: - count: 1 # Number of tasks to run for the "dev" environment. - deployment: # The deployment strategy for the "dev" environment. - rolling: 'recreate' # Stops existing tasks before new ones are started for faster deployments. \ No newline at end of file diff --git a/test/README.md b/test/README.md index 797cbf43f..e7fbe1534 100644 --- a/test/README.md +++ b/test/README.md @@ -25,14 +25,13 @@ The test script, by default, uses the ports assigned to the containers in this [docker-compose file](../TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml), so if the Docker solution in its current setup started with no issues, then the test script will run as is. If for any reason the user changed the ports in that file, they must also either, change the following environment variables: -`PROXY_ENDPOINT`, `SOURCE_ENDPOINT`, `TARGET_ENDPOINT` and `JUPYTER_NOTEBOOK` respectively, or update the default value +`PROXY_ENDPOINT`, `SOURCE_ENDPOINT`, and `TARGET_ENDPOINT` respectively, or update the default value (which can be found below) for them in [tests.py](tests.py). The following are the default values for the only endpoints touched by this script: * `PROXY_ENDPOINT = https://localhost:9200` * `SOURCE_ENDPOINT = http://localhost:19200` * `TARGET_ENDPOINT = https://localhost:29200` -* `JUPYTER_NOTEBOOK = http://localhost:8888/api` #### Clean Up The test script is implemented with a setup and teardown functions that are ran after diff --git a/test/tests.py b/test/tests.py index ce27a9604..fb268963b 100644 --- a/test/tests.py +++ b/test/tests.py @@ -50,7 +50,6 @@ def set_common_values(self): self.proxy_endpoint = os.getenv('PROXY_ENDPOINT', 'https://localhost:9200') self.source_endpoint = os.getenv('SOURCE_ENDPOINT', 'http://localhost:19200') self.target_endpoint = os.getenv('TARGET_ENDPOINT', 'https://localhost:29200') - self.jupyter_endpoint = os.getenv('JUPYTER_NOTEBOOK', 'http://localhost:8888/api') self.username = os.getenv('username', 'admin') self.password = os.getenv('password', 'admin') self.auth = (self.username, self.password) @@ -152,9 +151,4 @@ def test_002_document(self): self.assertEqual(target_response.status_code, HTTPStatus.NOT_FOUND) source_response = retry_request(check_index, args=(self.source_endpoint, self.index, self.auth), expected_status_code=HTTPStatus.NOT_FOUND) - self.assertEqual(source_response.status_code, HTTPStatus.NOT_FOUND) - - def test_003_jupyterAwake(self): - # Making sure that the Jupyter notebook is up and can be reached. - response = requests.get(self.jupyter_endpoint) - self.assertEqual(response.status_code, HTTPStatus.OK) + self.assertEqual(source_response.status_code, HTTPStatus.NOT_FOUND) \ No newline at end of file From 8b37d4983b15325b24d23e42751628c0e7c3dc3a Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Fri, 18 Aug 2023 16:25:54 -0500 Subject: [PATCH 02/13] [capture-and-replay-v0.1.0 7b9d77e] Merge branch 'capture-and-replay-v0.1.0' of github.com:opensearch-project/opensearch-migrations into capture-and-replay-v0.1.0 Signed-off-by: Brian Presley --- TrafficCapture/README.md | 17 +++++ TrafficCapture/dockerSolution/README.md | 2 +- .../migrationConsole/runTestBenchmarks.sh | 65 +++++++++++++++++-- 3 files changed, 78 insertions(+), 6 deletions(-) diff --git a/TrafficCapture/README.md b/TrafficCapture/README.md index f1c569c53..596d81a8e 100644 --- a/TrafficCapture/README.md +++ b/TrafficCapture/README.md @@ -73,6 +73,23 @@ Running check-cluster-health [ Running index-append [100% done] ``` +The `runTestBenchmarks` tool has a few configurable options. It will attempt to guess the correct endpoint to send traffic to, +and it will automatically attach the basic auth user/password `admin`/`admin`. + +To set a custom endpoint, specify it with `--endpoint`, for example `./runTestBenchmarks --endpoint https://capture-proxy-domain.com:9200`. + +To set custom basic auth params, use `--auth_user` and `--auth_pass`. To prevent the script from attaching _any_ auth params, use the `--no_auth` flag. +This flag overrides any other auth params, so if you use both `--auth_user` and `--no_auth`, the end result will be no auth being applied. + +As an example of including multiple options: +```sh +./runTestBenchmarks --endpoint https://capture-proxy-domain.com:9200 --auth_pass Admin123! +``` + +will send requests to `capture-proxy-domain.com`, using the auth combo `admin`/`Admin123!`. + +Support for Sigv4 signing and other auth options may be a future option. + ### Capture Kafka Offloader The Capture Kafka Offloader will act as a Kafka Producer for offloading captured traffic logs to the configured Kafka cluster. diff --git a/TrafficCapture/dockerSolution/README.md b/TrafficCapture/dockerSolution/README.md index 6d2dc1967..38a2ac191 100644 --- a/TrafficCapture/dockerSolution/README.md +++ b/TrafficCapture/dockerSolution/README.md @@ -10,7 +10,7 @@ to ensure that changes will make it into containers to be launched. ### Running the Docker Solution -While in the TrafficCapture directory, run the following command: +While Docker is running, in the TrafficCapture directory run the following command: `./gradlew :dockerSolution:composeUp` diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/runTestBenchmarks.sh b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/runTestBenchmarks.sh index b0db3f72d..1fea2199b 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/runTestBenchmarks.sh +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/runTestBenchmarks.sh @@ -1,10 +1,65 @@ -#!/bin/sh +#!/bin/bash +# Default values +default_docker_endpoint="https://capture_proxy_es:9200" +default_copilot_endpoint="https://capture-proxy-es:443" +auth_user="admin" +auth_pass="admin" +no_auth=false + +# Check for the presence of COPILOT_SERVICE_NAME environment variable +if [ -n "$COPILOT_SERVICE_NAME" ]; then + ENDPOINT="$default_copilot_endpoint" +else + ENDPOINT="$default_docker_endpoint" +fi + +# Override default values with optional command-line arguments +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + --endpoint) + ENDPOINT="$2" + shift + shift + ;; + --auth_user) + auth_user="$2" + shift + shift + ;; + --auth_pass) + auth_pass="$2" + shift + shift + ;; + --no-auth) + no_auth=true + shift + ;; + *) + shift + ;; + esac +done + +# Populate auth string +if [ "$no_auth" = true ]; then + auth_string="" +else + auth_string=",basic_auth_user:${auth_user},basic_auth_password:${auth_pass}" +fi + +# Construct the final client options string +base_options_string="use_ssl:true,verify_certs:false" +client_options="${base_options_string}${auth_string}" + +echo "Running opensearch-benchmark workloads against ${ENDPOINT}" echo "Running opensearch-benchmark w/ 'geonames' workload..." && -opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=https://capture_proxy_es:9200 --workload=geonames --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1" --client-options="use_ssl:true,verify_certs:false,basic_auth_user:admin,basic_auth_password:admin" && +opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=$ENDPOINT --workload=geonames --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1" --client-options=$client_options && echo "Running opensearch-benchmark w/ 'http_logs' workload..." && -opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=https://capture_proxy_es:9200 --workload=http_logs --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1" --client-options="use_ssl:true,verify_certs:false,basic_auth_user:admin,basic_auth_password:admin" && +opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=$ENDPOINT --workload=http_logs --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1" --client-options=$client_options && echo "Running opensearch-benchmark w/ 'nested' workload..." && -opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=https://capture_proxy_es:9200 --workload=nested --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1" --client-options="use_ssl:true,verify_certs:false,basic_auth_user:admin,basic_auth_password:admin" && +opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=$ENDPOINT --workload=nested --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1" --client-options=$client_options && echo "Running opensearch-benchmark w/ 'nyc_taxis' workload..." && -opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=https://capture_proxy_es:9200 --workload=nyc_taxis --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1" --client-options="use_ssl:true,verify_certs:false,basic_auth_user:admin,basic_auth_password:admin" +opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=$ENDPOINT --workload=nyc_taxis --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1" --client-options=$client_options \ No newline at end of file From 51ced23e2f0bf055d64b8f044996e32173897a28 Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Fri, 18 Aug 2023 16:31:44 -0500 Subject: [PATCH 03/13] MIGRATIONS-1200 - Extract comparator and jupyter notebook Signed-off-by: Brian Presley --- TrafficCapture/dockerSolution/README.md | 6 -- TrafficCapture/dockerSolution/build.gradle | 64 +------------------ .../src/main/docker/docker-compose.yml | 33 +--------- .../ExpiringSubstitutableItemPoolTest.java | 4 +- .../http/HttpJsonTransformingConsumer.java | 4 +- .../lib/migration-assistance-stack.ts | 14 ---- deployment/copilot/README.md | 12 +--- deployment/copilot/devDeploy.sh | 9 +-- .../traffic-comparator-jupyter/manifest.yml | 56 ---------------- .../copilot/traffic-comparator/manifest.yml | 41 ------------ test/README.md | 3 +- test/tests.py | 8 +-- 12 files changed, 12 insertions(+), 242 deletions(-) delete mode 100644 deployment/copilot/traffic-comparator-jupyter/manifest.yml delete mode 100644 deployment/copilot/traffic-comparator/manifest.yml diff --git a/TrafficCapture/dockerSolution/README.md b/TrafficCapture/dockerSolution/README.md index 969a3c452..38a2ac191 100644 --- a/TrafficCapture/dockerSolution/README.md +++ b/TrafficCapture/dockerSolution/README.md @@ -8,12 +8,6 @@ down again. Notice that most of the Dockerfiles are dynamically constructed in the build hierarchy. Some efforts have been made to ensure that changes will make it into containers to be launched. -If a user wants to use their own checkout of the traffic-comparator repo, just set the environment variable " -TRAFFIC_COMPARATOR_DIRECTORY" to the directory that contains `setup.py`. Otherwise, if that isn't set, the traffic -comparator repo will be checked out to the build directory and that will be used. Notice that the checkout happens when -the directory wasn't present and there wasn't an environment variable specifying a directory. Once a directory exists, -it will be mounted to the traffic-comparator and jupyter services. - ### Running the Docker Solution While Docker is running, in the TrafficCapture directory run the following command: diff --git a/TrafficCapture/dockerSolution/build.gradle b/TrafficCapture/dockerSolution/build.gradle index ecc2197b4..8a99a168d 100644 --- a/TrafficCapture/dockerSolution/build.gradle +++ b/TrafficCapture/dockerSolution/build.gradle @@ -9,19 +9,6 @@ import java.security.MessageDigest import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage import org.apache.tools.ant.taskdefs.condition.Os -def getTrafficComparatorDirectory() { - String overrideTrafficComparatorDirectory = System.getenv(TRAFFIC_COMPARATOR_DIRECTORY_ENV) - String rval = overrideTrafficComparatorDirectory != null ? - overrideTrafficComparatorDirectory : TRAFFIC_COMPARATOR_REPO_DIRECTORY; - return rval -} - -ext { - TRAFFIC_COMPARATOR_REPO_DIRECTORY = "build/traffic-comparator" - TRAFFIC_COMPARATOR_DIRECTORY_ENV = "TRAFFIC_COMPARATOR_DIRECTORY" - REALIZED_TRAFFIC_COMPARATOR_DIRECTORY = project.file(getTrafficComparatorDirectory()) - } - def calculateDockerHash = { projectName -> CommonUtils.calculateDockerHash(projectName, project) } @@ -31,17 +18,6 @@ dependencies { implementation project(':trafficReplayer') } -task cloneComparatorRepoIfNeeded(type: Exec) { - String comparatorDirectory = project.file(REALIZED_TRAFFIC_COMPARATOR_DIRECTORY); - String repo = 'https://github.com/opensearch-project/traffic-comparator.git' - onlyIf { - !(new File(comparatorDirectory).exists()) - } - commandLine = Os.isFamily(Os.FAMILY_WINDOWS) ? - ['git', 'clone', repo, TRAFFIC_COMPARATOR_REPO_DIRECTORY ] : - ['/bin/sh', '-c', "git clone ${repo} ${TRAFFIC_COMPARATOR_REPO_DIRECTORY}"] -} - def dockerFilesForExternalServices = [ "elasticsearchWithSearchGuard": "elasticsearch_searchguard", "migrationConsole": "migration_console" @@ -56,36 +32,6 @@ dockerFilesForExternalServices.each { projectName, dockerImageName -> } } -def trafficComparatorServices = [ - "trafficComparator": "traffic_comparator", - "jupyterNotebook": "jupyter_notebook" -] -trafficComparatorServices.forEach {projectName, dockerImageName -> - def dockerBuildDir = "build/docker/${projectName}" - task("copyArtifact_${projectName}", type: Copy) { - dependsOn(tasks.getByName('cloneComparatorRepoIfNeeded')) - from REALIZED_TRAFFIC_COMPARATOR_DIRECTORY - into dockerBuildDir - include '*.py' - include '/traffic_comparator/*' - if (projectName == 'jupyterNotebook') { - include '*.ipynb' - } - } - - task "createDockerfile_${projectName}"(type: com.bmuschko.gradle.docker.tasks.image.Dockerfile) { - dependsOn "copyArtifact_${projectName}" - destFile = project.file("${dockerBuildDir}/Dockerfile") - from 'python:3.10.10' - runCommand("apt-get update && apt-get install -y netcat lsof") - copyFile("setup.py", "/setup.py") - copyFile(".", "/containerTC/") - runCommand("pip3 install --editable \".[data]\"") - // container stay-alive - defaultCommand('tail', '-f', '/dev/null') - } -} - def javaContainerServices = [ "trafficCaptureProxyServer": "capture_proxy", "trafficReplayer": "traffic_replayer" @@ -101,7 +47,7 @@ javaContainerServices.each { projectName, dockerImageName -> CommonUtils.createDockerfile(project, projectName, baseImageProjectOverrides, dockerFilesForExternalServices) } -(javaContainerServices + trafficComparatorServices).forEach { projectName, dockerImageName -> +(javaContainerServices).forEach { projectName, dockerImageName -> def dockerBuildDir = "build/docker/${projectName}" task "buildDockerImage_${projectName}"(type: DockerBuildImage) { dependsOn "createDockerfile_${projectName}" @@ -112,11 +58,6 @@ javaContainerServices.each { projectName, dockerImageName -> } dockerCompose { - String overrideTrafficComparatorDirectory = System.getenv(TRAFFIC_COMPARATOR_DIRECTORY_ENV) - if (overrideTrafficComparatorDirectory == null) { - environment.put(TRAFFIC_COMPARATOR_DIRECTORY_ENV, REALIZED_TRAFFIC_COMPARATOR_DIRECTORY) - exposeAsEnvironment(this) - } useComposeFiles.add("src/main/docker/docker-compose.yml") } @@ -126,10 +67,7 @@ task buildDockerImages { dependsOn buildDockerImage_trafficCaptureProxyServer dependsOn buildDockerImage_trafficReplayer - dependsOn buildDockerImage_trafficComparator - dependsOn buildDockerImage_jupyterNotebook } tasks.getByName('composeUp') .dependsOn(tasks.getByName('buildDockerImages')) - .dependsOn(tasks.getByName('cloneComparatorRepoIfNeeded')) diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index 1d347c5bf..c2ff036a0 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -76,9 +76,7 @@ services: condition: service_started opensearchtarget: condition: service_started - trafficcomparator: - condition: service_healthy - command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46YWRtaW4= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group | nc trafficcomparator 9220" + command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46YWRtaW4= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group" opensearchtarget: image: 'opensearchproject/opensearch:latest' @@ -89,33 +87,6 @@ services: ports: - "29200:9200" - trafficcomparator: - image: 'migrations/traffic_comparator:latest' - networks: - - migrations - ports: - - "9220:9220" - healthcheck: - test: "lsof -i -P -n" - volumes: - - ${TRAFFIC_COMPARATOR_DIRECTORY}:/trafficComparator - - sharedComparatorSqlResults:/shared - command: /bin/sh -c "cd trafficComparator && pip3 install --editable . && nc -v -l -p 9220 | tee /dev/stderr | trafficcomparator -vv stream | trafficcomparator dump-to-sqlite --db /shared/comparisons.db" - - jupyter_notebook: - image: 'migrations/jupyter_notebook:latest' - networks: - - migrations - ports: - - "8888:8888" - volumes: - - ${TRAFFIC_COMPARATOR_DIRECTORY}:/trafficComparator - - sharedComparatorSqlResults:/shared - environment: - # this needs to match the output db that traffic_comparator writes to - - COMPARISONS_DB_LOCATION=/shared/comparisons.db - command: /bin/sh -c 'cd trafficComparator && pip3 install --editable ".[data]" && jupyter notebook --ip=0.0.0.0 --port=8888 --no-browser --allow-root' - migration_console: image: 'migrations/migration_console:latest' networks: @@ -128,8 +99,6 @@ volumes: driver: local kafka_data: driver: local - sharedComparatorSqlResults: - driver: local sharedReplayerOutput: driver: local diff --git a/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPoolTest.java b/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPoolTest.java index f5bd0a74e..eaf43424a 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPoolTest.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPoolTest.java @@ -6,6 +6,7 @@ import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Disabled; import java.time.Duration; import java.time.Instant; @@ -19,6 +20,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +@Disabled @Slf4j class ExpiringSubstitutableItemPoolTest { @@ -144,4 +146,4 @@ private static Integer getIntegerItem(AtomicInteger builtItemCursor, lastCreation.set(Instant.now()); return Integer.valueOf(builtItemCursor.incrementAndGet()); } -} +} \ No newline at end of file diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java index eec73ba03..875c9c90f 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java @@ -26,7 +26,7 @@ * transformation and the headers present (such as for gzipped or chunked encodings). * * There will be a number of reasons that we need to reuse the source captured packets - both today - * (for the output to the comparator) and in the future (for retrying transient network errors or + * (for the output to efs) and in the future (for retrying transient network errors or * transformation errors). With that in mind, the HttpJsonTransformer now keeps track of all of * the ByteBufs passed into it and can redrive them through the underlying network packet handler. * Cases where that would happen with this edit are where the payload wasn't being modified, nor @@ -52,7 +52,7 @@ public class HttpJsonTransformingConsumer implements IPacketFinalizingConsumer> chunkSizes; // This is here for recovery, in case anything goes wrong with a transformation & we want to // just dump it directly. Notice that we're already storing all of the bytes until the response - // comes back so that we can format the output that goes to the comparator. These should be + // comes back so that we can format the output that goes to the to EFS and eventually the comparator. These should be // backed by the exact same byte[] arrays, so the memory consumption should already be absorbed. private final List chunks; diff --git a/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts b/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts index f08189ec8..56b9f7020 100644 --- a/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts @@ -102,18 +102,6 @@ export class MigrationAssistanceStack extends Stack { }); this.mskARN = mskCluster.attrArn - const comparatorSQLiteSG = new SecurityGroup(this, 'comparatorSQLiteSG', { - vpc: props.vpc, - allowAllOutbound: false, - }); - comparatorSQLiteSG.addIngressRule(comparatorSQLiteSG, Port.allTraffic()); - - // Create an EFS file system for the traffic-comparator - const comparatorSQLiteEFS = new FileSystem(this, 'comparatorSQLiteEFS', { - vpc: props.vpc, - securityGroup: comparatorSQLiteSG - }); - const replayerOutputSG = new SecurityGroup(this, 'replayerOutputSG', { vpc: props.vpc, allowAllOutbound: false, @@ -131,8 +119,6 @@ export class MigrationAssistanceStack extends Stack { const exports = [ `export MIGRATION_VPC_ID=${props.vpc.vpcId}`, `export MIGRATION_CAPTURE_MSK_SG_ID=${mskSecurityGroup.securityGroupId}`, - `export MIGRATION_COMPARATOR_EFS_ID=${comparatorSQLiteEFS.fileSystemId}`, - `export MIGRATION_COMPARATOR_EFS_SG_ID=${comparatorSQLiteSG.securityGroupId}`, `export MIGRATION_REPLAYER_OUTPUT_EFS_ID=${replayerOutputEFS.fileSystemId}`, `export MIGRATION_REPLAYER_OUTPUT_EFS_SG_ID=${replayerOutputSG.securityGroupId}`] if (publicSubnetString) exports.push(`export MIGRATION_PUBLIC_SUBNETS=${publicSubnetString}`) diff --git a/deployment/copilot/README.md b/deployment/copilot/README.md index 54fdeaa15..407ec3302 100644 --- a/deployment/copilot/README.md +++ b/deployment/copilot/README.md @@ -8,7 +8,7 @@ Copilot is a tool for deploying containerized applications on AWS ECS. Official ###### Docker Docker is used by Copilot to build container images. If not installed, follow the steps [here](https://docs.docker.com/engine/install/) to set up. Later versions are recommended. ###### Git -Git is used by the opensearch-migrations repo to fetch associated repositories (such as the traffic-comparator repo) for constructing their respective Dockerfiles. Steps to set up can be found [here](https://github.com/git-guides/install-git). +Git is used by the opensearch-migrations repo. Steps to set up can be found [here](https://github.com/git-guides/install-git). ###### Java 11 Java is used by the opensearch-migrations repo and Gradle, its associated build tool. The current required version is Java 11. @@ -78,8 +78,6 @@ export MIGRATION_DOMAIN_USER_NAME=admin export MIGRATION_DOMAIN_USER_SECRET_ARN=arn:aws:secretsmanager:us-east-1:123456789123:secret:demo-user-secret-123abc export MIGRATION_VPC_ID=vpc-123; export MIGRATION_CAPTURE_MSK_SG_ID=sg-123; -export MIGRATION_COMPARATOR_EFS_ID=fs-123; -export MIGRATION_COMPARATOR_EFS_SG_ID=sg-123; export MIGRATION_REPLAYER_OUTPUT_EFS_ID=fs-124 export MIGRATION_REPLAYER_OUTPUT_EFS_SG_ID=sg-124 export MIGRATION_PUBLIC_SUBNETS=subnet-123,subnet-124; @@ -88,7 +86,7 @@ export MIGRATION_KAFKA_BROKER_ENDPOINTS=b-1-public.loggingmskcluster.123.45.kafk ``` Additionally, if not using the deploy script, the following export is needed for the Replayer service: ``` -export MIGRATION_REPLAYER_COMMAND=/bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer $MIGRATION_DOMAIN_ENDPOINT --insecure --kafka-traffic-brokers $MIGRATION_KAFKA_BROKER_ENDPOINTS --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group --kafka-traffic-enable-msk-auth --aws-auth-header-user $MIGRATION_DOMAIN_USER_NAME --aws-auth-header-secret $MIGRATION_DOMAIN_USER_SECRET_ARN | nc traffic-comparator 9220" +export MIGRATION_REPLAYER_COMMAND=/bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer $MIGRATION_DOMAIN_ENDPOINT --insecure --kafka-traffic-brokers $MIGRATION_KAFKA_BROKER_ENDPOINTS --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group --kafka-traffic-enable-msk-auth --aws-auth-header-user $MIGRATION_DOMAIN_USER_NAME --aws-auth-header-secret $MIGRATION_DOMAIN_USER_SECRET_ARN" ``` #### Setting up existing Copilot infrastructure @@ -115,8 +113,6 @@ copilot env init --name dev // Initialize services with their respective required name copilot svc init --name traffic-replayer -copilot svc init --name traffic-comparator -copilot svc init --name traffic-comparator-jupyter copilot svc init --name capture-proxy-es copilot svc init --name migration-console @@ -132,8 +128,6 @@ Currently, it seems that Copilot does not support deploying all services at once copilot env deploy --name dev // Deploy services to a deployed environment -copilot svc deploy --name traffic-comparator-jupyter --env dev -copilot svc deploy --name traffic-comparator --env dev copilot svc deploy --name traffic-replayer --env dev copilot svc deploy --name capture-proxy-es --env dev copilot svc deploy --name migration-console --env dev @@ -164,8 +158,6 @@ curl https://$MIGRATION_DOMAIN_ENDPOINT:443/_cat/indices?v --insecure -u admin:A A command shell can be opened in the service's container if that service has enabled `exec: true` in their `manifest.yml` and the SSM Session Manager plugin is installed when prompted. ``` -copilot svc exec -a migration-copilot -e dev -n traffic-comparator-jupyter -c "bash" -copilot svc exec -a migration-copilot -e dev -n traffic-comparator -c "bash" copilot svc exec -a migration-copilot -e dev -n traffic-replayer -c "bash" copilot svc exec -a migration-copilot -e dev -n elasticsearch -c "bash" copilot svc exec -a migration-copilot -e dev -n capture-proxy -c "bash" diff --git a/deployment/copilot/devDeploy.sh b/deployment/copilot/devDeploy.sh index 49b38dce6..15a2e7285 100755 --- a/deployment/copilot/devDeploy.sh +++ b/deployment/copilot/devDeploy.sh @@ -125,8 +125,6 @@ if [ "$DESTROY_ENV" = true ] ; then set +e # Reset AWS_DEFAULT_REGION as the SDK used by Copilot will first check here for region to use to locate the Copilot app (https://github.com/aws/copilot-cli/issues/5138) export AWS_DEFAULT_REGION="" - copilot svc delete -a $COPILOT_APP_NAME --name traffic-comparator-jupyter --env $COPILOT_DEPLOYMENT_STAGE --yes - copilot svc delete -a $COPILOT_APP_NAME --name traffic-comparator --env $COPILOT_DEPLOYMENT_STAGE --yes copilot svc delete -a $COPILOT_APP_NAME --name traffic-replayer --env $COPILOT_DEPLOYMENT_STAGE --yes copilot svc delete -a $COPILOT_APP_NAME --name capture-proxy-es --env $COPILOT_DEPLOYMENT_STAGE --yes copilot svc delete -a $COPILOT_APP_NAME --name migration-console --env $COPILOT_DEPLOYMENT_STAGE --yes @@ -186,8 +184,7 @@ else echo "No auth header options detected for Replayer, defaulting to not specifying an explicit auth header" fi replay_command_base="/bin/sh -c \"/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://${MIGRATION_DOMAIN_ENDPOINT}:443 --insecure --kafka-traffic-brokers ${MIGRATION_KAFKA_BROKER_ENDPOINTS} --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group --kafka-traffic-enable-msk-auth " -replay_command_end=" | nc traffic-comparator 9220\"" -replay_command="${replay_command_base}${auth_header_args}${replay_command_end}" +replay_command="${replay_command_base}${auth_header_args} echo "Constructed replay command: ${replay_command}" export MIGRATION_REPLAYER_COMMAND="${replay_command}" @@ -208,8 +205,6 @@ if [ "$SKIP_COPILOT_INIT" = false ] ; then #copilot env init -a $COPILOT_APP_NAME --name $COPILOT_DEPLOYMENT_STAGE --default-config --aws-access-key-id $AWS_ACCESS_KEY_ID --aws-secret-access-key $AWS_SECRET_ACCESS_KEY --aws-session-token $AWS_SESSION_TOKEN --region $REGION # Init services - copilot svc init -a $COPILOT_APP_NAME --name traffic-comparator-jupyter - copilot svc init -a $COPILOT_APP_NAME --name traffic-comparator copilot svc init -a $COPILOT_APP_NAME --name traffic-replayer copilot svc init -a $COPILOT_APP_NAME --name capture-proxy-es copilot svc init -a $COPILOT_APP_NAME --name migration-console @@ -220,8 +215,6 @@ fi copilot env deploy -a $COPILOT_APP_NAME --name $COPILOT_DEPLOYMENT_STAGE # Deploy services -copilot svc deploy -a $COPILOT_APP_NAME --name traffic-comparator-jupyter --env $COPILOT_DEPLOYMENT_STAGE -copilot svc deploy -a $COPILOT_APP_NAME --name traffic-comparator --env $COPILOT_DEPLOYMENT_STAGE copilot svc deploy -a $COPILOT_APP_NAME --name traffic-replayer --env $COPILOT_DEPLOYMENT_STAGE copilot svc deploy -a $COPILOT_APP_NAME --name capture-proxy-es --env $COPILOT_DEPLOYMENT_STAGE copilot svc deploy -a $COPILOT_APP_NAME --name migration-console --env $COPILOT_DEPLOYMENT_STAGE diff --git a/deployment/copilot/traffic-comparator-jupyter/manifest.yml b/deployment/copilot/traffic-comparator-jupyter/manifest.yml deleted file mode 100644 index 47e957561..000000000 --- a/deployment/copilot/traffic-comparator-jupyter/manifest.yml +++ /dev/null @@ -1,56 +0,0 @@ -# The manifest for the "traffic-comparator-jupyter" service. -# Read the full specification for the "Load Balanced Web Service" type at: -# https://aws.github.io/copilot-cli/docs/manifest/lb-web-service/ - -# Your service name will be used in naming your resources like log groups, ECS services, etc. -name: traffic-comparator-jupyter -type: Load Balanced Web Service - -# Distribute traffic to your service. -http: - # Requests to this path will be forwarded to your service. - # To match all requests you can use the "/" path. - path: '/' - # Bogus health check to pass startup - healthcheck: - path: '/' - port: 8888 - success_codes: '200-405' - healthy_threshold: 2 - unhealthy_threshold: 2 - interval: 15s - timeout: 10s - grace_period: 60s - -network: - connect: true - vpc: - security_groups: [ "${MIGRATION_COMPARATOR_EFS_SG_ID}" ] - -# Configuration for your containers and service. -image: - # Docker build arguments. For additional overrides: https://aws.github.io/copilot-cli/docs/manifest/backend-service/#image-build - build: - dockerfile: ../TrafficCapture/dockerSolution/build/docker/jupyterNotebook/Dockerfile - port: 8888 - -command: /bin/sh -c 'cd containerTC && pip3 install --editable ".[data]" && jupyter notebook --ip=0.0.0.0 --port=8888 --no-browser --allow-root' - -storage: - volumes: - sharedSQLVolume: # This is a variable key and can be set to an arbitrary string. - path: '/shared' - read_only: false - efs: - id: ${MIGRATION_COMPARATOR_EFS_ID} - -cpu: 512 # Number of CPU units for the task. -memory: 2048 # Amount of memory in MiB used by the task. -count: 1 # Number of tasks that should be running in your service. -exec: true # Enable getting a shell to your container (https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-exec.html). - -environments: - dev: - count: 1 # Number of tasks to run for the "dev" environment. - deployment: # The deployment strategy for the "dev" environment. - rolling: 'recreate' # Stops existing tasks before new ones are started for faster deployments. \ No newline at end of file diff --git a/deployment/copilot/traffic-comparator/manifest.yml b/deployment/copilot/traffic-comparator/manifest.yml deleted file mode 100644 index 10ddf8a80..000000000 --- a/deployment/copilot/traffic-comparator/manifest.yml +++ /dev/null @@ -1,41 +0,0 @@ -# The manifest for the "traffic-comparator" service. -# Read the full specification for the "Backend Service" type at: -# https://aws.github.io/copilot-cli/docs/manifest/backend-service/ - -# Your service name will be used in naming your resources like log groups, ECS services, etc. -name: traffic-comparator -type: Backend Service - -# Allow service-to-service communication with ECS Service Connect -network: - connect: true - vpc: - security_groups: [ "${MIGRATION_COMPARATOR_EFS_SG_ID}" ] - -# Configuration for your containers and service. -image: - # Docker build arguments. For additional overrides: https://aws.github.io/copilot-cli/docs/manifest/backend-service/#image-build - build: - dockerfile: ../TrafficCapture/dockerSolution/build/docker/trafficComparator/Dockerfile - port: 9220 - -command: /bin/sh -c "cd containerTC && pip3 install --editable . && nc -v -l -p 9220 | tee /dev/stderr | trafficcomparator -vv stream | trafficcomparator dump-to-sqlite --db /shared/comparisons.db" - -storage: - volumes: - sharedSQLVolume: # This is a variable key and can be set to an arbitrary string. - path: '/shared' - read_only: false - efs: - id: ${MIGRATION_COMPARATOR_EFS_ID} - -cpu: 512 # Number of CPU units for the task. -memory: 2048 # Amount of memory in MiB used by the task. -count: 1 # Number of tasks that should be running in your service. -exec: true # Enable getting a shell to your container (https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-exec.html). - -environments: - dev: - count: 1 # Number of tasks to run for the "dev" environment. - deployment: # The deployment strategy for the "dev" environment. - rolling: 'recreate' # Stops existing tasks before new ones are started for faster deployments. \ No newline at end of file diff --git a/test/README.md b/test/README.md index 797cbf43f..e7fbe1534 100644 --- a/test/README.md +++ b/test/README.md @@ -25,14 +25,13 @@ The test script, by default, uses the ports assigned to the containers in this [docker-compose file](../TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml), so if the Docker solution in its current setup started with no issues, then the test script will run as is. If for any reason the user changed the ports in that file, they must also either, change the following environment variables: -`PROXY_ENDPOINT`, `SOURCE_ENDPOINT`, `TARGET_ENDPOINT` and `JUPYTER_NOTEBOOK` respectively, or update the default value +`PROXY_ENDPOINT`, `SOURCE_ENDPOINT`, and `TARGET_ENDPOINT` respectively, or update the default value (which can be found below) for them in [tests.py](tests.py). The following are the default values for the only endpoints touched by this script: * `PROXY_ENDPOINT = https://localhost:9200` * `SOURCE_ENDPOINT = http://localhost:19200` * `TARGET_ENDPOINT = https://localhost:29200` -* `JUPYTER_NOTEBOOK = http://localhost:8888/api` #### Clean Up The test script is implemented with a setup and teardown functions that are ran after diff --git a/test/tests.py b/test/tests.py index ce27a9604..fb268963b 100644 --- a/test/tests.py +++ b/test/tests.py @@ -50,7 +50,6 @@ def set_common_values(self): self.proxy_endpoint = os.getenv('PROXY_ENDPOINT', 'https://localhost:9200') self.source_endpoint = os.getenv('SOURCE_ENDPOINT', 'http://localhost:19200') self.target_endpoint = os.getenv('TARGET_ENDPOINT', 'https://localhost:29200') - self.jupyter_endpoint = os.getenv('JUPYTER_NOTEBOOK', 'http://localhost:8888/api') self.username = os.getenv('username', 'admin') self.password = os.getenv('password', 'admin') self.auth = (self.username, self.password) @@ -152,9 +151,4 @@ def test_002_document(self): self.assertEqual(target_response.status_code, HTTPStatus.NOT_FOUND) source_response = retry_request(check_index, args=(self.source_endpoint, self.index, self.auth), expected_status_code=HTTPStatus.NOT_FOUND) - self.assertEqual(source_response.status_code, HTTPStatus.NOT_FOUND) - - def test_003_jupyterAwake(self): - # Making sure that the Jupyter notebook is up and can be reached. - response = requests.get(self.jupyter_endpoint) - self.assertEqual(response.status_code, HTTPStatus.OK) + self.assertEqual(source_response.status_code, HTTPStatus.NOT_FOUND) \ No newline at end of file From f994047e3d74a90e7e040f202dc4e8e011f12d1f Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Fri, 18 Aug 2023 16:36:26 -0500 Subject: [PATCH 04/13] MIGRATION-1200 - Removed historical capture stack Signed-off-by: Brian Presley --- .../lib/historical-capture-stack.ts | 62 ------------------- 1 file changed, 62 deletions(-) delete mode 100644 deployment/cdk/opensearch-service-migration/lib/historical-capture-stack.ts diff --git a/deployment/cdk/opensearch-service-migration/lib/historical-capture-stack.ts b/deployment/cdk/opensearch-service-migration/lib/historical-capture-stack.ts deleted file mode 100644 index 93f1408a4..000000000 --- a/deployment/cdk/opensearch-service-migration/lib/historical-capture-stack.ts +++ /dev/null @@ -1,62 +0,0 @@ -import {Stack, StackProps} from "aws-cdk-lib"; -import {IVpc} from "aws-cdk-lib/aws-ec2"; -import {Construct} from "constructs"; -import {Cluster, ContainerImage, FargateService, FargateTaskDefinition, LogDrivers} from "aws-cdk-lib/aws-ecs"; -import {DockerImageAsset} from "aws-cdk-lib/aws-ecr-assets"; -import {join} from "path"; -import { readFileSync } from "fs" - -export interface historicalCaptureStackProps extends StackProps { - readonly vpc: IVpc, - readonly logstashConfigFilePath: string, - readonly sourceEndpoint?: string, - readonly targetEndpoint: string -} - -/** - * This stack was a short exploratory task into having a deployable Logstash ECS cluster for historical data migration. - * NOTE: It should only be used for development purposes in its current state - */ -export class HistoricalCaptureStack extends Stack { - - constructor(scope: Construct, id: string, props: historicalCaptureStackProps) { - super(scope, id, props); - - const ecsCluster = new Cluster(this, "ecsHistoricalCaptureCluster", { - vpc: props.vpc - }); - - const historicalCaptureFargateTask = new FargateTaskDefinition(this, "historicalCaptureFargateTask", { - memoryLimitMiB: 2048, - cpu: 512 - }); - - let logstashConfigData: string = readFileSync(props.logstashConfigFilePath, 'utf8'); - if (props.sourceEndpoint) { - logstashConfigData = logstashConfigData.replace("", props.sourceEndpoint) - } - logstashConfigData = logstashConfigData.replace("", props.targetEndpoint + ":80") - // Temporary measure to allow multi-line env variable - logstashConfigData = logstashConfigData.replace(/(\n)/g, "PUT_LINE") - // Create Historical Capture Container - const historicalCaptureImage = new DockerImageAsset(this, "historicalCaptureImage", { - directory: join(__dirname, "../../..", "docker/logstash-setup") - }); - - const historicalCaptureContainer = historicalCaptureFargateTask.addContainer("historicalCaptureContainer", { - image: ContainerImage.fromDockerImageAsset(historicalCaptureImage), - // Add in region and stage - containerName: "logstash", - environment: {"LOGSTASH_CONFIG": '' + logstashConfigData}, - logging: LogDrivers.awsLogs({ streamPrefix: 'logstash-lg', logRetention: 30 }) - }); - - // Create Fargate Service - const historicalCaptureFargateService = new FargateService(this, "historicalCaptureFargateService", { - cluster: ecsCluster, - taskDefinition: historicalCaptureFargateTask, - desiredCount: 1 - }); - - } -} \ No newline at end of file From 252c055edbb702eaf8d042562867bc92901a7467 Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Fri, 18 Aug 2023 17:02:50 -0500 Subject: [PATCH 05/13] Remove unused import from python test Signed-off-by: Brian Presley --- test/tests.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/tests.py b/test/tests.py index fb268963b..5bffe3fed 100644 --- a/test/tests.py +++ b/test/tests.py @@ -6,8 +6,6 @@ import os import logging import time -import requests -from requests.exceptions import ConnectionError, SSLError logger = logging.getLogger(__name__) From a66784e1b27a1ec3487efcf1f3c59b8c4b699c3c Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Fri, 18 Aug 2023 17:03:46 -0500 Subject: [PATCH 06/13] Fix devDeploy unbalanced quotes Signed-off-by: Brian Presley --- deployment/copilot/devDeploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/copilot/devDeploy.sh b/deployment/copilot/devDeploy.sh index 15a2e7285..44cb1f955 100755 --- a/deployment/copilot/devDeploy.sh +++ b/deployment/copilot/devDeploy.sh @@ -184,7 +184,7 @@ else echo "No auth header options detected for Replayer, defaulting to not specifying an explicit auth header" fi replay_command_base="/bin/sh -c \"/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://${MIGRATION_DOMAIN_ENDPOINT}:443 --insecure --kafka-traffic-brokers ${MIGRATION_KAFKA_BROKER_ENDPOINTS} --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group --kafka-traffic-enable-msk-auth " -replay_command="${replay_command_base}${auth_header_args} +replay_command="${replay_command_base}${auth_header_args}" echo "Constructed replay command: ${replay_command}" export MIGRATION_REPLAYER_COMMAND="${replay_command}" From f6c225c3b7eebf28f0500476c652ef0028d489d8 Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Sun, 20 Aug 2023 13:14:31 -0500 Subject: [PATCH 07/13] Fix EFS verbiage Signed-off-by: Brian Presley --- .../datahandlers/http/HttpJsonTransformingConsumer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java index 875c9c90f..90e537aaa 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java @@ -26,7 +26,7 @@ * transformation and the headers present (such as for gzipped or chunked encodings). * * There will be a number of reasons that we need to reuse the source captured packets - both today - * (for the output to efs) and in the future (for retrying transient network errors or + * (for the output to Amazon Elastic File System (EFS) and in the future (for retrying transient network errors or * transformation errors). With that in mind, the HttpJsonTransformer now keeps track of all of * the ByteBufs passed into it and can redrive them through the underlying network packet handler. * Cases where that would happen with this edit are where the payload wasn't being modified, nor @@ -52,7 +52,7 @@ public class HttpJsonTransformingConsumer implements IPacketFinalizingConsumer> chunkSizes; // This is here for recovery, in case anything goes wrong with a transformation & we want to // just dump it directly. Notice that we're already storing all of the bytes until the response - // comes back so that we can format the output that goes to the to EFS and eventually the comparator. These should be + // comes back so that we can format the output that goes to EFS and eventually the comparator. These should be // backed by the exact same byte[] arrays, so the memory consumption should already be absorbed. private final List chunks; From 714db224c571d3cb3e9aa2e2bc58617f11ff89e1 Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Sun, 20 Aug 2023 21:26:44 -0500 Subject: [PATCH 08/13] Fix tests import error Signed-off-by: Brian Presley --- test/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/tests.py b/test/tests.py index 5bffe3fed..582a63b51 100644 --- a/test/tests.py +++ b/test/tests.py @@ -6,6 +6,7 @@ import os import logging import time +from requests.exceptions import ConnectionError, SSLError logger = logging.getLogger(__name__) From b16dc1bf00f8b91a491e70d94e3025786ff1d2c9 Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Sun, 20 Aug 2023 21:57:19 -0500 Subject: [PATCH 09/13] Remove reference to historical stack from CDK Signed-off-by: Brian Presley --- .../cdk/opensearch-service-migration/lib/stack-composer.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts index cb49bcf44..b8a29b652 100644 --- a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts +++ b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts @@ -7,7 +7,6 @@ import {AnyPrincipal, Effect, PolicyStatement} from "aws-cdk-lib/aws-iam"; import * as defaultValuesJson from "../default-values.json" import {NetworkStack} from "./network-stack"; import {MigrationAssistanceStack} from "./migration-assistance-stack"; -import {HistoricalCaptureStack} from "./historical-capture-stack"; import {MSKUtilityStack} from "./msk-utility-stack"; export interface StackPropsExt extends StackProps { From 36afe902e31613f6e5f8e02cdb582b9d87f77ee1 Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Sun, 20 Aug 2023 22:16:07 -0500 Subject: [PATCH 10/13] Remove references to historical capture in CDK Signed-off-by: Brian Presley --- .../lib/stack-composer.ts | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts index b8a29b652..cebe7870b 100644 --- a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts +++ b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts @@ -186,25 +186,6 @@ export class StackComposer { this.stacks.push(mskUtilityStack) } - // Currently, placing a requirement on a VPC for a historical capture stack but this can be revisited - // Note: Future work to provide orchestration between historical capture and migration assistance as the current - // state will potentially have both stacks trying to add the same data - if (historicalCaptureEnabled && networkStack) { - const historicalCaptureStack = new HistoricalCaptureStack(scope, "historicalCaptureStack", { - vpc: networkStack.vpc, - logstashConfigFilePath: logstashConfigFilePath, - sourceEndpoint: sourceClusterEndpoint, - targetEndpoint: opensearchStack.domainEndpoint, - stackName: `OSServiceHistoricalCDKStack-${stage}-${region}`, - description: "This stack contains resources to assist migrating historical data to an OpenSearch Service domain", - ...props, - }) - - historicalCaptureStack.addDependency(opensearchStack) - this.stacks.push(historicalCaptureStack) - } - - function getContextForType(optionName: string, expectedType: string): any { const option = scope.node.tryGetContext(optionName) From e19e9f3bfcc764a4b8448e221c673ef873f5aebb Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Sun, 20 Aug 2023 22:37:11 -0500 Subject: [PATCH 11/13] Redirect Replayer Docker output to /dev/null Signed-off-by: Brian Presley --- .../dockerSolution/src/main/docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index c2ff036a0..aa28d2d16 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -76,7 +76,7 @@ services: condition: service_started opensearchtarget: condition: service_started - command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46YWRtaW4= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group" + command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46YWRtaW4= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group >/dev/null" opensearchtarget: image: 'opensearchproject/opensearch:latest' From 37dccd5087f86f14622797dfec0f9a7bdae02257 Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Sun, 20 Aug 2023 22:40:39 -0500 Subject: [PATCH 12/13] Remove references to EFS in documentation Signed-off-by: Brian Presley --- .../datahandlers/http/HttpJsonTransformingConsumer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java index 90e537aaa..cdd193ff2 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java @@ -26,7 +26,7 @@ * transformation and the headers present (such as for gzipped or chunked encodings). * * There will be a number of reasons that we need to reuse the source captured packets - both today - * (for the output to Amazon Elastic File System (EFS) and in the future (for retrying transient network errors or + * (for the output and in the future (for retrying transient network errors or * transformation errors). With that in mind, the HttpJsonTransformer now keeps track of all of * the ByteBufs passed into it and can redrive them through the underlying network packet handler. * Cases where that would happen with this edit are where the payload wasn't being modified, nor @@ -52,7 +52,7 @@ public class HttpJsonTransformingConsumer implements IPacketFinalizingConsumer> chunkSizes; // This is here for recovery, in case anything goes wrong with a transformation & we want to // just dump it directly. Notice that we're already storing all of the bytes until the response - // comes back so that we can format the output that goes to EFS and eventually the comparator. These should be + // comes back so that we can format the output that goes downstream. These should be // backed by the exact same byte[] arrays, so the memory consumption should already be absorbed. private final List chunks; From f60b0c47557c64f3a2d7db482b69778e9d7eb19a Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Sun, 20 Aug 2023 23:00:47 -0500 Subject: [PATCH 13/13] Add new line at the end of python module to fix linting error Signed-off-by: Brian Presley --- test/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tests.py b/test/tests.py index 582a63b51..fc94dc72e 100644 --- a/test/tests.py +++ b/test/tests.py @@ -150,4 +150,4 @@ def test_002_document(self): self.assertEqual(target_response.status_code, HTTPStatus.NOT_FOUND) source_response = retry_request(check_index, args=(self.source_endpoint, self.index, self.auth), expected_status_code=HTTPStatus.NOT_FOUND) - self.assertEqual(source_response.status_code, HTTPStatus.NOT_FOUND) \ No newline at end of file + self.assertEqual(source_response.status_code, HTTPStatus.NOT_FOUND)