Merge branch 'main' into NewBackpressureForRequestsOutstanding

* main:
  On board 1-click release for OpenSearch Migrations (opensearch-project#334)
  Add support for Kafka management scripts (opensearch-project#330)
  Bump urllib3 from 2.0.3 to 2.0.6 in /test
  Removing extra character + moving project specific requirement back to where it belongs
  Update CDK documentation + minor changes to devDeploy script

Signed-off-by: Greg Schohn <[email protected]>

.github/workflows/release-drafter.yml

@@ -0,0 +1,35 @@
+name: Release drafter
+
+# Push events to every tag not containing "/"
+on:
+  push:
+    tags:
+      - "*"
+
+jobs:
+  draft-a-release:
+    name: Draft a release
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - id: get_data
+        run: |
+          echo "approvers=$(cat .github/CODEOWNERS | grep @ | tr -d '*\n ' | sed 's/@/,/g' | sed 's/,//1')" >> $GITHUB_OUTPUT
+          echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+      - uses: trstringer/manual-approval@v1
+        with:
+          secret: ${{ github.TOKEN }}
+          approvers: ${{ steps.get_data.outputs.approvers }}
+          minimum-approvals: 2
+          issue-title: 'Release opensearch-migrations version ${{ steps.get_data.outputs.version }}'
+          issue-body: "Please approve or deny the release of opensearch-migrations **TAG**: ${{ github.ref_name }}  **COMMIT**: ${{ github.sha }} **VERSION** : ${{ steps.get_data.outputs.version }} "
+          exclude-workflow-initiator-as-approver: true
+      - name: Download Repo Tar
+        run: |
+          wget https://github.com/opensearch-project/opensearch-migrations/archive/refs/tags/${{ steps.get_data.outputs.version }}.tar.gz -O artifacts.tar.gz
+      - name: Draft a release
+        uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          generate_release_notes: true
+          files: artifacts.tar.gz

README.md

@@ -60,3 +60,13 @@ See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more inform
 ## License
 
 This project is licensed under the Apache-2.0 License.
+
+
+## Releasing
+
+The release process is standard across repositories in this org and is run by a release manager volunteering from amongst [maintainers](MAINTAINERS.md).
+
+1. Create a tag, e.g. 0.1.0, and push it to this GitHub repository.
+2. The [release-drafter.yml](.github/workflows/release-drafter.yml) will be automatically kicked off and a draft release will be created.
+3. This draft release triggers the [jenkins release workflow](https://build.ci.opensearch.org/job/opensearch-migrations-release) as a result of which the opensearch-migrations toolset is released and published on artifacts.opensearch.org example as https://artifacts.opensearch.org/migrations/0.1.0/opensearch-migrations-0.1.0.tar.gz. 
+4. Once the above release workflow is successful, the drafted release on GitHub is published automatically.

TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml

@@ -124,6 +124,8 @@ services:
       - migrations
     volumes:
       - sharedReplayerOutput:/shared-replayer-output
+    environment:
+      - MIGRATION_KAFKA_BROKER_ENDPOINTS=kafka:9092
 
 volumes:
   zookeeper_data:

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile

@@ -3,15 +3,22 @@ FROM ubuntu:jammy
 ENV DEBIAN_FRONTEND noninteractive
 
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends python3.9 python3-pip python3-dev gcc libc-dev git curl vim jq && \
+    apt-get install -y --no-install-recommends python3.9 python3-pip python3-dev openjdk-11-jre-headless wget gcc libc-dev git curl vim jq && \
     pip3 install urllib3==1.25.11 opensearch-benchmark==1.1.0 awscurl tqdm
-
+RUN mkdir /root/kafka-tools
+RUN mkdir /root/kafka-tools/aws
 COPY runTestBenchmarks.sh /root/
 COPY humanReadableLogs.py /root/
 COPY catIndices.sh /root/
+COPY msk-iam-auth.properties /root/kafka-tools/aws
+COPY kafkaCmdRef.md /root/kafka-tools
 RUN chmod ug+x /root/runTestBenchmarks.sh
 RUN chmod ug+x /root/humanReadableLogs.py
 RUN chmod ug+x /root/catIndices.sh
+WORKDIR /root/kafka-tools
+# Get kafka distribution and unpack to 'kafka'
+RUN wget -qO- https://archive.apache.org/dist/kafka/3.5.0/kafka_2.13-3.5.0.tgz | tar --transform 's!^[^/]*!kafka!' -xvz
+RUN wget -O kafka/libs/msk-iam-auth.jar https://github.com/aws/aws-msk-iam-auth/releases/download/v1.1.9/aws-msk-iam-auth-1.1.9-all.jar
 WORKDIR /root
 
 CMD tail -f /dev/null

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/kafkaCmdRef.md

@@ -0,0 +1,55 @@
+## Kafka Sample Commands
+
+Sample commands to be run from the kafka/bin directory
+
+### AWS
+
+When running these commands in an AWS environment, the following piece should be added to the end of the commands to allow IAM communication with MSK
+```shell
+--command-config ../../aws/msk-iam-auth.properties
+```
+
+### Topics
+
+List all Topics
+```shell
+./kafka-topics.sh --bootstrap-server "$MIGRATION_KAFKA_BROKER_ENDPOINTS" --list
+```
+
+Create a Topic
+```shell
+./kafka-topics.sh --bootstrap-server "$MIGRATION_KAFKA_BROKER_ENDPOINTS" --create --replication-factor 1 --topic test-topic
+```
+
+Delete a Topic
+```shell
+./kafka-topics.sh --bootstrap-server "$MIGRATION_KAFKA_BROKER_ENDPOINTS" --delete --topic test-topic
+```
+
+### Consumer Groups
+
+List all Consumer Groups
+```shell
+./kafka-consumer-groups.sh --bootstrap-server "$MIGRATION_KAFKA_BROKER_ENDPOINTS" --timeout 100000 --list
+```
+
+Describe all Consumer Groups
+```shell
+./kafka-consumer-groups.sh --bootstrap-server "$MIGRATION_KAFKA_BROKER_ENDPOINTS" --timeout 100000 --describe --all-groups
+```
+
+Describe a particular Consumer Group
+```shell
+./kafka-consumer-groups.sh --bootstrap-server "$MIGRATION_KAFKA_BROKER_ENDPOINTS" --timeout 100000 --describe --group logging-group-default
+```
+
+Delete a particular Consumer Group (Requires Consumer Group to be empty to perform)
+```shell
+./kafka-consumer-groups.sh --bootstrap-server "$MIGRATION_KAFKA_BROKER_ENDPOINTS" --timeout 100000 --delete --group logging-group-default
+```
+
+Reset a Consumer Group offset to latest (Requires Consumer Group to not be active to perform)
+More options for different types of reset [here](https://docs.cloudera.com/runtime/7.2.8/kafka-managing/topics/kafka-manage-cli-cgroups.html#pnavId2)
+```shell
+./kafka-consumer-groups.sh --bootstrap-server "$MIGRATION_KAFKA_BROKER_ENDPOINTS" --timeout 100000 --reset-offsets --to-latest --topic logging-traffic-topic --group logging-group-default --execute
+```

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/msk-iam-auth.properties

@@ -0,0 +1,13 @@
+# --- Additional setup to use AWS MSK IAM library for communication with an AWS MSK cluster
+# Sets up TLS for encryption and SASL for authN.
+security.protocol = SASL_SSL
+
+# Identifies the SASL mechanism to use.
+sasl.mechanism = AWS_MSK_IAM
+
+# Binds SASL client implementation.
+sasl.jaas.config = software.amazon.msk.auth.iam.IAMLoginModule required;
+
+# Encapsulates constructing a SigV4 signature based on extracted credentials.
+# The SASL client bound by "sasl.jaas.config" invokes this class.
+sasl.client.callback.handler.class = software.amazon.msk.auth.iam.IAMClientCallbackHandler

deployment/cdk/opensearch-service-migration/README.md

@@ -4,28 +4,31 @@ This repo contains an IaC CDK solution for deploying an OpenSearch Service Domai
 
 ### Getting Started
 
+#### Project required setup
+
+1- It is necessary to run `npm install` within this current directory to install required packages that this app and CDK need for operation.
+
+2- Set the `CDK_DEPLOYMENT_STAGE` environment variable to assist in naming resources and preventing collisions. Typically, this would be set to a value such as `dev`, `gamma`, `Wave1`, `PROD` and will be used to distinguish AWS resources for a given region and deployment stage. For example the CloudFormation stack may be named like `OSServiceDomain-dev-us-east-1`. This stage environment variable should only be used for the disambiguation of user resources.
+
 #### First time using CDK?
 
-You can install the CDK CLI tool by running:
+If this is your first experience with CDK, follow the steps below to get started:
+
+1- Install the **CDK CLI** tool by running:
 ```
 npm install -g aws-cdk
 ```
 
-You then will need to configure the desired [AWS credentials](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html#getting_started_prerequisites), as these will dictate the region and account used for deployment.
+2- Configure the desired **[AWS credentials](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html#getting_started_prerequisites)**, as these will dictate the region and account used for deployment.
+
+3- **Bootstrap CDK**: if you have not run CDK previously in the configured region of you account, it is necessary to run the following command to set up a small CloudFormation stack of resources that CDK needs to function within your account
 
-Next if you have not run CDK previously in the configured region of you account, it is necessary to run the following command to set up a small CloudFormation stack of resources that CDK needs to function within your account
 ```
 cdk bootstrap
 ```
 
 Further CDK documentation [here](https://docs.aws.amazon.com/cdk/v2/guide/cli.html)
 
-#### Project required setup
-
-It is necessary to run `npm install` within this current directory to install required packages that this app and CDK need for operation.
-
-A `CDK_DEPLOYMENT_STAGE` environment variable must be set to assist in naming resources and preventing collisions. Typically, this would be set to a value such as `dev`, `gamma`, `Wave1`, `PROD` and will be used to distinguish AWS resources for a given region and deployment stage. For example the CloudFormation stack may be named like `OSServiceDomain-dev-us-east-1`. This stage environment variable should only be used for the disambiguation of user resources.
-
 ### Deploying your CDK
 Before deploying your CDK you should fill in any desired context parameters that will dictate the composition of your OpenSearch Service Domain
 

deployment/copilot/devDeploy.sh

@@ -117,7 +117,7 @@ fi
 if [ "$DESTROY_ALL_COPILOT" = true ] ; then
     # Reset AWS_DEFAULT_REGION as the SDK used by Copilot will first check here for region to use to locate the Copilot app (https://github.com/aws/copilot-cli/issues/5138)
     export AWS_DEFAULT_REGION=""
-    copilot app delete
+    copilot app delete --name $COPILOT_APP_NAME
     echo "Destroying a Copilot app will not remove generated manifest.yml files in the copilot/environments directory. These should be manually deleted before deploying again. "
     exit 1
 fi
@@ -140,6 +140,9 @@ cdk deploy "*" --tags $TAGS --c domainName="aos-domain" --c engineVersion="OS_2.
 
 # Collect export commands from CDK output, which are needed by Copilot, wrap the commands in double quotes and store them within the "environment" dir
 export_file_path=../../copilot/environments/$COPILOT_DEPLOYMENT_STAGE/envExports.sh
+
+mkdir -p $(dirname "${export_file_path}")
+
 grep -o "export [a-zA-Z0-9_]*=[^\\;\"]*" cdk.out/cdkOutput.json | sed 's/=/="/' | sed 's/.*/&"/' > "${export_file_path}"
 source "${export_file_path}"
 chmod +x "${export_file_path}"

deployment/copilot/migration-console/addons/taskRole.yml

@@ -0,0 +1,56 @@
+AWSTemplateFormatVersion: "2010-09-09"
+
+# You can use any of these parameters to create conditions or mappings in your template.
+Parameters:
+  App:
+    Type: String
+    Description: Your application's name.
+  Env:
+    Type: String
+    Description: The environment name your service, job, or workflow is being deployed to.
+  Name:
+    Type: String
+    Description: Your workload's name.
+
+Resources:
+  MSKAdminAccessPolicy:
+    Type: AWS::IAM::ManagedPolicy
+    Properties:
+      Description: Allow compute host to have admin access to MSK
+      PolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          # Cluster level actions
+          - Action:
+              - kafka-cluster:*
+            Effect: Allow
+            Resource:
+              - Fn::ImportValue: !Sub "${App}-${Env}-msk-cluster-arn"
+          # Topic level actions
+          - Action:
+              - kafka-cluster:*
+            Effect: Allow
+            Resource:
+              !Join
+              # Delimiter
+              - ''
+              # Values to join
+              - - { "Fn::Join": [ ":topic", { "Fn::Split": [ ":cluster", { "Fn::ImportValue": !Sub "${App}-${Env}-msk-cluster-arn" } ] } ] }
+                - "/*"
+          # Consumer group level actions
+          - Action:
+              - kafka-cluster:*
+            Effect: Allow
+            Resource:
+              !Join
+              # Delimiter
+              - ''
+              # Values to join
+              - - { "Fn::Join": [ ":group", { "Fn::Split": [ ":cluster", { "Fn::ImportValue": !Sub "${App}-${Env}-msk-cluster-arn" } ] } ] }
+                - "/*"
+
+Outputs:
+  # 1. You need to output the IAM ManagedPolicy so that Copilot can add it as a managed policy to your ECS task role.
+  MSKAdminAccessPolicyArn:
+    Description: "The ARN of the ManagedPolicy to attach to the task role."
+    Value: !Ref MSKAdminAccessPolicy

deployment/copilot/migration-console/manifest.yml

@@ -10,7 +10,7 @@ type: Backend Service
 network:
   connect: true
   vpc:
-    security_groups: [ "${MIGRATION_DOMAIN_SG_ID}", "${MIGRATION_REPLAYER_OUTPUT_EFS_SG_ID}" ]
+    security_groups: [ "${MIGRATION_DOMAIN_SG_ID}", "${MIGRATION_REPLAYER_OUTPUT_EFS_SG_ID}", "${MIGRATION_CAPTURE_MSK_SG_ID}" ]
 
 # Configuration for your containers and service.
 image:
@@ -33,6 +33,7 @@ exec: true     # Enable getting a shell to your container (https://docs.aws.amaz
 # Pass environment variables as key value pairs.
 variables:
   MIGRATION_DOMAIN_ENDPOINT: ${MIGRATION_DOMAIN_ENDPOINT}
+  MIGRATION_KAFKA_BROKER_ENDPOINTS: ${MIGRATION_KAFKA_BROKER_ENDPOINTS}
 
 environments:
   dev:

jenkins/release.jenkinsFile

@@ -0,0 +1,19 @@
+lib = library(identifier: '[email protected]', retriever: modernSCM([
+    $class: 'GitSCMSource',
+    remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
+]))
+
+standardReleasePipelineWithGenericTrigger(
+    tokenIdCredential: 'jenkins-opensearch-migrations-generic-webhook-token',
+    causeString: 'A tag was cut on opensearch-project/opensearch-migrations repository causing this workflow to run',
+    downloadReleaseAsset: true,
+    publishRelease: true) {
+                        publishToArtifactsProdBucket(
+                            assumedRoleName: 'opensearch-migrations-upload-role',
+                            source: "${WORKSPACE}/artifacts.tar.gz",
+                            destination: "migrations/${tag}/opensearch-migrations-${tag}.tar.gz",
+                            signingPlatform: 'linux',
+                            sigType: '.sig',
+                            sigOverwrite: true
+                        )
+    }

test/requirements.txt

@@ -6,4 +6,4 @@ packaging==23.1
 pluggy==1.0.0
 pytest==7.3.1
 requests==2.31.0
-urllib3==2.0.3
+urllib3==2.0.6