From 3c5187cc6fa465104b5a251665b97607d8f97ce7 Mon Sep 17 00:00:00 2001 From: Nishant Goel Date: Tue, 20 Jun 2023 08:39:37 +0530 Subject: [PATCH 1/8] Adding check in index replication task in case of error coming on restore state Signed-off-by: Nishant Goel --- .../index/TransportReplicateIndexClusterManagerNodeAction.kt | 3 ++- .../opensearch/replication/task/index/IndexReplicationTask.kt | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/org/opensearch/replication/action/index/TransportReplicateIndexClusterManagerNodeAction.kt b/src/main/kotlin/org/opensearch/replication/action/index/TransportReplicateIndexClusterManagerNodeAction.kt index 1a926798..aa45681b 100644 --- a/src/main/kotlin/org/opensearch/replication/action/index/TransportReplicateIndexClusterManagerNodeAction.kt +++ b/src/main/kotlin/org/opensearch/replication/action/index/TransportReplicateIndexClusterManagerNodeAction.kt @@ -127,7 +127,8 @@ class TransportReplicateIndexClusterManagerNodeAction @Inject constructor(transp persistentTasksService.waitForTaskCondition(task.id, replicateIndexReq.timeout()) { t -> val replicationState = (t.state as IndexReplicationState?)?.state replicationState == ReplicationState.FOLLOWING || - (!replicateIndexReq.waitForRestore && replicationState == ReplicationState.RESTORING) + (!replicateIndexReq.waitForRestore && replicationState == ReplicationState.RESTORING) || + (!replicateIndexReq. waitForRestore && replicationState == ReplicationState. FAILED) } listener.onResponse(AcknowledgedResponse(true)) diff --git a/src/main/kotlin/org/opensearch/replication/task/index/IndexReplicationTask.kt b/src/main/kotlin/org/opensearch/replication/task/index/IndexReplicationTask.kt index d51af7c0..c7891bea 100644 --- a/src/main/kotlin/org/opensearch/replication/task/index/IndexReplicationTask.kt +++ b/src/main/kotlin/org/opensearch/replication/task/index/IndexReplicationTask.kt @@ -828,7 +828,11 @@ open class IndexReplicationTask(id: Long, type: String, action: String, descript } } catch(e: Exception) { val err = "Unable to initiate restore call for $followerIndexName from $leaderAlias:${leaderIndex.name}" + val aliasErrMsg = "cannot rename index [${leaderIndex.name}] into [$followerIndexName] because of conflict with an alias with the same name" log.error(err, e) + if (e.message!!.contains(aliasErrMsg)) { + return FailedState (Collections.emptyMap(), aliasErrMsg) + } return FailedState(Collections.emptyMap(), err) } cso.waitForNextChange("remote restore start") { inProgressRestore(it) != null } From 40b9eee5e6368aa0467bbd7e7d78eae54c1a25d9 Mon Sep 17 00:00:00 2001 From: Nishant Goel Date: Wed, 21 Jun 2023 15:56:56 +0530 Subject: [PATCH 2/8] Adding exception message in failed status error message Signed-off-by: Nishant Goel --- .../TransportReplicateIndexClusterManagerNodeAction.kt | 2 +- .../replication/task/index/IndexReplicationTask.kt | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/org/opensearch/replication/action/index/TransportReplicateIndexClusterManagerNodeAction.kt b/src/main/kotlin/org/opensearch/replication/action/index/TransportReplicateIndexClusterManagerNodeAction.kt index aa45681b..07cc16a9 100644 --- a/src/main/kotlin/org/opensearch/replication/action/index/TransportReplicateIndexClusterManagerNodeAction.kt +++ b/src/main/kotlin/org/opensearch/replication/action/index/TransportReplicateIndexClusterManagerNodeAction.kt @@ -128,7 +128,7 @@ class TransportReplicateIndexClusterManagerNodeAction @Inject constructor(transp val replicationState = (t.state as IndexReplicationState?)?.state replicationState == ReplicationState.FOLLOWING || (!replicateIndexReq.waitForRestore && replicationState == ReplicationState.RESTORING) || - (!replicateIndexReq. waitForRestore && replicationState == ReplicationState. FAILED) + (!replicateIndexReq.waitForRestore && replicationState == ReplicationState.FAILED) } listener.onResponse(AcknowledgedResponse(true)) diff --git a/src/main/kotlin/org/opensearch/replication/task/index/IndexReplicationTask.kt b/src/main/kotlin/org/opensearch/replication/task/index/IndexReplicationTask.kt index c7891bea..d190da68 100644 --- a/src/main/kotlin/org/opensearch/replication/task/index/IndexReplicationTask.kt +++ b/src/main/kotlin/org/opensearch/replication/task/index/IndexReplicationTask.kt @@ -828,12 +828,8 @@ open class IndexReplicationTask(id: Long, type: String, action: String, descript } } catch(e: Exception) { val err = "Unable to initiate restore call for $followerIndexName from $leaderAlias:${leaderIndex.name}" - val aliasErrMsg = "cannot rename index [${leaderIndex.name}] into [$followerIndexName] because of conflict with an alias with the same name" log.error(err, e) - if (e.message!!.contains(aliasErrMsg)) { - return FailedState (Collections.emptyMap(), aliasErrMsg) - } - return FailedState(Collections.emptyMap(), err) + return FailedState(Collections.emptyMap(), e.message!!) } cso.waitForNextChange("remote restore start") { inProgressRestore(it) != null } return RestoreState From 0f00726d6f9a5dd7784cf3dea6a5896db09d91c2 Mon Sep 17 00:00:00 2001 From: Nishant Goel Date: Fri, 3 May 2024 16:53:27 +0530 Subject: [PATCH 3/8] checking integ test script changes on windows setup --- .github/workflows/build-local.yml | 33 +++++++++++++++++++++ scripts/integtest.sh | 49 +++++++++++++++++++++++-------- 2 files changed, 70 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/build-local.yml diff --git a/.github/workflows/build-local.yml b/.github/workflows/build-local.yml new file mode 100644 index 00000000..3b604ef7 --- /dev/null +++ b/.github/workflows/build-local.yml @@ -0,0 +1,33 @@ +name: Build Replication plugin +# This workflow is triggered on pull requests to main branch +on: + push: + branches: + - '*' + +# We build for other platforms except linux which is already covered in build-and-test. +# Also, We're not running tests here as those are already covered with linux build. +jobs: + build: + continue-on-error: true + strategy: + matrix: + java: + - 17 + os: + - windows-latest + # Job name + name: Java ${{ matrix.java }} On ${{ matrix.os }} + runs-on: ${{ matrix.os }} + steps: + # This step uses the setup-java Github action: https://github.com/actions/setup-java + - name: Set Up JDK ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + # This step uses the checkout Github action: https://github.com/actions/checkout + - name: Checkout Branch + uses: actions/checkout@v2 + - name: Build and run Replication tests + run: | + scripts/integtest.sh -e '[{"cluster_name": "leader","data_nodes": [{"endpoint": "localhost","port": 9200,"transport": 9300},{"endpoint": "localhost","port": 9201,"transport": 9301}],"cluster_manager_nodes": []},{"cluster_name": "follower","data_nodes": [{"endpoint": "localhost","port": 9202,"transport": 9302},{"endpoint": "localhost","port": 9203,"transport":9303}],"cluster_manager_nodes": []}]' -s false -v 2.13.0 diff --git a/scripts/integtest.sh b/scripts/integtest.sh index f967dc8a..30306d49 100644 --- a/scripts/integtest.sh +++ b/scripts/integtest.sh @@ -103,24 +103,49 @@ then exit 1 fi - data=$(python3 -c "import json; cluster=$ENDPOINT_LIST ; data_nodes=cluster; print(data_nodes[0][\"data_nodes\"][0][\"endpoint\"],':',data_nodes[0][\"data_nodes\"][0][\"port\"],':',data_nodes[0][\"data_nodes\"][0][\"transport\"],',',data_nodes[1][\"data_nodes\"][0][\"endpoint\"],':',data_nodes[1][\"data_nodes\"][0][\"port\"],':',data_nodes[1][\"data_nodes\"][0][\"transport\"])" | tr -d "[:blank:]") + extract_values() { + local cluster_name="$1" + local field="$2" + echo "$ENDPOINT_LIST" | awk -v cluster="$cluster_name" -v field="$field" ' + BEGIN { RS=","; FS=":" } + $1 ~ "\"cluster_name\"" && $2 ~ "\"" cluster "\"" { + while (getline) { + if ($1 ~ "\"" field "\"") { + gsub(/"/, "", $2) + print $2 + exit + } + } + } + ' | tr -d '{}' + } - leader=$(echo $data | cut -d ',' -f1 | cut -d ':' -f1,2 ) - follower=$(echo $data | cut -d ',' -f2 | cut -d ':' -f1,2 ) - echo "leader: $leader" - echo "follower: $follower" + + # Extract values for leader cluster + leader_endpoint=$(extract_values "leader" "endpoint") + leader_port=$(extract_values "leader" "port") + leader_transport=$(extract_values "leader" "transport") + + # Extract values for follower cluster + follower_endpoint=$(extract_values "follower" "endpoint") + follower_port=$(extract_values "follower" "port") + follower_transport=$(extract_values "follower" "transport") + + # Print extracted data + echo "Leader Endpoint: $leader_endpoint" + echo "Leader Port: $leader_port" + echo "Leader Transport: $leader_transport" + echo "Follower Endpoint: $follower_endpoint" + echo "Follower Port: $follower_port" + echo "Follower Transport: $follower_transport" # Get number of nodes, assuming both leader and follower have same number of nodes - numNodes=$((${follower##*:} - ${leader##*:})) + numNodes=$((${follower_port} - ${leader_port})) echo "numNodes: $numNodes" + - LTRANSPORT_PORT=$(echo $data | cut -d ',' -f1 | cut -d ':' -f1,3 ) - FTRANSPORT_PORT=$(echo $data | cut -d ',' -f2 | cut -d ':' -f1,3 ) - echo "LTRANSPORT_PORT: $LTRANSPORT_PORT" - echo "FTRANSPORT_PORT: $FTRANSPORT_PORT" - - eval "./gradlew integTestRemote -Dleader.http_host=\"$leader\" -Dfollower.http_host=\"$follower\" -Dfollower.transport_host=\"$FTRANSPORT_PORT\" -Dleader.transport_host=\"$LTRANSPORT_PORT\" -Dsecurity_enabled=\"$SECURITY_ENABLED\" -Duser=\"$USERNAME\" -Dpassword=\"$PASSWORD\" -PnumNodes=$numNodes --console=plain " + eval "./gradlew integTestRemote -Dleader.http_host=$leader_endpoint:$leader_port -Dfollower.http_host=$follower_endpoint:$follower_port -Dfollower.transport_host=$follower_endpoint:$follower_transport -Dleader.transport_host=$leader_endpoint:$leader_transport -Dsecurity_enabled=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD -PnumNodes=$numNodes --console=plain " else # Single cluster From 006933124ccb6bcfd4141ffcc587d1be0faf17aa Mon Sep 17 00:00:00 2001 From: Nishant Goel Date: Fri, 3 May 2024 17:04:41 +0530 Subject: [PATCH 4/8] fixing workflow --- .github/workflows/build-local.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-local.yml b/.github/workflows/build-local.yml index 3b604ef7..49ee0215 100644 --- a/.github/workflows/build-local.yml +++ b/.github/workflows/build-local.yml @@ -1,4 +1,4 @@ -name: Build Replication plugin +name: IntegTest Remote testing for windows # This workflow is triggered on pull requests to main branch on: push: @@ -30,4 +30,4 @@ jobs: uses: actions/checkout@v2 - name: Build and run Replication tests run: | - scripts/integtest.sh -e '[{"cluster_name": "leader","data_nodes": [{"endpoint": "localhost","port": 9200,"transport": 9300},{"endpoint": "localhost","port": 9201,"transport": 9301}],"cluster_manager_nodes": []},{"cluster_name": "follower","data_nodes": [{"endpoint": "localhost","port": 9202,"transport": 9302},{"endpoint": "localhost","port": 9203,"transport":9303}],"cluster_manager_nodes": []}]' -s false -v 2.13.0 + ./scripts/integtest.sh -e '[{"cluster_name": "leader","data_nodes": [{"endpoint": "localhost","port": 9200,"transport": 9300},{"endpoint": "localhost","port": 9201,"transport": 9301}],"cluster_manager_nodes": []},{"cluster_name": "follower","data_nodes": [{"endpoint": "localhost","port": 9202,"transport": 9302},{"endpoint": "localhost","port": 9203,"transport":9303}],"cluster_manager_nodes": []}]' -s false -v 2.13.0 From 278f08dd891aa03976c2ee7526bea8f86afaafd1 Mon Sep 17 00:00:00 2001 From: Nishant Goel Date: Fri, 3 May 2024 17:15:15 +0530 Subject: [PATCH 5/8] testing workflow --- .github/workflows/build-local.yml | 6 +++++- .github/workflows/build.yml | 3 +-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-local.yml b/.github/workflows/build-local.yml index 49ee0215..5f5c50fd 100644 --- a/.github/workflows/build-local.yml +++ b/.github/workflows/build-local.yml @@ -30,4 +30,8 @@ jobs: uses: actions/checkout@v2 - name: Build and run Replication tests run: | - ./scripts/integtest.sh -e '[{"cluster_name": "leader","data_nodes": [{"endpoint": "localhost","port": 9200,"transport": 9300},{"endpoint": "localhost","port": 9201,"transport": 9301}],"cluster_manager_nodes": []},{"cluster_name": "follower","data_nodes": [{"endpoint": "localhost","port": 9202,"transport": 9302},{"endpoint": "localhost","port": 9203,"transport":9303}],"cluster_manager_nodes": []}]' -s false -v 2.13.0 + king=$(ls) + khan=$(pwd) + echo $khan + jon=$(bash scripts/integtest.sh -e '[{"cluster_name": "leader","data_nodes": [{"endpoint": "localhost","port": 9200,"transport": 9300},{"endpoint": "localhost","port": 9201,"transport": 9301}],"cluster_manager_nodes": []},{"cluster_name": "follower","data_nodes": [{"endpoint": "localhost","port": 9202,"transport": 9302},{"endpoint": "localhost","port": 9203,"transport":9303}],"cluster_manager_nodes": []}]' -s false -v 2.13.0) + echo $jon \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ca44eebf..4c1f27f9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,5 +33,4 @@ jobs: - name: Checkout Branch uses: actions/checkout@v2 - name: Build and run Replication tests - run: | - ./gradlew --refresh-dependencies clean release -D"build.snapshot=true" -x test -x IntegTest + run: ./gradlew --refresh-dependencies clean release -D"build.snapshot=true" -x test -x IntegTest From 731ce6d6e5a36664d162c648cd2514718797f805 Mon Sep 17 00:00:00 2001 From: Nishant Goel Date: Fri, 3 May 2024 17:21:20 +0530 Subject: [PATCH 6/8] test2 --- .github/workflows/build-local.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build-local.yml b/.github/workflows/build-local.yml index 5f5c50fd..845021d3 100644 --- a/.github/workflows/build-local.yml +++ b/.github/workflows/build-local.yml @@ -30,8 +30,5 @@ jobs: uses: actions/checkout@v2 - name: Build and run Replication tests run: | - king=$(ls) - khan=$(pwd) - echo $khan jon=$(bash scripts/integtest.sh -e '[{"cluster_name": "leader","data_nodes": [{"endpoint": "localhost","port": 9200,"transport": 9300},{"endpoint": "localhost","port": 9201,"transport": 9301}],"cluster_manager_nodes": []},{"cluster_name": "follower","data_nodes": [{"endpoint": "localhost","port": 9202,"transport": 9302},{"endpoint": "localhost","port": 9203,"transport":9303}],"cluster_manager_nodes": []}]' -s false -v 2.13.0) echo $jon \ No newline at end of file From aa9502751da390a8a6b85bfc508949c9dc3800cf Mon Sep 17 00:00:00 2001 From: Nishant Goel Date: Fri, 3 May 2024 17:24:10 +0530 Subject: [PATCH 7/8] test3 --- .github/workflows/build-local.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-local.yml b/.github/workflows/build-local.yml index 845021d3..dfe179b9 100644 --- a/.github/workflows/build-local.yml +++ b/.github/workflows/build-local.yml @@ -29,6 +29,7 @@ jobs: - name: Checkout Branch uses: actions/checkout@v2 - name: Build and run Replication tests + shell: bash run: | jon=$(bash scripts/integtest.sh -e '[{"cluster_name": "leader","data_nodes": [{"endpoint": "localhost","port": 9200,"transport": 9300},{"endpoint": "localhost","port": 9201,"transport": 9301}],"cluster_manager_nodes": []},{"cluster_name": "follower","data_nodes": [{"endpoint": "localhost","port": 9202,"transport": 9302},{"endpoint": "localhost","port": 9203,"transport":9303}],"cluster_manager_nodes": []}]' -s false -v 2.13.0) echo $jon \ No newline at end of file From faa161208d6e37e5180ae60d192f14507befd7aa Mon Sep 17 00:00:00 2001 From: Nishant Goel Date: Fri, 3 May 2024 21:05:16 +0530 Subject: [PATCH 8/8] test4 --- build.gradle | 2 ++ scripts/integtest.sh | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) mode change 100644 => 100755 scripts/integtest.sh diff --git a/build.gradle b/build.gradle index e862827b..edf539b0 100644 --- a/build.gradle +++ b/build.gradle @@ -937,6 +937,8 @@ task integTestRemote (type: RestIntegTestTask) { systemProperty "tests.cluster.followCluster.total_nodes", "${-> numberOfNodes.toString()}" systemProperty "tests.cluster.leaderCluster.total_nodes", "${-> numberOfNodes.toString()}" systemProperty "build.dir", "${buildDir}" + systemProperty "user", System.getProperty("user") + systemProperty "password", System.getProperty("password") } filter { diff --git a/scripts/integtest.sh b/scripts/integtest.sh old mode 100644 new mode 100755 index 30306d49..0d2bebe0 --- a/scripts/integtest.sh +++ b/scripts/integtest.sh @@ -113,6 +113,7 @@ then while (getline) { if ($1 ~ "\"" field "\"") { gsub(/"/, "", $2) + gsub(/ /, "", $2) print $2 exit } @@ -145,7 +146,7 @@ then echo "numNodes: $numNodes" - eval "./gradlew integTestRemote -Dleader.http_host=$leader_endpoint:$leader_port -Dfollower.http_host=$follower_endpoint:$follower_port -Dfollower.transport_host=$follower_endpoint:$follower_transport -Dleader.transport_host=$leader_endpoint:$leader_transport -Dsecurity_enabled=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD -PnumNodes=$numNodes --console=plain " + ./gradlew integTestRemote -Dleader.http_host="$leader_endpoint:$leader_port" -Dfollower.http_host="$follower_endpoint:$follower_port" -Dfollower.transport_host="$follower_endpoint:$follower_transport" -Dleader.transport_host="$leader_endpoint:$leader_transport" -Dsecurity_enabled=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD -PnumNodes=$numNodes --console=plain else # Single cluster