Multi Node Cluster support for move-node

bin/get-shard-mapping.js

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+
+const { get_shard_mapping } = require('../src/get-shard-mapping');
+
+(async () => {
+  try {
+    const shardMapping = await get_shard_mapping();
+    console.log(shardMapping);
+  } catch (err) {
+    console.error('An unexpected error occurred', err);
+    process.exit(1);
+  }
+})();

bin/move-node.js

@@ -1,16 +1,34 @@
 #!/usr/bin/env node
 
+const [,, toNode, shardMapJson] = process.argv;
+
 const { moveNode, syncShards } = require('../src/move-node');
 const { removeNode } = require('../src/remove-node');
 
+const parseNodeMapping = function (input) {
+  if (!input) {
+    return undefined;
+  } else if (input.startsWith('{')) {
+    return JSON.parse(input);
+  } else if (input.includes(':')) {
+    const [oldNode, newNode] = input.split(':');
+    return { [oldNode]: newNode };
+  }
+  return input;
+};
+
 (async () => {
   try {
-    const removedNodes = await moveNode();
+    const parsedToNode = parseNodeMapping(toNode);
+
+    const removedNodes = await moveNode(parsedToNode, shardMapJson);
+    console.log('Node moved successfully');
+
     for (const node of removedNodes) {
       await removeNode(node);
     }
     await syncShards();
-    console.log('Node moved successfully');
+    console.log('Removed nodes:', removedNodes);
   } catch (err) {
     console.error('An unexpected error occurred', err);
     process.exit(1);

docker-compose.yml

@@ -2,7 +2,11 @@ version: '3.9'
 
 services:
   couch-migration:
-    image: public.ecr.aws/medic/couchdb-migration:1.0.3
+    image: public.ecr.aws/medic/couchdb-migration:1.0.4
+    # use local image
+    #build:
+    #  context: .
+    #  dockerfile: Dockerfile
     networks:
       - cht-net
     environment:

package.json

@@ -1,6 +1,6 @@
 {
   "name": "couchdb-migration",
-  "version": "1.0.3",
+  "version": "1.0.4",
   "description": "Tool that interfaces with CouchDB to help migrating data within a cluster.",
   "scripts": {
     "eslint": "eslint --color --cache ./src ./test",
@@ -22,7 +22,8 @@
     "remove-node": "bin/remove-node.js",
     "check-couchdb-up": "bin/check-couchdb-up.js",
     "pre-index-views": "bin/pre-index-views.js",
-    "verify": "bin/verify.js"
+    "verify": "bin/verify.js",
+    "get-shard-mapping": "bin/get-shard-mapping.js"
   },
   "repository": {
     "type": "git",

src/check-couch-up.js

@@ -1,5 +1,5 @@
 const utils = require('./utils');
-const NUM_RETRY = 200;
+const NUM_RETRY = 300;
 const TIMEOUT_RETRY = 1000; // 1 second
 
 const isCouchUp = async () => {
@@ -33,14 +33,14 @@ const repeatRetry = async (promiseFn) => {
 const checkCouchUp = async () => {
   const isUp = await repeatRetry(isCouchUp);
   if (!isUp) {
-    throw new Error('CouchDb is not up after 100 seconds.');
+    throw new Error('CouchDb is not up after 300 seconds.');
   }
 };
 
 const checkClusterReady = async (nbrNodes) => {
   const isReady = await repeatRetry(isClusterReady.bind({}, nbrNodes));
   if (!isReady) {
-    throw new Error('CouchDb Cluster is not ready after 100 seconds.');
+    throw new Error('CouchDb Cluster is not ready after 300 seconds.');
   }
 };
 

src/get-shard-mapping.js

@@ -0,0 +1,31 @@
+const utils = require('./utils');
+
+const get_shard_mapping = async () => {
+  try {
+    const allDbs = await utils.getDbs();
+    const shardMap = {};
+
+    for (const db of allDbs) {
+      const url = await utils.getUrl(`${db}/_shards`);
+      const shardInfo = await utils.request({ url });
+
+      for (const [shardRange, nodeList] of Object.entries(shardInfo.shards)) {
+        // In n=1 setup, there should be only one node per shard range.
+        // We will have to revisit this if we ever support n>1.
+        if (nodeList.length !== 1) {
+          console.warn(`Unexpected number of nodes for range ${shardRange}: ${nodeList.length}`);
+        }
+        shardMap[shardRange] = nodeList[0];
+      }
+    }
+
+    return JSON.stringify(shardMap);
+  } catch (err) {
+    console.error('Error getting shard mapping:', err);
+    throw new Error('Failed to get shard mapping');
+  }
+};
+
+module.exports = {
+  get_shard_mapping
+};

src/move-node.js

@@ -1,22 +1,45 @@
 const utils = require('./utils');
 const moveShard = require('./move-shard');
 
-const moveNode = async (toNode) => {
+const moveNode = async (toNode, shardMapJson) => {
   const removedNodes = [];
 
-  if (!toNode) {
-    const nodes = await utils.getNodes();
-    if (nodes.length > 1) {
-      throw new Error('More than one node found.');
+  if (typeof toNode === 'object') {
+    // Multi-node migration
+    if (!shardMapJson) {
+      throw new Error('Shard map JSON is required for multi-node migration');
     }
-    toNode = nodes[0];
-  }
+    const shardMap = JSON.parse(shardMapJson);
+    const [oldNode, newNode] = Object.entries(toNode)[0];
+    console.log(`Migrating from ${oldNode} to ${newNode}`);
 
-  const shards = await utils.getShards();
-  for (const shard of shards) {
-    const oldNodes = await moveShard.moveShard(shard, toNode);
-    removedNodes.push(...oldNodes);
+    for (const [shardRange, currentNode] of Object.entries(shardMap)) {
+      if (currentNode === oldNode) {
+        console.log(`Moving shard ${shardRange} from ${oldNode} to ${newNode}`);
+        const oldNodes = await moveShard.moveShard(shardRange, newNode);
+        removedNodes.push(...oldNodes);
+      }
+    }
+    if (!removedNodes.includes(oldNode)) {
+      removedNodes.push(oldNode);
+    }
+  } else {
+    // Single node migration
+    if (!toNode) {
+      const nodes = await utils.getNodes();
+      if (nodes.length > 1) {
+        throw new Error('More than one node found. Please specify a node mapping in the format oldNode:newNode.');
+      }
+      toNode = nodes[0];
+    }
+
+    const shards = await utils.getShards();
+    for (const shard of shards) {
+      const oldNodes = await moveShard.moveShard(shard, toNode);
+      removedNodes.push(...oldNodes);
+    }
   }
+
   return [...new Set(removedNodes)];
 };
 

src/verify.js

@@ -1,4 +1,5 @@
 const utils = require('./utils');
+const DBS_TO_IGNORE = ['_global_changes', '_replicator'];
 
 const verifyViews = async (dbName, numDocs) => {
   if (!numDocs) {
@@ -29,6 +30,9 @@ const verifyViews = async (dbName, numDocs) => {
 };
 
 const verifyDb = async (dbName) => {
+  if (DBS_TO_IGNORE.includes(dbName)) {
+    return;
+  }
   console.info(`Verifying ${dbName}`);
   await utils.syncShards(dbName);
 

test/docker-compose-test.yml

@@ -11,3 +11,4 @@ services:
 networks:
   cht-net:
     name: ${CHT_NETWORK:-cht-net}
+    external: true

test/e2e/multi-node-3.sh

@@ -29,37 +29,37 @@ docker rm -f -v scripts-couchdb-1.local-1 scripts-couchdb-2.local-1 scripts-couc
 # create docker network
 docker network create $CHT_NETWORK || true
 # build service image
-docker-compose -f ../docker-compose-test.yml up --build
+docker compose -f ../docker-compose-test.yml up --build
 
 # launch vanilla couch, populate with some data
-docker-compose -f ./scripts/couchdb-vanilla.yml up -d
-docker-compose -f ../docker-compose-test.yml run couch-migration check-couchdb-up
+docker compose -f ./scripts/couchdb-vanilla.yml up -d
+docker compose -f ../docker-compose-test.yml run couch-migration check-couchdb-up
 node ./scripts/generate-documents $jsondataddir
 # pre-index 4.0.1 views
-docker-compose -f ../docker-compose-test.yml run couch-migration pre-index-views 4.4.0
+docker compose -f ../docker-compose-test.yml run couch-migration pre-index-views 4.4.0
 sleep 5 # this is needed, CouchDb runs fsync with a 5 second delay
 # export env for cht 4.x couch
-export $(docker-compose -f ../docker-compose-test.yml run couch-migration get-env | xargs)
-docker-compose -f ./scripts/couchdb-vanilla.yml down --remove-orphans --volumes
+export $(docker compose -f ../docker-compose-test.yml run couch-migration get-env | xargs)
+docker compose -f ./scripts/couchdb-vanilla.yml down --remove-orphans --volumes
 
 # launch cht 4.x CouchDb cluster
-docker-compose -f ./scripts/couchdb3-cluster.yml up -d
-docker-compose -f ../docker-compose-test.yml run couch-migration check-couchdb-up 3
+docker compose -f ./scripts/couchdb3-cluster.yml up -d
+docker compose -f ../docker-compose-test.yml run couch-migration check-couchdb-up 3
 
 # generate shard matrix
 # this is an object that assigns every shard to one of the nodes
-shard_matrix=$(docker-compose -f ../docker-compose-test.yml run couch-migration generate-shard-distribution-matrix)
+shard_matrix=$(docker compose -f ../docker-compose-test.yml run couch-migration generate-shard-distribution-matrix)
 file_matrix="{\"[email protected]\":\"$couch1dir\",\"[email protected]\":\"$couch2dir\",\"[email protected]\":\"$couch3dir\"}"
 echo $shard_matrix
 echo $file_matrix
 # moves shard data files to their corresponding nodes, according to the matrix
-docker-compose -f ../docker-compose-test.yml run couch-migration shard-move-instructions $shard_matrix
+docker compose -f ../docker-compose-test.yml run couch-migration shard-move-instructions $shard_matrix
 node ./scripts/distribute-shards.js $shard_matrix $file_matrix
 # change database metadata to match the shard physical locations
-docker-compose -f ../docker-compose-test.yml run couch-migration move-shards $shard_matrix
-docker-compose -f ../docker-compose-test.yml run couch-migration verify
+docker compose -f ../docker-compose-test.yml run couch-migration move-shards $shard_matrix
+docker compose -f ../docker-compose-test.yml run couch-migration verify
 # test that data exists, database shard maps are correct and view indexes are preserved
 node ./scripts/assert-dbs.js $jsondataddir $shard_matrix
 
-docker-compose -f ./scripts/couchdb-cluster.yml down --remove-orphans --volumes
+docker compose -f ./scripts/couchdb-cluster.yml down --remove-orphans --volumes
 

test/e2e/multi-node.sh

@@ -6,11 +6,12 @@ cd "$BASEDIR"
 user=admin
 password=pass
 
-couch1dir=$(mktemp -d -t couchdb-2x-XXXXXXXXXX)
-couch2dir=$(mktemp -d -t couchdb-2x-XXXXXXXXXX)
-couch3dir=$(mktemp -d -t couchdb-2x-XXXXXXXXXX)
-mkdir -p $couch2dir/shards $couch2dir/.shards $couch3dir/shards $couch3dir/.shards
-jsondataddir=$(mktemp -d -t json-XXXXXXXXXX)
+couch1dir=$(mktemp -d -t couchdb-2x.XXXXXXXXXX)
+couch2dir=$(mktemp -d -t couchdb-2x.XXXXXXXXXX)
+couch3dir=$(mktemp -d -t couchdb-2x.XXXXXXXXXX)
+
+mkdir -p $couch1dir/shards $couch1dir/.shards $couch2dir/shards $couch2dir/.shards $couch3dir/shards $couch3dir/.shards
+jsondataddir=$(mktemp -d -t json.XXXXXXXXXX)
 
 export DB1_DATA=$couch1dir
 export DB2_DATA=$couch2dir
@@ -25,41 +26,72 @@ export COUCH_URL=http://$user:[email protected]:$COUCH_PORT
 
 # cleanup from last test, in case of interruptions
 docker rm -f -v scripts-couchdb-1.local-1 scripts-couchdb-2.local-1 scripts-couchdb-3.local-1
+docker rm -f -v scripts-couchdb-1-namespace-svc-cluster.local-1 scripts-couchdb-2-namespace-svc-cluster.local-1 scripts-couchdb-3-namespace-svc-cluster.local-1
 
 # create docker network
 docker network create $CHT_NETWORK || true
 # build service image
-docker-compose -f ../docker-compose-test.yml up --build
+docker compose -f ../docker-compose-test.yml up --build
 
 # launch vanilla couch, populate with some data
-docker-compose -f ./scripts/couchdb-vanilla.yml up -d
-docker-compose -f ../docker-compose-test.yml run couch-migration check-couchdb-up
+docker compose -f ./scripts/couchdb-vanilla.yml up -d
+docker compose -f ../docker-compose-test.yml run couch-migration check-couchdb-up
 node ./scripts/generate-documents $jsondataddir
 # pre-index 4.0.1 views
-docker-compose -f ../docker-compose-test.yml run couch-migration pre-index-views 4.0.1
+docker compose -f ../docker-compose-test.yml run couch-migration pre-index-views 4.0.1
 sleep 5 # this is needed, CouchDb runs fsync with a 5 second delay
+
 # export env for cht 4.x couch
-export $(docker-compose -f ../docker-compose-test.yml run couch-migration get-env | xargs)
-docker-compose -f ./scripts/couchdb-vanilla.yml down --remove-orphans --volumes
+export $(docker compose -f ../docker-compose-test.yml run couch-migration get-env | xargs)
+docker compose -f ./scripts/couchdb-vanilla.yml down --remove-orphans --volumes
 
 # launch cht 4.x CouchDb cluster
-docker-compose -f ./scripts/couchdb-cluster.yml up -d
-docker-compose -f ../docker-compose-test.yml run couch-migration check-couchdb-up 3
+docker compose -f ./scripts/couchdb-cluster.yml up -d
+docker compose -f ../docker-compose-test.yml run couch-migration check-couchdb-up 3
 
 # generate shard matrix
 # this is an object that assigns every shard to one of the nodes
-shard_matrix=$(docker-compose -f ../docker-compose-test.yml run couch-migration generate-shard-distribution-matrix)
+shard_matrix=$(docker compose -f ../docker-compose-test.yml run couch-migration generate-shard-distribution-matrix)
 file_matrix="{\"[email protected]\":\"$couch1dir\",\"[email protected]\":\"$couch2dir\",\"[email protected]\":\"$couch3dir\"}"
 echo $shard_matrix
 echo $file_matrix
 # moves shard data files to their corresponding nodes, according to the matrix
-docker-compose -f ../docker-compose-test.yml run couch-migration shard-move-instructions $shard_matrix
-node ./scripts/distribute-shards.js $shard_matrix $file_matrix
+docker compose -f ../docker-compose-test.yml run couch-migration shard-move-instructions $shard_matrix
+node ./scripts/distribute-shards.js "$shard_matrix" "$file_matrix"
+
 # change database metadata to match the shard physical locations
-docker-compose -f ../docker-compose-test.yml run couch-migration move-shards $shard_matrix
-docker-compose -f ../docker-compose-test.yml run couch-migration verify
+docker compose -f ../docker-compose-test.yml run couch-migration move-shards $shard_matrix
+# Remove old node from cluster
+docker compose -f ../docker-compose-test.yml run couch-migration remove-node [email protected]
+
+docker compose -f ../docker-compose-test.yml run couch-migration verify
 # test that data exists, database shard maps are correct and view indexes are preserved
 node ./scripts/assert-dbs.js $jsondataddir $shard_matrix
 
-docker-compose -f ./scripts/couchdb-cluster.yml down --remove-orphans --volumes
+# Let's get the shard mapping - we need this to migrate to a different cluster.
+shard_mapping=$(docker compose -f ../docker-compose-test.yml run couch-migration get-shard-mapping)
+echo $shard_mapping
+
+docker compose -f ./scripts/couchdb-cluster.yml down --remove-orphans --volumes
+
+# launch a different cht 4.x CouchDb cluster
+docker compose -f ./scripts/couchdb-cluster-2.yml up -d
+
+# set new COUCH_URL for cluster-2
+export COUCH_URL=http://$user:[email protected]:$COUCH_PORT
+
+docker compose -f ../docker-compose-test.yml run couch-migration check-couchdb-up 3
+
+# change database metadata to match new node name
+# Move nodes one by one using move-node.js oldNode:newNode
+docker compose -f ../docker-compose-test.yml run couch-migration move-node [email protected]:[email protected] $shard_mapping
+docker compose -f ../docker-compose-test.yml run couch-migration move-node [email protected]:[email protected] $shard_mapping
+docker compose -f ../docker-compose-test.yml run couch-migration move-node [email protected]:[email protected] $shard_mapping
+
+docker compose -f ../docker-compose-test.yml run couch-migration verify
+
+# test that data exists, database shard maps are correct and view indexes are preserved
+shard_mapping2=$(docker compose -f ../docker-compose-test.yml run couch-migration get-shard-mapping)
+node ./scripts/assert-dbs.js $jsondataddir $shard_mapping2
 
+docker compose -f ./scripts/couchdb-cluster-2.yml down --remove-orphans --volumes