feat: Add sync for remote DDB to local instance (#27)

README.md

@@ -238,6 +238,26 @@ the slave table(s).
 See the heading below entitled "Authentication and Authorization to AWS DynamoDB API" for
 more information related to credentials.
 
+### Syncing Tables to Local Instance
+
+When developing locally using the DynamoDB emulator or Docker image, it can be useful to pull
+data from upstream environments.
+
+See the [AWS documentation on how to set up the emulator or Docker image for local development](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html).
+
+To synchronize from a remote table to a local table, set the `--slave-profile` parameter
+to the URL of the local endpoint:
+
+```bash
+node src/cli.js \
+   --master us-east-1:my-dynamodb-table \
+   --slave-profile http://localhost:8000 \
+   --slave us-west-2:my-dynamodb-table \
+   --write-missing \
+   --write-differing
+```
+
+Note: the region does not matter on the `--slave`; feel free to use any valid region.
 
 ### "Dry Run" Mode
 
@@ -318,7 +338,6 @@ receives will only last for one hour, and cannot be refreshed without a new MFA
 parallelism, and related limiting arguments must be configured to allow your entire table
 to be scanned during a single hour if you are using MFA.
 
-
 ### What Permissions Are Needed?
 
 On the **master** table you will need:

src/Synchronizer.js

@@ -41,7 +41,15 @@ module.exports = Class.extend({
       this._master = _.extend({}, master, { id: (master.region + ':' + master.name), docs: this._makeDocClient(master) });
 
       this._slaves = _.map(slaves, function(def) {
-         return _.extend({}, def, { id: (def.region + ':' + def.name), docs: this._makeDocClient(def, opts.slaveCredentials) });
+         var client;
+
+         if (opts.localhostTarget) {
+            client = this._makeLocalDocClient(def, opts.localhostTarget);
+         } else {
+            client = this._makeDocClient(def, opts.slaveCredentials);
+         }
+
+         return _.extend({}, def, { id: (def.region + ':' + def.name), docs: client });
       }.bind(this));
 
       this._abortScanning = false;
@@ -507,7 +515,9 @@ module.exports = Class.extend({
    _compareTableDescriptions: function() {
       var def = Q.defer(),
           describeMaster = this._describeTable(this._master),
-          describeSlaves = Q.all(_.map(this._slaves, _.partial(this._describeTable.bind(this), _, this._opts.slaveCredentials)));
+          slaveCreds = this._opts.slaveCredentials,
+          localTarget = this._opts.localhostTarget,
+          describeSlaves = Q.all(_.map(this._slaves, _.partial(this._describeTable.bind(this), _, slaveCreds, localTarget)));
 
       function logDescription(title, tableDef, tableDesc) {
          console.log('%s table %s', title, tableDef.id);
@@ -560,8 +570,17 @@ module.exports = Class.extend({
       return def.promise;
    },
 
-   _describeTable: function(tableDef, creds) {
-      var dyn = new AWS.DynamoDB({ region: tableDef.region, credentials: creds || AWS.config.credentials });
+   _describeTable: function(tableDef, creds, localhostTarget) {
+      var options = { region: tableDef.region },
+          dyn;
+
+      if (localhostTarget) {
+         options.endpoint = localhostTarget;
+      } else {
+         options.credentials = creds || AWS.config.credentials;
+      }
+
+      dyn = new AWS.DynamoDB(options);
 
       return Q.ninvoke(dyn, 'describeTable', { TableName: tableDef.name })
          .then(function(resp) {
@@ -580,6 +599,17 @@ module.exports = Class.extend({
       });
    },
 
+   _makeLocalDocClient: function(def, localhostTarget) {
+      return new AWS.DynamoDB.DocumentClient({
+         region: def.region,
+         endpoint: localhostTarget,
+         maxRetries: this._opts.maxRetries,
+         retryDelayOptions: {
+            base: this._opts.retryDelayBase,
+         },
+      });
+   },
+
    _outputStats: function() {
       console.log('\nSynchronization completed. Stats:');
 

src/cli.js

@@ -156,8 +156,13 @@ if (!_.isEmpty(argv.profile)) {
 AWS.config.credentials = setupRoleRelatedCredentials('', 'for master', AWS.config.credentials);
 
 if (!_.isEmpty(argv['slave-profile'])) {
-   console.log('Setting AWS credentials provider to use profile %s for slaves', argv['slave-profile']);
-   options.slaveCredentials = new AWS.SharedIniFileCredentials({ profile: argv['slave-profile'] });
+   if (argv['slave-profile'].indexOf('localhost') > -1) {
+      console.log('Using localhost endpoint.');
+      options.localhostTarget = argv['slave-profile'];
+   } else {
+      console.log('Setting AWS credentials provider to use profile %s for slaves', argv['slave-profile']);
+      options.slaveCredentials = new AWS.SharedIniFileCredentials({ profile: argv['slave-profile'] });
+   }
 }
 
 options.slaveCredentials = setupRoleRelatedCredentials('slave-', 'for slaves', options.slaveCredentials || AWS.config.credentials);