Feature/wayback machine

functions/package.json

@@ -37,7 +37,7 @@
     "object.entries": "^1.0.4",
     "raven": "^2.6.0",
     "replaceall": "^0.1.6",
-    "supports-color": "^5.4.0"
+    "xml2js": "^0.4.19"
   },
   "devDependencies": {
     "axios-mock-adapter": "^1.15.0",

functions/src/actions/wayback-machine.js

@@ -0,0 +1,138 @@
+// Third party imports
+const axios = require('axios');
+const mustache = require('mustache');
+const xml2js = require('xml2js');
+
+// Local imports
+const config = require('../config');
+const {debug} = require('../utils/logger')('ia:actions:wayback-machine');
+const dialog = require('../dialog');
+const endpointProcessor = require('../network/endpoint-processor');
+const traverse = require('../utils/traverse');
+const waybackStrings = require('../strings').intents.wayback;
+
+/**
+ * Handle wayback query action
+ * - fill slots of wayback query
+ * - perform data requests to archive and alexa rankings
+ * - construct response speech for action
+ *
+ * @param app
+ */
+function handler (app) {
+  // Create wayback object
+  let waybackObject = {
+    url: '',
+    earliestYear: 0,
+    latestYear: 0,
+    totalUniqueURLs: 0,
+    alexaWorldRank: 0,
+    alexaUSRank: 0,
+    speech: waybackStrings.default,
+  };
+
+  // Check to see that both parameters have content
+  if (!app.params.getByName('wayback') && !app.params.getByName('url')) {
+    debug('wayback action called by mistake');
+    dialog.ask(app, waybackObject);
+  }
+
+  // Get url parameter and make url queries
+  waybackObject.url = app.params.getByName('url');
+  const archiveQueryURL = endpointProcessor.preprocess(
+    config.wayback.ARCHIVE, app, waybackObject
+  );
+  const alexaQueryURL = endpointProcessor.preprocess(
+    config.wayback.ALEXA, app, waybackObject
+  );
+
+  return Promise.all([axios.get(archiveQueryURL), axios.get(alexaQueryURL)])
+    .then(([archiveRes, alexaRes]) => {
+      return ([archiveEngine(archiveRes.data), xmlConverter(alexaRes.data)]);
+    })
+    .then(([archiveEngineRes, xmlRes]) => {
+      waybackObject = Object.assign(waybackObject, archiveEngineRes);
+      waybackObject = Object.assign(waybackObject, alexaEngine(xmlRes));
+
+      // Construct response dialog for action
+      if (waybackObject.alexaUSRank !== 0) {
+        waybackObject.speech = mustache.render(waybackStrings.speech, waybackObject);
+        waybackObject.speech += mustache.render(waybackStrings.additionalSpeech, waybackObject);
+      } else {
+        waybackObject.speech = mustache.render(waybackStrings.speech, waybackObject);
+        waybackObject.speech += '.';
+      }
+      dialog.close(app, waybackObject);
+    })
+    .catch(err => {
+      debug('Wayback handler has an error: ', err);
+      waybackObject.speech = waybackStrings.error;
+      dialog.ask(app, waybackObject);
+    });
+} // End of handler
+
+function archiveEngine (archiveJSON) {
+  debug('Inside archiveEngine...');
+  const res = { earliestYear: 0, latestYear: 0, totalUniqueURLs: 0 };
+  // Create array of capture years and then find earliest year
+  //  and most recent year.
+  let yearsArray = Object.keys(archiveJSON.captures);
+  res.earliestYear = yearsArray[0];
+  res.latestYear = yearsArray[yearsArray.length - 1];
+
+  // Traverse URL category
+
+  // Find baseline of URL count
+  res.totalUniqueURLs += traverse(archiveJSON.urls[res.earliestYear]);
+  // debug('Baseline url count: ' + obj.totalUniqueURLs);
+
+  res.totalUniqueURLs += traverse(archiveJSON.new_urls);
+  // debug('Final url count: ' + obj.totalUniqueURLs);
+
+  if (res.totalUniqueURLs > 0) {
+    debug('archiveEngine successful!');
+    return res;
+  } else {
+    let error = new Error('archiveEngine didn\'t work.');
+    throw error;
+  }
+}
+
+function alexaEngine (alexaJSON) {
+  debug('Inside alexaEngine...');
+  const res = { alexaWorldRank: 0, alexaUSRank: 0 };
+  res.alexaWorldRank = alexaJSON['ALEXA']['SD'][0]['POPULARITY'][0]['$']['TEXT'];
+  try {
+    res.alexaUSRank = alexaJSON['ALEXA']['SD'][0]['COUNTRY'][0]['$']['RANK'];
+  } catch (e) {
+    debug('Country not found');
+    debug(e);
+  }
+  if (res.alexaWorldRank > 0) {
+    debug('alexaEngine successful!');
+    return res;
+  } else {
+    let error = new Error('alexaEngine didn\'t work.');
+    throw error;
+  }
+}
+
+function xmlConverter (data) {
+  return new Promise(function (resolve, reject) {
+    debug('Inside xmlConverter promise...');
+    let XMLparser = new xml2js.Parser();
+    XMLparser.parseString(data, function (err, result) {
+      if (err) {
+        let error = new Error('The XML parser didn\'t work.');
+        reject(error);
+      } else {
+        debug('xmlConverter promise successful!');
+        resolve(JSON.parse(JSON.stringify(result)));
+      }
+    });
+  });
+}
+
+module.exports = {
+  handler,
+};

functions/src/config.js

@@ -31,6 +31,11 @@ module.exports = {
     DEFAULT_SONG_IMAGE: 'http://archive.org/images/notfound.png',
   },
 
+  wayback: {
+    ARCHIVE: 'http://web.archive.org/__wb/search/metadata?q={{url}}',
+    ALEXA: 'http://data.alexa.com/data?cli=10&url={{url}}',
+  },
+
   /**
    * settings specific for supported platforms
    */

functions/src/strings.js

@@ -444,6 +444,14 @@ module.exports = {
       speech: 'Version is {{version}}.',
     },
 
+    wayback: {
+      speech: '{{url}} was first captured by the Internet Archive in {{earliestYear}} and most recently in {{latestYear}}. The archive has {{totalUniqueURLs}} unique url\'s for this website. <break></break>{{url}} is ranked <say-as interpret-as="ordinal">{{alexaWorldRank}}</say-as> in the world in popularity',
+      additionalSpeech: ' and <say-as interpret-as="ordinal">{{alexaUSRank}}</say-as> in the United States.',
+      default: 'Would you like to use the wayback machine to hear the history of a website? Simply say Wayback Machine and the name of the website you\'d like to hear.',
+      error: "I'm sorry I'm having trouble here. Maybe we should try this again later.",
+      suggestions: ['wayback machine google.com', 'wayback machine archive.org']
+    },
+
     welcome: {
       acknowledges: [
         'Welcome to music at the Internet Archive.'

functions/src/utils/traverse.js

@@ -0,0 +1,31 @@
+const _ = require('lodash');
+// const {debug} = require('../utils/logger')('ia:actions:utils:traverse');
+
+/**
+ * Traverse a given object
+ *
+ * @param {Object} obj
+ */
+module.exports = function (obj) {
+  let results = [];
+  function traverse (obj) {
+    _.forOwn(obj, (val, key) => {
+      if (_.isArray(val)) {
+        val.forEach(el => {
+          traverse(el);
+        });
+      } else if (_.isObject(val)) {
+        traverse(val);
+      } else {
+        results.push(val);
+      }
+    });
+  }
+  traverse(obj);
+  let count = 0;
+  while (results.length !== 0) {
+    count += results.pop();
+  }
+  // debug('final count inside traverse = ' + count);
+  return count;
+};

functions/tests/actions/wayback-machine.spec.js

@@ -0,0 +1,58 @@
+const {expect} = require('chai');
+const rewire = require('rewire');
+
+const action = rewire('../../src/actions/wayback-machine');
+
+const archiveRequest = require('../fixtures/wayback-archive.json');
+const alexaRequest = require('../fixtures/wayback-alexa.xml');
+const mockApp = require('../_utils/mocking/platforms/app');
+const mockDialog = require('../_utils/mocking/dialog');
+
+describe('actions', () => {
+  describe('wayback machine', () => {
+    let app;
+    let dialog;
+
+    beforeEach(() => {
+      app = mockApp();
+      dialog = mockDialog();
+      action.__set__('dialog', dialog);
+    });
+
+    it('check to see that overall action eventually returns a promise', () => {
+      action.handler(app);
+      expect(Promise.resolve()).to.be.a('promise');
+    });
+
+    it('check to see that archiveEngine is working', () => {
+      let obj = { earliestYear: 0, latestYear: 0, totalUniqueURLs: 0 };
+      let result;
+
+      action.__set__('archiveEngine', function (a, b) {
+        result = action.archiveEngine(archiveRequest, obj);
+        expect(result).to.change(obj, 'earliestYear');
+        expect(result).to.change(obj, 'latestYear');
+        expect(result).to.change(obj, 'totalUniqueURLs');
+      });
+    });
+
+    it('check to see that alexaEngine is working', () => {
+      let obj = { alexaWorldRank: 0 };
+      let result;
+
+      action.__set__('alexaEngine', function (a, b) {
+        result = action.alexaEngine(alexaRequest, obj);
+        expect(result).to.change(obj, 'alexaWorldRank');
+      });
+    });
+
+    it('check to see that xmlConverter is working', () => {
+      let result;
+
+      action.__set__('xmlConverter', function (a) {
+        result = action.xmlConverter(alexaRequest);
+        expect(result).to.not.throw('The XML parser didn\'t work.');
+      });
+    });
+  });
+});

functions/tests/fixtures/wayback-alexa.xml

@@ -0,0 +1 @@
+<!-- Need more Alexa data?  Find our APIs here: https://aws.amazon.com/alexa/ --><ALEXA VER="0.9" URL="cnn.com/" HOME="0" AID="=" IDN="cnn.com/"><SD><POPULARITY URL="cnn.com/" TEXT="104" SOURCE="panel"/><REACH RANK="88"/><RANK DELTA="-4"/><COUNTRY CODE="US" NAME="United States" RANK="27"/></SD></ALEXA>

functions/tests/utils/traverse.spec.js

@@ -0,0 +1,35 @@
+const {expect} = require('chai');
+const traverse = require('../../src/utils/traverse');
+
+let testJSON = {'captures': {
+  '1999': {
+    'text/html': 18360
+  },
+  '2000': {
+    'application/x-director': 19,
+    'video/quicktime': 1584,
+    'application/x-troff-man': 1,
+    'x-world/x-vrml': 1,
+    'audio/x-pn-realaudio': 176,
+    'audio/mpeg': 195,
+    'audio/x-wav': 3098,
+    'image/png': 97,
+    'text/html': 901401,
+    'video/x-ms-asf': 142,
+    'image/gif': 17388,
+    'text/plain': 394428,
+    'image/jpeg': 82903,
+    'application/x-shockwave-flash': 39,
+    'application/zip': 108,
+    'audio/x-aiff': 2767,
+    'text/css': 55,
+    'application/pdf': 291
+  }}};
+
+describe('utils', () => {
+  describe('traverse', () => {
+    it('should traverse a given object to return the sum of it\'s leaf nodes', () => {
+      expect(traverse(testJSON)).to.be.equal(1423053);
+    });
+  });
+});