Feature/wayback machine

functions/package.json

@@ -37,7 +37,7 @@
     "object.entries": "^1.0.4",
     "raven": "^2.6.0",
     "replaceall": "^0.1.6",
-    "supports-color": "^5.4.0"
+    "xml2js": "^0.4.19"
   },
   "devDependencies": {
     "axios-mock-adapter": "^1.15.0",

functions/src/actions/wayback-machine.js

@@ -0,0 +1,122 @@
+// Third party imports
+const axios = require('axios');
+const mustache = require('mustache');
+const xml2js = require('xml2js');
+
+// Local imports
+const config = require('../config');
+const {debug} = require('../utils/logger')('ia:actions:wayback-machine');
+const dialog = require('../dialog');
+const endpointProcessor = require('../network/endpoint-processor');
+const traverse = require('../utils/traverse');
+const waybackStrings = require('../strings').intents.wayback;
+
+/**
+ * Handle wayback query action
+ * - fill slots of wayback query
+ * - perform data requests to archive and alexa rankings
+ * - construct response speech for action
+ *
+ * @param app
+ */
+function handler (app) {
+  // Create wayback object
+  const waybackObject = {
+    url: '',
+    earliestYear: 0,
+    latestYear: 0,
+    totalUniqueURLs: 0,
+    alexaWorldRank: 0,
+    alexaUSRank: 0,
+    speech: waybackStrings.default,
+  };
+
+  // Check to see that both parameters have content
+  if (!app.params.getByName('wayback') && !app.params.getByName('url')) {
+    debug('wayback action called by mistake');
+    dialog.ask(app, waybackObject);
+  }
+
+  // Get url parameter and make url queries
+  waybackObject.url = app.params.getByName('url');
+  const archiveQueryURL = endpointProcessor.preprocess(
+    config.wayback.ARCHIVE, app, waybackObject
+  );
+  const alexaQueryURL = endpointProcessor.preprocess(
+    config.wayback.ALEXA, app, waybackObject
+  );
+
+  return Promise.all([axios.get(archiveQueryURL), axios.get(alexaQueryURL)])
+    .then(function (allData) {
+      // All data available here in the order it was called.
+
+      // Parse data from archive request
+      let archiveJSON = allData[0].data;
+      archiveEngine(archiveJSON, waybackObject);
+
+      // Parse data from alexa request
+      let XMLparser = new xml2js.Parser();
+      let convertXML = new Promise((resolve, reject) => {
+        XMLparser.parseString(allData[1].data, function (err, result) {
+          if (err) {
+            let error = new Error('The XML parser didn\'t work. Error message: ' + err);
+            reject(error);
+          } else {
+            resolve(result);
+          }
+        });
+      });
+      convertXML
+        .then(function (fulfilled) {
+          debug('XML parse successful!');
+          alexaEngine(JSON.parse(JSON.stringify(fulfilled)), waybackObject);
+        })
+        .catch(function (error) {
+          debug(error.message);
+          waybackObject.speech = waybackStrings.error;
+          dialog.ask(app, waybackObject);
+        });
+
+      // Construct response dialog for action
+      if (waybackObject.alexaUSRank !== 0) {
+        waybackObject.speech = mustache.render(waybackStrings.speech, waybackObject);
+        waybackObject.speech += mustache.render(waybackStrings.additionalSpeech, waybackObject);
+      } else {
+        waybackObject.speech = mustache.render(waybackStrings.speech, waybackObject);
+        waybackObject.speech += '.';
+      }
+
+      dialog.close(app, waybackObject);
+    });
+} // End of handler
+
+function archiveEngine (archiveJSON, waybackObject) {
+  // Create array of capture years and then find earliest year
+  //  and most recent year.
+  let yearsArray = Object.keys(archiveJSON.captures);
+  waybackObject.earliestYear = yearsArray[0];
+  waybackObject.latestYear = yearsArray[yearsArray.length - 1];
+
+  // Traverse URL category
+
+  // Find baseline of URL count
+  waybackObject.totalUniqueURLs += traverse(archiveJSON.urls[waybackObject.earliestYear]);
+  // debug('Baseline url count: ' + waybackObject.totalUniqueURLs);
+
+  waybackObject.totalUniqueURLs += traverse(archiveJSON.new_urls);
+  // debug('Final url count: ' + waybackObject.totalUniqueURLs);
+}
+
+function alexaEngine (alexaJSON, waybackObject) {
+  waybackObject.alexaWorldRank = alexaJSON['ALEXA']['SD'][0]['POPULARITY'][0]['$']['TEXT'];
+  try {
+    waybackObject.alexaUSRank = alexaJSON['ALEXA']['SD'][0]['COUNTRY'][0]['$']['RANK'];
+  } catch (e) {
+    debug('Country not found');
+    debug(e);
+  }
+}
+
+module.exports = {
+  handler,
+};

functions/src/config.js

@@ -31,6 +31,11 @@ module.exports = {
     DEFAULT_SONG_IMAGE: 'http://archive.org/images/notfound.png',
   },
 
+  wayback: {
+    ARCHIVE: 'http://web.archive.org/__wb/search/metadata?q={{url}}',
+    ALEXA: 'http://data.alexa.com/data?cli=10&url={{url}}',
+  },
+
   /**
    * settings specific for supported platforms
    */

functions/src/strings.js

@@ -444,6 +444,14 @@ module.exports = {
       speech: 'Version is {{version}}.',
     },
 
+    wayback: {
+      speech: '{{url}} was first captured by the Internet Archive in {{earliestYear}} and most recently in {{latestYear}}. The archive has {{totalUniqueURLs}} unique url\'s for this website. <break></break>{{url}} is ranked <say-as interpret-as="ordinal">{{alexaWorldRank}}</say-as> in the world in popularity',
+      additionalSpeech: ' and <say-as interpret-as="ordinal">{{alexaUSRank}}</say-as> in the United States.',
+      default: 'Would you like to use the wayback machine to hear the history of a website? Simply say Wayback Machine and the name of the website you\'d like to hear.',
+      error: "I'm sorry I'm having trouble here. Maybe we should try this again later.",
+      suggestions: ['wayback machine google.com', 'wayback machine archive.org']
+    },
+
     welcome: {
       acknowledges: [
         'Welcome to music at the Internet Archive.'

functions/src/utils/traverse.js

@@ -0,0 +1,31 @@
+const _ = require('lodash');
+const {debug} = require('../utils/logger')('ia:actions:utils:traverse');
+
+/**
+ * Traverse a given object
+ *
+ * @param {Object} obj
+ */
+module.exports = function (obj) {
+  let results = [];
+  function traverse (obj) {
+    _.forOwn(obj, (val, key) => {
+      if (_.isArray(val)) {
+        val.forEach(el => {
+          traverse(el);
+        });
+      } else if (_.isObject(val)) {
+        traverse(val);
+      } else {
+        results.push(val);
+      }
+    });
+  }
+  traverse(obj);
+  let count = 0;
+  while (results.length !== 0) {
+    count += results.pop();
+  }
+  debug('final count inside traverse = ' + count);
+  return count;
+};

functions/tests/actions/wayback-machine.spec.js

@@ -0,0 +1,25 @@
+const {expect} = require('chai');
+const rewire = require('rewire');
+
+const action = rewire('../../src/actions/wayback-machine');
+
+const mockApp = require('../_utils/mocking/platforms/app');
+const mockDialog = require('../_utils/mocking/dialog');
+
+describe('actions', () => {
+  describe('wayback machine', () => {
+    let app;
+    let dialog;
+
+    beforeEach(() => {
+      app = mockApp();
+      dialog = mockDialog();
+      action.__set__('dialog', dialog);
+    });
+
+    it('check to see that a promise is returned with network requests', () => {
+      action.handler(app);
+      expect(Promise.resolve()).to.be.a('promise');
+    });
+  });
+});

functions/tests/utils/traverse.spec.js

@@ -0,0 +1,35 @@
+const {expect} = require('chai');
+const traverse = require('../../src/utils/traverse');
+
+let testJSON = {'captures': {
+  '1999': {
+    'text/html': 18360
+  },
+  '2000': {
+    'application/x-director': 19,
+    'video/quicktime': 1584,
+    'application/x-troff-man': 1,
+    'x-world/x-vrml': 1,
+    'audio/x-pn-realaudio': 176,
+    'audio/mpeg': 195,
+    'audio/x-wav': 3098,
+    'image/png': 97,
+    'text/html': 901401,
+    'video/x-ms-asf': 142,
+    'image/gif': 17388,
+    'text/plain': 394428,
+    'image/jpeg': 82903,
+    'application/x-shockwave-flash': 39,
+    'application/zip': 108,
+    'audio/x-aiff': 2767,
+    'text/css': 55,
+    'application/pdf': 291
+  }}};
+
+describe('utils', () => {
+  describe('traverse', () => {
+    it('should traverse a given object to return the sum of it\'s leaf nodes', () => {
+      expect(traverse(testJSON)).to.be.equal(1423053);
+    });
+  });
+});