From ce1fb7b4e1523aaba541694bc96e9e0a26d21ee5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Tue, 3 Oct 2017 18:47:24 +0200
Subject: [PATCH 01/17] Start fleshing out the IPC system

---
 package.json            |  1 +
 src/commands/nettest.js |  9 ++++-----
 src/ipc/config.js       | 21 +++++++++++++++++++++
 src/ipc/start.js        | 19 +++++++++++++++++++
 src/ooni.js             | 21 ++++++++++++++++++++-
 5 files changed, 65 insertions(+), 6 deletions(-)
 create mode 100644 src/ipc/config.js
 create mode 100644 src/ipc/start.js

diff --git a/package.json b/package.json
index 38f89f7..c18a139 100644
--- a/package.json
+++ b/package.json
@@ -20,6 +20,7 @@
     "inquirer": "^3.3.0",
     "lodash.range": "^3.2.0",
     "mri": "^1.1.0",
+    "node-ipc": "^9.1.1",
     "ora": "^1.3.0",
     "shebang-loader": "^0.0.1",
     "string-length": "^2.0.0",
diff --git a/src/commands/nettest.js b/src/commands/nettest.js
index cf46842..cbf7b9d 100644
--- a/src/commands/nettest.js
+++ b/src/commands/nettest.js
@@ -69,7 +69,7 @@ class WebConnectivityRunner extends NettestRunner {
   get name() { return 'Web Connectivity' }
   get shortDescription() { return 'Test for web censorship' }
 
-  async run() {
+  async run(ctx) {
     let currentUrl
     if (argv.file) {
       // Handle testing input file
@@ -130,13 +130,13 @@ const help = () => {
   `)
 }
 
-const run = async ({nettest}) => {
+const run = async ({ctx, nettest}) => {
   if (nettest === undefined) {
     console.log(error(`Could not find nettest called ${name} (${camelName})`))
     return exit(1)
   }
   console.log(info(`Running ${chalk.bold(nettest.name)}`))
-  await nettest.run()
+  await nettest.run(ctx)
 }
 
 // Define these as module level variables so we don't have to pass them along
@@ -167,7 +167,7 @@ const main = async ctx => {
       nettest.help()
       await exit(0)
     } else {
-      await run({nettest})
+      await run({ctx, nettest})
     }
   } catch(err) {
     if (err.usageError) {
@@ -177,7 +177,6 @@ const main = async ctx => {
     }
     await exit(1)
   }
-
 }
 
 export default main
diff --git a/src/ipc/config.js b/src/ipc/config.js
new file mode 100644
index 0000000..00fca1b
--- /dev/null
+++ b/src/ipc/config.js
@@ -0,0 +1,21 @@
+import * as fs from 'fs-extra'
+import path from 'path'
+import { getOoniDir } from '../config/global-path'
+
+const configIpc = async (ipcId) => {
+  const ipc = require('node-ipc')
+  // XXX we should somewhere do some sanity checks on the permissions of the
+  // sockets directory to ensure the world cannot read and write to them.
+  const socketRoot = path.join(getOoniDir(), 'sockets')
+
+  await fs.ensureDir(socketRoot)
+
+  ipc.config.id = ipcId || ''+Date.now()
+  ipc.config.appspace = 'ooni.'
+  ipc.config.retry = 1500
+  ipc.config.socketRoot = socketRoot
+  // XXX check what this maxConnections does and ensure it's actually secure
+  ipc.config.maxConnections = 1
+  return ipc
+}
+export default configIpc
diff --git a/src/ipc/start.js b/src/ipc/start.js
new file mode 100644
index 0000000..fc3eb33
--- /dev/null
+++ b/src/ipc/start.js
@@ -0,0 +1,19 @@
+const startIpc = (ipc) => {
+  return new Promise((resolve, reject) => {
+    ipc.serveNet(() => {
+      ip.server.on(
+        'something',
+        (data, socket) => {
+          ipc.log('got message', data)
+          ipc.server.emit(
+            socket,
+            'message',
+            data + 'world'
+          )
+        })
+    })
+    ipc.server.start()
+    resolve()
+  })
+}
+export default startIpc
diff --git a/src/ooni.js b/src/ooni.js
index 17973a2..8cfab5e 100644
--- a/src/ooni.js
+++ b/src/ooni.js
@@ -17,6 +17,9 @@ import {
   writeToConfigFile
 } from './config/config-files'
 
+import configIpc from './ipc/config'
+import startIpc from './ipc/start'
+
 const OONI_DIR = getOoniDir()
 const OONI_CONFIG_PATH = getConfigFilePath()
 
@@ -115,8 +118,24 @@ const main = async (argv_) => {
     }
   }
 
+  let ipc
+  if (argv.ipc) {
+    try {
+      ipc = await configIpc(argv.ipc)
+      // XXX do we maybe want to have an CLI option to have the process wait
+      // until the consumer connects to the IPC subsystem?
+      await startIpc(ipc)
+    } catch(err) {
+      console.error(error('An error occurred while starting IPC subsystem ' +
+        `ooni config file in "${OONI_CONFIG_PATH}"` + err.message
+      ))
+      return 1
+    }
+  }
+
   const ctx = {
-    argv: argv_
+    argv: argv_,
+    ipc
   }
 
   if (subcommand === 'help') {

From b7e3ce3e5422d335a8366510f89802c16a948e72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Tue, 3 Oct 2017 18:47:38 +0200
Subject: [PATCH 02/17] [chore] update yarn.lock

---
 yarn.lock | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/yarn.lock b/yarn.lock
index fc8a8a1..d3d9441 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -983,6 +983,10 @@ domain-browser@^1.1.1:
   version "1.1.7"
   resolved "https://registry.yarnpkg.com/domain-browser/-/domain-browser-1.1.7.tgz#867aa4b093faa05f1de08c06f4d7b21fdf8698bc"
 
+easy-stack@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/easy-stack/-/easy-stack-1.0.0.tgz#12c91b3085a37f0baa336e9486eac4bf94e3e788"
+
 ecc-jsbn@~0.1.1:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz#0fc73a9ed5f0d53c38193398523ef7e543777505"
@@ -1113,6 +1117,10 @@ event-emitter@~0.3.5:
     d "1"
     es5-ext "~0.10.14"
 
+event-pubsub@4.3.0:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/event-pubsub/-/event-pubsub-4.3.0.tgz#f68d816bc29f1ec02c539dc58c8dd40ce72cb36e"
+
 events@^1.0.0:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
@@ -1630,6 +1638,16 @@ isstream@~0.1.2:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a"
 
+js-message@1.0.5:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/js-message/-/js-message-1.0.5.tgz#2300d24b1af08e89dd095bc1a4c9c9cfcb892d15"
+
+js-queue@2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/js-queue/-/js-queue-2.0.0.tgz#362213cf860f468f0125fc6c96abc1742531f948"
+  dependencies:
+    easy-stack "^1.0.0"
+
 js-tokens@^3.0.0, js-tokens@^3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-3.0.2.tgz#9866df395102130e38f7f996bceb65443209c25b"
@@ -1916,6 +1934,14 @@ node-dir@^0.1.10:
   dependencies:
     minimatch "^3.0.2"
 
+node-ipc@^9.1.1:
+  version "9.1.1"
+  resolved "https://registry.yarnpkg.com/node-ipc/-/node-ipc-9.1.1.tgz#4e245ed6938e65100e595ebc5dc34b16e8dd5d69"
+  dependencies:
+    event-pubsub "4.3.0"
+    js-message "1.0.5"
+    js-queue "2.0.0"
+
 node-libs-browser@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/node-libs-browser/-/node-libs-browser-2.0.0.tgz#a3a59ec97024985b46e958379646f96c4b616646"

From 3d3673719120398c5a9ea74583ca8f18253dc4c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 10 Jan 2018 11:48:11 +0100
Subject: [PATCH 03/17] Bump measurement-kit version

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 4959bd7..ad15cff 100644
--- a/package.json
+++ b/package.json
@@ -52,6 +52,6 @@
     "wrap-ansi": "^3.0.1"
   },
   "dependencies": {
-    "measurement-kit": "^0.1.0-alpha.5"
+    "measurement-kit": "^0.1.0-alpha.6"
   }
 }

From 425d1db64c8594963410081fee0bc26b10ad18eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 10 Jan 2018 11:50:29 +0100
Subject: [PATCH 04/17] Add code generator for nettests

---
 scripts/gen-nettests.py | 102 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 102 insertions(+)
 create mode 100644 scripts/gen-nettests.py

diff --git a/scripts/gen-nettests.py b/scripts/gen-nettests.py
new file mode 100644
index 0000000..7f53f6b
--- /dev/null
+++ b/scripts/gen-nettests.py
@@ -0,0 +1,102 @@
+import os
+import pystache
+# pip install https://github.com/okunishinishi/python-stringcase/archive/cc7d5eb58ff4a959a508c3f2296459daf7c3a1f2.zip
+import stringcase
+
+PKG_TMPL = """
+{
+  "name": "ooni-{{nettest_dash_case}}",
+  "version": "0.3.0",
+  "description": "Official OONI test for running {{nettest_name}}",
+  "dependencies": {
+    "measurement-kit": "0.1.0"
+  }
+}
+"""
+
+INDEX_TMPL = """
+import { {{nettest_pascal_case}} } from 'measurement-kit'
+
+export const renderSummary = (measurements, {React, Cli, Components, chalk}) => {
+  const summary = measurements[0].summary
+
+  // When this function is called from the Cli the Cli will be set, when it's
+  // called from the Desktop app we have React set instead.
+  if (Cli) {
+    Cli.log(Cli.output.labelValue('Label', uploadMbit, {unit: 'Mbit'}))
+  } else if (React) {
+    /*
+    XXX this is broken currently as it depends on react
+    const {
+      Container,
+      Heading
+    } = Components
+    return class extends React.Component {
+      render() {
+        return <Container>
+          <Heading h={1}>Results for NDT</Heading>
+          {uploadMbit}
+        </Container>
+      }
+    }
+    */
+  }
+}
+
+export const renderHelp = () => {
+}
+
+export const makeSummary = ({test_keys}) => ({
+})
+
+export const run = ({ooni, argv}) => {
+  const {{nettest_camel_case}} = {{nettest_pascal_case}}(ooni.mkOptions)
+  ooni.init({{nettest_camel_case}})
+
+  {{nettest_camel_case}}.on('begin', () => ooni.onProgress(0.0, 'starting {{nettest_dash_case}}'))
+  {{nettest_camel_case}}.on('progress', (percent, message) => {
+    ooni.onProgress(percent, message, persist)
+  })
+  return ooni.run({{nettest_camel_case}}.run)
+}
+"""
+
+nettests = [
+    ['Web Connectivity', 'web-connectivity'],
+    ['HTTP Invalid Request Line', 'http-invalid-request-line'],
+    ['HTTP Header Field Manipulation', 'http-header-field-manipulation'],
+    #['NDT', 'ndt'],
+    ['Dash', 'dash'],
+    ['Facebook Messenger', 'facebook-messenger'],
+    ['Telegram', 'telegram'],
+    ['WhatsApp', 'whatsapp']
+]
+
+def gen():
+    for nettest_name, nettest_dash_case in nettests:
+        nettest_camel_case = stringcase.camelcase(nettest_dash_case)
+        nettest_pascal_case = stringcase.pascalcase(nettest_dash_case)
+        partials = dict(
+            nettest_dash_case=nettest_dash_case,
+            nettest_camel_case=nettest_camel_case,
+            nettest_pascal_case=nettest_pascal_case,
+            nettest_name=nettest_name
+        )
+        dst_path = os.path.join(os.path.dirname(__file__), '..', 'src', 'nettests', nettest_dash_case)
+        if not os.path.exists(dst_path):
+            os.mkdir(dst_path)
+
+        index_path = os.path.join(dst_path, 'index.js')
+        package_path = os.path.join(dst_path, 'package.json')
+        with open(index_path, 'w+') as out_file:
+            out_file.write(pystache.render(INDEX_TMPL, partials))
+        print('wrote {}'.format(index_path))
+        with open(package_path, 'w+') as out_file:
+            out_file.write(pystache.render(PKG_TMPL, partials))
+        print('wrote {}'.format(package_path))
+
+def main():
+    gen()
+
+if __name__ == '__main__':
+    main()

From 4f18783e1a7f516056e4e52c704e57c13650b247 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 10 Jan 2018 11:50:40 +0100
Subject: [PATCH 05/17] Auto generate the various nettest plugins

---
 src/nettests/dash/index.js                    | 45 ++++++++++
 src/nettests/dash/package.json                |  9 ++
 src/nettests/facebook-messenger/index.js      | 45 ++++++++++
 src/nettests/facebook-messenger/package.json  |  9 ++
 .../http-header-field-manipulation/index.js   | 45 ++++++++++
 .../package.json                              |  9 ++
 .../http-invalid-request-line/index.js        | 45 ++++++++++
 .../http-invalid-request-line/package.json    |  9 ++
 src/nettests/telegram/index.js                | 45 ++++++++++
 src/nettests/telegram/package.json            |  9 ++
 src/nettests/web-connectivity/index.js        | 87 ++++++++-----------
 src/nettests/web-connectivity/package.json    |  1 +
 src/nettests/whatsapp/index.js                | 45 ++++++++++
 src/nettests/whatsapp/package.json            |  9 ++
 14 files changed, 362 insertions(+), 50 deletions(-)
 create mode 100644 src/nettests/dash/index.js
 create mode 100644 src/nettests/dash/package.json
 create mode 100644 src/nettests/facebook-messenger/index.js
 create mode 100644 src/nettests/facebook-messenger/package.json
 create mode 100644 src/nettests/http-header-field-manipulation/index.js
 create mode 100644 src/nettests/http-header-field-manipulation/package.json
 create mode 100644 src/nettests/http-invalid-request-line/index.js
 create mode 100644 src/nettests/http-invalid-request-line/package.json
 create mode 100644 src/nettests/telegram/index.js
 create mode 100644 src/nettests/telegram/package.json
 create mode 100644 src/nettests/whatsapp/index.js
 create mode 100644 src/nettests/whatsapp/package.json

diff --git a/src/nettests/dash/index.js b/src/nettests/dash/index.js
new file mode 100644
index 0000000..361b8f2
--- /dev/null
+++ b/src/nettests/dash/index.js
@@ -0,0 +1,45 @@
+
+import { Dash } from 'measurement-kit'
+
+export const renderSummary = (measurements, {React, Cli, Components, chalk}) => {
+  const summary = measurements[0].summary
+
+  // When this function is called from the Cli the Cli will be set, when it's
+  // called from the Desktop app we have React set instead.
+  if (Cli) {
+    Cli.log(Cli.output.labelValue('Label', uploadMbit, {unit: 'Mbit'}))
+  } else if (React) {
+    /*
+    XXX this is broken currently as it depends on react
+    const {
+      Container,
+      Heading
+    } = Components
+    return class extends React.Component {
+      render() {
+        return <Container>
+          <Heading h={1}>Results for NDT</Heading>
+          {uploadMbit}
+        </Container>
+      }
+    }
+    */
+  }
+}
+
+export const renderHelp = () => {
+}
+
+export const makeSummary = ({test_keys}) => ({
+})
+
+export const run = ({ooni, argv}) => {
+  const dash = Dash(ooni.mkOptions)
+  ooni.init(dash)
+
+  dash.on('begin', () => ooni.onProgress(0.0, 'starting dash'))
+  dash.on('progress', (percent, message) => {
+    ooni.onProgress(percent, message, persist)
+  })
+  return ooni.run(dash.run)
+}
diff --git a/src/nettests/dash/package.json b/src/nettests/dash/package.json
new file mode 100644
index 0000000..8a9e16a
--- /dev/null
+++ b/src/nettests/dash/package.json
@@ -0,0 +1,9 @@
+
+{
+  "name": "ooni-dash",
+  "version": "0.3.0",
+  "description": "Official OONI test for running Dash",
+  "dependencies": {
+    "measurement-kit": "0.1.0"
+  }
+}
diff --git a/src/nettests/facebook-messenger/index.js b/src/nettests/facebook-messenger/index.js
new file mode 100644
index 0000000..77a24fb
--- /dev/null
+++ b/src/nettests/facebook-messenger/index.js
@@ -0,0 +1,45 @@
+
+import { FacebookMessenger } from 'measurement-kit'
+
+export const renderSummary = (measurements, {React, Cli, Components, chalk}) => {
+  const summary = measurements[0].summary
+
+  // When this function is called from the Cli the Cli will be set, when it's
+  // called from the Desktop app we have React set instead.
+  if (Cli) {
+    Cli.log(Cli.output.labelValue('Label', uploadMbit, {unit: 'Mbit'}))
+  } else if (React) {
+    /*
+    XXX this is broken currently as it depends on react
+    const {
+      Container,
+      Heading
+    } = Components
+    return class extends React.Component {
+      render() {
+        return <Container>
+          <Heading h={1}>Results for NDT</Heading>
+          {uploadMbit}
+        </Container>
+      }
+    }
+    */
+  }
+}
+
+export const renderHelp = () => {
+}
+
+export const makeSummary = ({test_keys}) => ({
+})
+
+export const run = ({ooni, argv}) => {
+  const facebookMessenger = FacebookMessenger(ooni.mkOptions)
+  ooni.init(facebookMessenger)
+
+  facebookMessenger.on('begin', () => ooni.onProgress(0.0, 'starting facebook-messenger'))
+  facebookMessenger.on('progress', (percent, message) => {
+    ooni.onProgress(percent, message, persist)
+  })
+  return ooni.run(facebookMessenger.run)
+}
diff --git a/src/nettests/facebook-messenger/package.json b/src/nettests/facebook-messenger/package.json
new file mode 100644
index 0000000..b08bdf4
--- /dev/null
+++ b/src/nettests/facebook-messenger/package.json
@@ -0,0 +1,9 @@
+
+{
+  "name": "ooni-facebook-messenger",
+  "version": "0.3.0",
+  "description": "Official OONI test for running Facebook Messenger",
+  "dependencies": {
+    "measurement-kit": "0.1.0"
+  }
+}
diff --git a/src/nettests/http-header-field-manipulation/index.js b/src/nettests/http-header-field-manipulation/index.js
new file mode 100644
index 0000000..5efa6f0
--- /dev/null
+++ b/src/nettests/http-header-field-manipulation/index.js
@@ -0,0 +1,45 @@
+
+import { HttpHeaderFieldManipulation } from 'measurement-kit'
+
+export const renderSummary = (measurements, {React, Cli, Components, chalk}) => {
+  const summary = measurements[0].summary
+
+  // When this function is called from the Cli the Cli will be set, when it's
+  // called from the Desktop app we have React set instead.
+  if (Cli) {
+    Cli.log(Cli.output.labelValue('Label', uploadMbit, {unit: 'Mbit'}))
+  } else if (React) {
+    /*
+    XXX this is broken currently as it depends on react
+    const {
+      Container,
+      Heading
+    } = Components
+    return class extends React.Component {
+      render() {
+        return <Container>
+          <Heading h={1}>Results for NDT</Heading>
+          {uploadMbit}
+        </Container>
+      }
+    }
+    */
+  }
+}
+
+export const renderHelp = () => {
+}
+
+export const makeSummary = ({test_keys}) => ({
+})
+
+export const run = ({ooni, argv}) => {
+  const httpHeaderFieldManipulation = HttpHeaderFieldManipulation(ooni.mkOptions)
+  ooni.init(httpHeaderFieldManipulation)
+
+  httpHeaderFieldManipulation.on('begin', () => ooni.onProgress(0.0, 'starting http-header-field-manipulation'))
+  httpHeaderFieldManipulation.on('progress', (percent, message) => {
+    ooni.onProgress(percent, message, persist)
+  })
+  return ooni.run(httpHeaderFieldManipulation.run)
+}
diff --git a/src/nettests/http-header-field-manipulation/package.json b/src/nettests/http-header-field-manipulation/package.json
new file mode 100644
index 0000000..b4ba4d9
--- /dev/null
+++ b/src/nettests/http-header-field-manipulation/package.json
@@ -0,0 +1,9 @@
+
+{
+  "name": "ooni-http-header-field-manipulation",
+  "version": "0.3.0",
+  "description": "Official OONI test for running HTTP Header Field Manipulation",
+  "dependencies": {
+    "measurement-kit": "0.1.0"
+  }
+}
diff --git a/src/nettests/http-invalid-request-line/index.js b/src/nettests/http-invalid-request-line/index.js
new file mode 100644
index 0000000..8681adc
--- /dev/null
+++ b/src/nettests/http-invalid-request-line/index.js
@@ -0,0 +1,45 @@
+
+import { HttpInvalidRequestLine } from 'measurement-kit'
+
+export const renderSummary = (measurements, {React, Cli, Components, chalk}) => {
+  const summary = measurements[0].summary
+
+  // When this function is called from the Cli the Cli will be set, when it's
+  // called from the Desktop app we have React set instead.
+  if (Cli) {
+    Cli.log(Cli.output.labelValue('Label', uploadMbit, {unit: 'Mbit'}))
+  } else if (React) {
+    /*
+    XXX this is broken currently as it depends on react
+    const {
+      Container,
+      Heading
+    } = Components
+    return class extends React.Component {
+      render() {
+        return <Container>
+          <Heading h={1}>Results for NDT</Heading>
+          {uploadMbit}
+        </Container>
+      }
+    }
+    */
+  }
+}
+
+export const renderHelp = () => {
+}
+
+export const makeSummary = ({test_keys}) => ({
+})
+
+export const run = ({ooni, argv}) => {
+  const httpInvalidRequestLine = HttpInvalidRequestLine(ooni.mkOptions)
+  ooni.init(httpInvalidRequestLine)
+
+  httpInvalidRequestLine.on('begin', () => ooni.onProgress(0.0, 'starting http-invalid-request-line'))
+  httpInvalidRequestLine.on('progress', (percent, message) => {
+    ooni.onProgress(percent, message, persist)
+  })
+  return ooni.run(httpInvalidRequestLine.run)
+}
diff --git a/src/nettests/http-invalid-request-line/package.json b/src/nettests/http-invalid-request-line/package.json
new file mode 100644
index 0000000..890b504
--- /dev/null
+++ b/src/nettests/http-invalid-request-line/package.json
@@ -0,0 +1,9 @@
+
+{
+  "name": "ooni-http-invalid-request-line",
+  "version": "0.3.0",
+  "description": "Official OONI test for running HTTP Invalid Request Line",
+  "dependencies": {
+    "measurement-kit": "0.1.0"
+  }
+}
diff --git a/src/nettests/telegram/index.js b/src/nettests/telegram/index.js
new file mode 100644
index 0000000..b19d7b4
--- /dev/null
+++ b/src/nettests/telegram/index.js
@@ -0,0 +1,45 @@
+
+import { Telegram } from 'measurement-kit'
+
+export const renderSummary = (measurements, {React, Cli, Components, chalk}) => {
+  const summary = measurements[0].summary
+
+  // When this function is called from the Cli the Cli will be set, when it's
+  // called from the Desktop app we have React set instead.
+  if (Cli) {
+    Cli.log(Cli.output.labelValue('Label', uploadMbit, {unit: 'Mbit'}))
+  } else if (React) {
+    /*
+    XXX this is broken currently as it depends on react
+    const {
+      Container,
+      Heading
+    } = Components
+    return class extends React.Component {
+      render() {
+        return <Container>
+          <Heading h={1}>Results for NDT</Heading>
+          {uploadMbit}
+        </Container>
+      }
+    }
+    */
+  }
+}
+
+export const renderHelp = () => {
+}
+
+export const makeSummary = ({test_keys}) => ({
+})
+
+export const run = ({ooni, argv}) => {
+  const telegram = Telegram(ooni.mkOptions)
+  ooni.init(telegram)
+
+  telegram.on('begin', () => ooni.onProgress(0.0, 'starting telegram'))
+  telegram.on('progress', (percent, message) => {
+    ooni.onProgress(percent, message, persist)
+  })
+  return ooni.run(telegram.run)
+}
diff --git a/src/nettests/telegram/package.json b/src/nettests/telegram/package.json
new file mode 100644
index 0000000..188a96b
--- /dev/null
+++ b/src/nettests/telegram/package.json
@@ -0,0 +1,9 @@
+
+{
+  "name": "ooni-telegram",
+  "version": "0.3.0",
+  "description": "Official OONI test for running Telegram",
+  "dependencies": {
+    "measurement-kit": "0.1.0"
+  }
+}
diff --git a/src/nettests/web-connectivity/index.js b/src/nettests/web-connectivity/index.js
index 39c5b59..01d6ed4 100644
--- a/src/nettests/web-connectivity/index.js
+++ b/src/nettests/web-connectivity/index.js
@@ -1,58 +1,45 @@
-import chalk from 'chalk'
 
-import exit from '../../util/exit'
-import wait from '../../cli/output/wait'
-import ok from '../../cli/output/ok'
-import notok from '../../cli/output/notok'
-import error from '../../cli/output/error'
-
-import nettestHelp from '../../cli/output/nettest-help'
-import rightPad from '../../cli/output/right-pad'
-import sleep from '../../util/sleep'
-
-exports.renderRunSummary = (measurements, {React, Cli, Components, chalk}) => {
-}
-
-exports.makeSummary = (test_keys) => {
+import { WebConnectivity } from 'measurement-kit'
+
+export const renderSummary = (measurements, {React, Cli, Components, chalk}) => {
+  const summary = measurements[0].summary
+
+  // When this function is called from the Cli the Cli will be set, when it's
+  // called from the Desktop app we have React set instead.
+  if (Cli) {
+    Cli.log(Cli.output.labelValue('Label', uploadMbit, {unit: 'Mbit'}))
+  } else if (React) {
+    /*
+    XXX this is broken currently as it depends on react
+    const {
+      Container,
+      Heading
+    } = Components
+    return class extends React.Component {
+      render() {
+        return <Container>
+          <Heading h={1}>Results for NDT</Heading>
+          {uploadMbit}
+        </Container>
+      }
+    }
+    */
+  }
 }
 
-exports.renderHelp = () => {
-  const meta = {
-    name: WebConnectivity.name,
-    shortDescription: WebConnectivity.shortDescription
-  }
-  return nettestHelp(meta, 'webConnectivity', [
-    {
-      option: '-h, --help',
-      description: 'Display usage information'
-    },
-    {
-      option: `-f, --file ${chalk.bold.underline('FILE')}`,
-      description: 'The path to a list of websites to test'
-    }
-  ])
+export const renderHelp = () => {
 }
 
-exports.run = async ({ooni, argv}) => {
-  let currentUrl
-  if (argv.file) {
-    // Handle testing input file
-  } else {
-    currentUrl = argv._.slice(1)
-    if (currentUrl.length == 0) {
-      console.log(
-        error(
-          'Your must specify either a URL or an input file'
-        )
-      )
-      return exit(1)
-    }
-  }
+export const makeSummary = ({test_keys}) => ({
+})
 
-  const testDone = wait(`testing ${currentUrl}`)
-  await sleep(10000)
-  testDone()
+export const run = ({ooni, argv}) => {
+  const webConnectivity = WebConnectivity(ooni.mkOptions)
+  ooni.init(webConnectivity)
 
-  console.log(ok(`${currentUrl} is OK`))
-  console.log(notok(`${currentUrl} is NOT OK`))
+  webConnectivity.on('begin', () => ooni.onProgress(0.0, 'starting web-connectivity'))
+  webConnectivity.on('progress', (percent, message) => {
+    ooni.onProgress(percent, message, persist)
+  })
+  return ooni.run(webConnectivity.run)
 }
diff --git a/src/nettests/web-connectivity/package.json b/src/nettests/web-connectivity/package.json
index 85f6bd0..78fec1f 100644
--- a/src/nettests/web-connectivity/package.json
+++ b/src/nettests/web-connectivity/package.json
@@ -1,3 +1,4 @@
+
 {
   "name": "ooni-web-connectivity",
   "version": "0.3.0",
diff --git a/src/nettests/whatsapp/index.js b/src/nettests/whatsapp/index.js
new file mode 100644
index 0000000..c92c8fe
--- /dev/null
+++ b/src/nettests/whatsapp/index.js
@@ -0,0 +1,45 @@
+
+import { Whatsapp } from 'measurement-kit'
+
+export const renderSummary = (measurements, {React, Cli, Components, chalk}) => {
+  const summary = measurements[0].summary
+
+  // When this function is called from the Cli the Cli will be set, when it's
+  // called from the Desktop app we have React set instead.
+  if (Cli) {
+    Cli.log(Cli.output.labelValue('Label', uploadMbit, {unit: 'Mbit'}))
+  } else if (React) {
+    /*
+    XXX this is broken currently as it depends on react
+    const {
+      Container,
+      Heading
+    } = Components
+    return class extends React.Component {
+      render() {
+        return <Container>
+          <Heading h={1}>Results for NDT</Heading>
+          {uploadMbit}
+        </Container>
+      }
+    }
+    */
+  }
+}
+
+export const renderHelp = () => {
+}
+
+export const makeSummary = ({test_keys}) => ({
+})
+
+export const run = ({ooni, argv}) => {
+  const whatsapp = Whatsapp(ooni.mkOptions)
+  ooni.init(whatsapp)
+
+  whatsapp.on('begin', () => ooni.onProgress(0.0, 'starting whatsapp'))
+  whatsapp.on('progress', (percent, message) => {
+    ooni.onProgress(percent, message, persist)
+  })
+  return ooni.run(whatsapp.run)
+}
diff --git a/src/nettests/whatsapp/package.json b/src/nettests/whatsapp/package.json
new file mode 100644
index 0000000..1361c94
--- /dev/null
+++ b/src/nettests/whatsapp/package.json
@@ -0,0 +1,9 @@
+
+{
+  "name": "ooni-whatsapp",
+  "version": "0.3.0",
+  "description": "Official OONI test for running WhatsApp",
+  "dependencies": {
+    "measurement-kit": "0.1.0"
+  }
+}

From a54bdeeddb688a5bd70795b156b22e62d8e17f04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 10 Jan 2018 13:12:11 +0100
Subject: [PATCH 06/17] Add functions to download geoip data

---
 package.json        |  4 ++-
 src/config/geoip.js | 69 +++++++++++++++++++++++++++++++++++++++++++++
 yarn.lock           | 23 +++++++++++++--
 3 files changed, 92 insertions(+), 4 deletions(-)
 create mode 100644 src/config/geoip.js

diff --git a/package.json b/package.json
index ad15cff..7149b26 100644
--- a/package.json
+++ b/package.json
@@ -49,9 +49,11 @@
     "webpack": "^3.6.0",
     "webpack-node-externals": "^1.6.0",
     "window-size": "^1.1.0",
-    "wrap-ansi": "^3.0.1"
+    "wrap-ansi": "^3.0.1",
+    "zlib": "^1.0.5"
   },
   "dependencies": {
+    "axios": "^0.17.1",
     "measurement-kit": "^0.1.0-alpha.6"
   }
 }
diff --git a/src/config/geoip.js b/src/config/geoip.js
new file mode 100644
index 0000000..fb4cf75
--- /dev/null
+++ b/src/config/geoip.js
@@ -0,0 +1,69 @@
+import * as fs from 'fs-extra'
+import path from 'path'
+
+import axios from 'axios'
+import zlib from 'zlib'
+
+import { getOoniDir } from './global-path'
+
+const BASE_URL = 'https://github.com/OpenObservatory/ooni-resources/releases/download/'
+const LATEST_VERSION = '21'
+const GEOIP_ASN_FILENAME = 'maxmind-geoip.GeoIPASNum.dat'
+const GEOIP_COUNTRY_FILENAME = 'maxmind-geoip.GeoIP.dat'
+
+const downloadFile = ({url, dst, uncompress}) => {
+  return new Promise((resolve, reject) => {
+    axios({
+      method: 'get',
+      url: url,
+      responseType: 'stream'
+    }).then(response => {
+      let outputStream
+      if (uncompress) {
+        outputStream = zlib.createGunzip()
+        outputStream.pipe(fs.createWriteStream(dst))
+        response.data.pipe(outputStream)
+      } else {
+        outputStream = response.data
+        outputStream.pipe(fs.createWriteStream(dst))
+      }
+      outputStream.on('end', () => {
+        resolve()
+      })
+    })
+  })
+}
+
+export const getGeoipPaths = async () => {
+  const geoipDir = path.join(getOoniDir(), 'geoip')
+
+  // XXX exception handling
+  const geoipDirExists = await fs.pathExists(geoipDir)
+  if (!geoipDirExists) {
+    await fs.ensureDir(geoipDir)
+  }
+
+  const geoipCountryPath = path.join(geoipDir, GEOIP_COUNTRY_FILENAME)
+  const geoipCountryExists = await fs.pathExists(geoipCountryPath)
+  if (!geoipCountryExists) {
+    await downloadFile({
+      url: `${BASE_URL}${LATEST_VERSION}/${GEOIP_COUNTRY_FILENAME}.gz`,
+      dst: geoipCountryPath,
+      uncompress: true
+    })
+  }
+
+  const geoipAsnPath = path.join(geoipDir, GEOIP_ASN_FILENAME)
+  const geoipAsnExists = await fs.pathExists(geoipAsnPath)
+  if (!geoipAsnExists) {
+    await downloadFile({
+      url: `${BASE_URL}${LATEST_VERSION}/${GEOIP_ASN_FILENAME}.gz`,
+      dst: geoipAsnPath,
+      uncompress: true
+    })
+  }
+  return {
+    countryPath: geoipCountryPath,
+    asnPath: geoipAsnPath,
+  }
+}
diff --git a/yarn.lock b/yarn.lock
index a858d31..f10bbc2 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -197,6 +197,13 @@ aws4@^1.2.1, aws4@^1.6.0:
   version "1.6.0"
   resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e"
 
+axios@^0.17.1:
+  version "0.17.1"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-0.17.1.tgz#2d8e3e5d0bdbd7327f91bc814f5c57660f81824d"
+  dependencies:
+    follow-redirects "^1.2.5"
+    is-buffer "^1.1.5"
+
 babel-code-frame@^6.26.0:
   version "6.26.0"
   resolved "https://registry.yarnpkg.com/babel-code-frame/-/babel-code-frame-6.26.0.tgz#63fd43f7dc1e3bb7ce35947db8fe369a3f58c74b"
@@ -1389,6 +1396,12 @@ flow-babel-webpack-plugin@^1.1.0:
   version "0.56.0"
   resolved "https://registry.yarnpkg.com/flow-bin/-/flow-bin-0.56.0.tgz#ce43092203a344ba9bf63c0cabe95d95145f6cad"
 
+follow-redirects@^1.2.5:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.3.0.tgz#f684871fc116d2e329fda55ef67687f4fabc905c"
+  dependencies:
+    debug "^3.1.0"
+
 for-in@^1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80"
@@ -2102,9 +2115,9 @@ md5.js@^1.3.4:
     hash-base "^3.0.0"
     inherits "^2.0.1"
 
-measurement-kit@^0.1.0-alpha.5:
-  version "0.1.0-alpha.5"
-  resolved "https://registry.yarnpkg.com/measurement-kit/-/measurement-kit-0.1.0-alpha.5.tgz#f0be2b3bb2e5dcf0aea40265837376a63ff79297"
+measurement-kit@^0.1.0-alpha.6:
+  version "0.1.0-alpha.6"
+  resolved "https://registry.yarnpkg.com/measurement-kit/-/measurement-kit-0.1.0-alpha.6.tgz#9c92534877ed6204d30435377169b72d666d7f60"
   dependencies:
     any-promise "^1.3.0"
     bindings "^1.3.0"
@@ -3487,3 +3500,7 @@ yargs@~3.10.0:
     cliui "^2.1.0"
     decamelize "^1.0.0"
     window-size "0.1.0"
+
+zlib@^1.0.5:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/zlib/-/zlib-1.0.5.tgz#6e7c972fc371c645a6afb03ab14769def114fcc0"

From a7884145e0b75ac3d6367cb0ca0ca2f5bd58b94c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 10 Jan 2018 13:12:21 +0100
Subject: [PATCH 07/17] Implement facebook and telegram tests

---
 src/cli/make-cli.js                      |  2 +
 src/commands/run.js                      |  6 ++-
 src/nettests/facebook-messenger/index.js | 54 ++++++++++++++++--------
 src/nettests/index.js                    | 27 ++++++++----
 src/nettests/telegram/index.js           | 40 ++++++++++--------
 src/nettests/whatsapp/index.js           | 39 +++++++++--------
 6 files changed, 107 insertions(+), 61 deletions(-)

diff --git a/src/cli/make-cli.js b/src/cli/make-cli.js
index ffc1d90..e56a038 100644
--- a/src/cli/make-cli.js
+++ b/src/cli/make-cli.js
@@ -2,6 +2,8 @@ export const makeCli = (log = console.log) => ({
   output: {
     toMbit: require('./output/to-mbit').default,
     labelValue: require('./output/label-value').default,
+    ok: require('./output/ok'),
+    notok: require('./output/notok'),
   },
   log
 })
diff --git a/src/commands/run.js b/src/commands/run.js
index 018fc3f..22b8bdb 100644
--- a/src/commands/run.js
+++ b/src/commands/run.js
@@ -20,6 +20,8 @@ import {
 
 import makeCli from '../cli/make-cli'
 
+import { getGeoipPaths } from '../config/geoip'
+
 const debug = require('debug')('commands.run')
 
 const help = () => {
@@ -70,11 +72,13 @@ const run = async ({camelName, argv}) => {
               chalk.bold(`${nettestType.nettests.length} ${nettestType.name} `) +
               `test${sOrNot}`))
 
+  const geoip = await getGeoipPaths()
+
   for (const nettestLoader of nettestType.nettests) {
     const loader = nettestLoader()
     const { nettest, meta } = loader
     console.log(info(`${chalk.bold(meta.name)}`))
-    const measurements = await nettest.run({ooni: makeOoni(loader), argv})
+    const measurements = await nettest.run({ooni: makeOoni(loader, geoip), argv})
     nettest.renderSummary(measurements, {
       Cli: makeCli(),
       chalk: chalk,
diff --git a/src/nettests/facebook-messenger/index.js b/src/nettests/facebook-messenger/index.js
index 77a24fb..c00acb8 100644
--- a/src/nettests/facebook-messenger/index.js
+++ b/src/nettests/facebook-messenger/index.js
@@ -7,23 +7,18 @@ export const renderSummary = (measurements, {React, Cli, Components, chalk}) =>
   // When this function is called from the Cli the Cli will be set, when it's
   // called from the Desktop app we have React set instead.
   if (Cli) {
-    Cli.log(Cli.output.labelValue('Label', uploadMbit, {unit: 'Mbit'}))
-  } else if (React) {
-    /*
-    XXX this is broken currently as it depends on react
-    const {
-      Container,
-      Heading
-    } = Components
-    return class extends React.Component {
-      render() {
-        return <Container>
-          <Heading h={1}>Results for NDT</Heading>
-          {uploadMbit}
-        </Container>
-      }
+    if (summary.facebookDnsBlocking === true) {
+      Cli.log(Cli.output.notok('Facebook is blocked via DNS'))
+    } else {
+      Cli.log(Cli.output.ok('Facebook is not blocked via DNS'))
+    }
+    if (summary.facebookTcpBlocking === true) {
+      Cli.log(Cli.output.notok('Facebook is blocked via TCP'))
+    } else {
+      Cli.log(Cli.output.ok('Facebook is not blocked via TCP'))
     }
-    */
+  } else if (React) {
+    // XXX this is broken currently as it depends on react
   }
 }
 
@@ -31,6 +26,31 @@ export const renderHelp = () => {
 }
 
 export const makeSummary = ({test_keys}) => ({
+  facebookTcpBlocking: test_keys.facebook_tcp_blocking,
+  facebookDnsBlocking: test_keys.facebook_dns_blocking,
+  /*
+  XXX do we want to expose these too?
+  facebook_b_api_dns_consistent
+  facebook_b_api_reachable
+
+  facebook_b_graph_dns_consistent
+  facebook_b_graph_reachable
+
+  facebook_edge_dns_consistent
+  facebook_edge_reachable
+
+  facebook_external_cdn_dns_consistent
+  facebook_external_cdn_reachable
+
+  facebook_scontent_cdn_dns_consistent
+  facebook_scontent_cdn_reachable
+
+  facebook_star_dns_consistent
+  facebook_star_reachable
+
+  facebook_stun_dns_consistent
+  facebook_stun_reachable
+  */
 })
 
 export const run = ({ooni, argv}) => {
@@ -39,7 +59,7 @@ export const run = ({ooni, argv}) => {
 
   facebookMessenger.on('begin', () => ooni.onProgress(0.0, 'starting facebook-messenger'))
   facebookMessenger.on('progress', (percent, message) => {
-    ooni.onProgress(percent, message, persist)
+    ooni.onProgress(percent, message)
   })
   return ooni.run(facebookMessenger.run)
 }
diff --git a/src/nettests/index.js b/src/nettests/index.js
index 296bf0a..ed0c389 100644
--- a/src/nettests/index.js
+++ b/src/nettests/index.js
@@ -24,7 +24,7 @@ const makeReportFile = (name) => {
   )
 }
 
-export const makeOoni = (loader) => {
+export const makeOoni = (loader, geoip) => {
   let dbOperations = [],
       measurements = [],
       mkOptions = {},
@@ -50,6 +50,10 @@ export const makeOoni = (loader) => {
     if (isMk) {
       nt.test.set_options('no_file_report', '0')
       nt.test.set_output_filepath(reportFile)
+      nt.setOptions({
+        geoipCountryPath: geoip.countryPath,
+        geoipAsnPath: geoip.asnPath
+      })
       nt.on('log', (severity, message) => {
         debug('<'+severity+'>'+message)
         // XXX this a workaround due to a bug in MK
@@ -148,14 +152,12 @@ export const nettests = {
   httpInvalidRequestLine: makeNettestLoader('http-invalid-request-line'),
   httpHeaderFieldManipulation: makeNettestLoader('http-header-field-manipulation'),
   ndt: makeNettestLoader('ndt'),
-
-  // Missing wrapper
   dash: makeNettestLoader('dash'),
   facebookMessenger: makeNettestLoader('facebook-messenger'),
   telegram: makeNettestLoader('telegram'),
+  whatsapp: makeNettestLoader('whatsapp'),
 
   // These don't exist in MK
-  whatsapp: makeNettestLoader('whatsapp'),
   captivePortal: makeNettestLoader('captive-portal'),
   httpHost: makeNettestLoader('http-host'),
   traceroute: makeNettestLoader('traceroute'),
@@ -217,11 +219,13 @@ export const nettestTypes = {
       nettests.httpInvalidRequestLine,
       nettests.httpHeaderFieldManipulation
     ],
-    name: 'Middleboxes',
-    shortDescription: 'Detect the presence of "Middle boxes"',
+    name: 'Middle Boxes',
+    shortDescription: 'Detect the presence of Middle Boxes',
     help: 'No help for you',
     makeSummary: (measurements) => {
-      return {}
+      return {
+        foundMiddlebox: true
+      }
     },
     renderSummary: (measurement, {Cli, chalk}) => {}
   },
@@ -235,9 +239,14 @@ export const nettestTypes = {
     shortDescription: 'Check if Instant Messagging apps are blocked.',
     help: 'No help for you',
     makeSummary: (measurements) => {
-      return {}
+      return {
+        facebookMessengerBlocked: true,
+        whatsappBlocked: true,
+        telegramBlocked: true,
+      }
     },
-    renderSummary: (measurement, {Cli, chalk}) => {}
+    renderSummary: (measurement, {Cli, chalk}) => {
+    }
   },
   circumvention: {
     nettests: [
diff --git a/src/nettests/telegram/index.js b/src/nettests/telegram/index.js
index b19d7b4..f110854 100644
--- a/src/nettests/telegram/index.js
+++ b/src/nettests/telegram/index.js
@@ -7,23 +7,26 @@ export const renderSummary = (measurements, {React, Cli, Components, chalk}) =>
   // When this function is called from the Cli the Cli will be set, when it's
   // called from the Desktop app we have React set instead.
   if (Cli) {
-    Cli.log(Cli.output.labelValue('Label', uploadMbit, {unit: 'Mbit'}))
-  } else if (React) {
-    /*
-    XXX this is broken currently as it depends on react
-    const {
-      Container,
-      Heading
-    } = Components
-    return class extends React.Component {
-      render() {
-        return <Container>
-          <Heading h={1}>Results for NDT</Heading>
-          {uploadMbit}
-        </Container>
-      }
+
+    if (summary.telegramHttpBlocked === true) {
+      Cli.log(Cli.output.notok('Telegram via HTTP is blocked'))
+    } else {
+      Cli.log(Cli.output.ok('Telegram via HTTP is not blocked'))
+    }
+
+    if (summary.telegramTcpBlocked === true) {
+      Cli.log(Cli.output.notok('Telegram via TCP is blocked'))
+    } else {
+      Cli.log(Cli.output.ok('Telegram via TCP is not blocked'))
     }
-    */
+
+    if (summary.telegramWebBlocked === true) {
+      Cli.log(Cli.output.notok('Telegram Web is blocked'))
+    } else {
+      Cli.log(Cli.output.ok('Telegram Web is not blocked'))
+    }
+  } else if (React) {
+    // XXX this is broken currently as it depends on react
   }
 }
 
@@ -31,6 +34,9 @@ export const renderHelp = () => {
 }
 
 export const makeSummary = ({test_keys}) => ({
+  telegramWebBlocked: test_keys.telegram_web_status === 'blocked',
+  telegramHttpBlocked: test_keys.telegram_http_blocking,
+  telegramTcpBlocked: test_keys.telegram_tcp_blocking,
 })
 
 export const run = ({ooni, argv}) => {
@@ -39,7 +45,7 @@ export const run = ({ooni, argv}) => {
 
   telegram.on('begin', () => ooni.onProgress(0.0, 'starting telegram'))
   telegram.on('progress', (percent, message) => {
-    ooni.onProgress(percent, message, persist)
+    ooni.onProgress(percent, message)
   })
   return ooni.run(telegram.run)
 }
diff --git a/src/nettests/whatsapp/index.js b/src/nettests/whatsapp/index.js
index c92c8fe..a16c90d 100644
--- a/src/nettests/whatsapp/index.js
+++ b/src/nettests/whatsapp/index.js
@@ -7,23 +7,25 @@ export const renderSummary = (measurements, {React, Cli, Components, chalk}) =>
   // When this function is called from the Cli the Cli will be set, when it's
   // called from the Desktop app we have React set instead.
   if (Cli) {
-    Cli.log(Cli.output.labelValue('Label', uploadMbit, {unit: 'Mbit'}))
-  } else if (React) {
-    /*
-    XXX this is broken currently as it depends on react
-    const {
-      Container,
-      Heading
-    } = Components
-    return class extends React.Component {
-      render() {
-        return <Container>
-          <Heading h={1}>Results for NDT</Heading>
-          {uploadMbit}
-        </Container>
-      }
+    if (summary.whatsappEndpointsBlocked === true) {
+      Cli.log(Cli.output.notok('Whatapp is blocked'))
+    } else {
+      Cli.log(Cli.output.ok('Whatapp is not blocked'))
+    }
+
+    if (summary.whatsappWebBlocked === true) {
+      Cli.log(Cli.output.notok('Whatapp Web is blocked'))
+    } else {
+      Cli.log(Cli.output.ok('Whatapp Web is not blocked'))
     }
-    */
+
+    if (summary.registrationServerBlocked === true) {
+      Cli.log(Cli.output.notok('Whatapp registration server is blocked'))
+    } else {
+      Cli.log(Cli.output.ok('Whatapp registration server is not blocked'))
+    }
+  } else if (React) {
+    // XXX this is broken currently as it depends on react
   }
 }
 
@@ -31,6 +33,9 @@ export const renderHelp = () => {
 }
 
 export const makeSummary = ({test_keys}) => ({
+  whatsappEndpointsBlocked: test_keys.whatsapp_endpoints_status === 'blocked',
+  whatsappWebBlocked: test_keys.whatsapp_web_status === 'blocked',
+  registrationServerBlocked: test_keys.registration_server_status === 'blocked'
 })
 
 export const run = ({ooni, argv}) => {
@@ -39,7 +44,7 @@ export const run = ({ooni, argv}) => {
 
   whatsapp.on('begin', () => ooni.onProgress(0.0, 'starting whatsapp'))
   whatsapp.on('progress', (percent, message) => {
-    ooni.onProgress(percent, message, persist)
+    ooni.onProgress(percent, message)
   })
   return ooni.run(whatsapp.run)
 }

From 247459b8c584b9e53e0441c3f4d1e923ecfafa1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 10 Jan 2018 13:39:34 +0100
Subject: [PATCH 08/17] Fix a bug in handling of the summary generation

---
 src/commands/run.js | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/commands/run.js b/src/commands/run.js
index 22b8bdb..e8a20bb 100644
--- a/src/commands/run.js
+++ b/src/commands/run.js
@@ -78,18 +78,20 @@ const run = async ({camelName, argv}) => {
     const loader = nettestLoader()
     const { nettest, meta } = loader
     console.log(info(`${chalk.bold(meta.name)}`))
-    const measurements = await nettest.run({ooni: makeOoni(loader, geoip), argv})
-    nettest.renderSummary(measurements, {
+    const measurement = await nettest.run({ooni: makeOoni(loader, geoip), argv})
+    nettest.renderSummary(measurement, {
       Cli: makeCli(),
       chalk: chalk,
       Components: null,
       React: null,
     })
-    dbOperations.push(result.setMeasurements(measurements))
+    dbOperations.push(result.addMeasurements(measurement))
   }
   await Promise.all(dbOperations)
 
   const measurements = await result.getMeasurements()
+  debug('updating the result table')
+  debug(measurements)
   await result.update({
     summary: nettestType.makeSummary(measurements),
     endTime: moment.utc().toDate(),

From 596883862e8bf85c623ec834f3f8de6afaa76e0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 10 Jan 2018 13:39:46 +0100
Subject: [PATCH 09/17] Add summary functions for IM tests

---
 src/nettests/index.js | 28 ++++++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/src/nettests/index.js b/src/nettests/index.js
index ed0c389..5319eec 100644
--- a/src/nettests/index.js
+++ b/src/nettests/index.js
@@ -240,12 +240,32 @@ export const nettestTypes = {
     help: 'No help for you',
     makeSummary: (measurements) => {
       return {
-        facebookMessengerBlocked: true,
-        whatsappBlocked: true,
-        telegramBlocked: true,
+        whatsappBlocked: (measurements[0].summary.whatsappEndpointsBlocked ||
+                          measurements[0].summary.whatsappWebBlocked ||
+                          measurements[0].summary.registrationServerBlocked),
+        facebookMessengerBlocked: (measurements[1].summary.facebookTcpBlocking ||
+                                   measurements[1].summary.facebookDnsBlocking),
+        telegramBlocked: (measurements[2].summary.telegramWebBlocked ||
+                          measurements[2].summary.telegramHttpBlocked ||
+                          measurements[2].summary.telegramTcpBlocked)
       }
     },
-    renderSummary: (measurement, {Cli, chalk}) => {
+    renderSummary: (result, {Cli, chalk}) => {
+      if (result.summary.whatsappBlocked === true) {
+        Cli.log(Cli.output.notok('WhatsApp NOT ok'))
+      } else {
+        Cli.log(Cli.output.ok('WhatsApp is OK'))
+      }
+      if (result.summary.facebookMessengerBlocked === true) {
+        Cli.log(Cli.output.notok('Facebook NOT ok'))
+      } else {
+        Cli.log(Cli.output.ok('Facebook is OK'))
+      }
+      if (result.summary.telegramBlocked === true) {
+        Cli.log(Cli.output.ok('Telegram NOT ok'))
+      } else {
+        Cli.log(Cli.output.ok('Telegram is OK'))
+      }
     }
   },
   circumvention: {

From 05a4293633cc961ed94bac66948d72b80b2b5926 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 10 Jan 2018 15:46:40 +0100
Subject: [PATCH 10/17] Implement middlebox tests

---
 src/commands/list.js                          |  4 ++-
 src/commands/run.js                           |  6 +++--
 .../http-header-field-manipulation/index.js   | 27 +++++++------------
 .../http-invalid-request-line/index.js        | 25 ++++++-----------
 src/nettests/index.js                         | 16 ++++++++---
 src/nettests/ndt/index.js                     | 20 +++-----------
 6 files changed, 41 insertions(+), 57 deletions(-)

diff --git a/src/commands/list.js b/src/commands/list.js
index 3776426..6711f06 100644
--- a/src/commands/list.js
+++ b/src/commands/list.js
@@ -87,7 +87,9 @@ const listAction = async ctx => {
         summary.push(m)
       })
       // XXX we should figure out how this will work when we have many measurements
-      nettest.renderSummary([measurement], {Cli, chalk})
+      if (measurement.summary) {
+        nettest.renderSummary([measurement], {Cli, chalk})
+      }
       return {
         name: measurement.name,
         network: measurement.asn,
diff --git a/src/commands/run.js b/src/commands/run.js
index e8a20bb..7117756 100644
--- a/src/commands/run.js
+++ b/src/commands/run.js
@@ -92,10 +92,12 @@ const run = async ({camelName, argv}) => {
   const measurements = await result.getMeasurements()
   debug('updating the result table')
   debug(measurements)
+  const summary = nettestType.makeSummary(measurements)
+  debug('summary: ', summary)
   await result.update({
-    summary: nettestType.makeSummary(measurements),
     endTime: moment.utc().toDate(),
-    done: true
+    done: true,
+    summary
   })
 }
 
diff --git a/src/nettests/http-header-field-manipulation/index.js b/src/nettests/http-header-field-manipulation/index.js
index 5efa6f0..798c816 100644
--- a/src/nettests/http-header-field-manipulation/index.js
+++ b/src/nettests/http-header-field-manipulation/index.js
@@ -7,23 +7,13 @@ export const renderSummary = (measurements, {React, Cli, Components, chalk}) =>
   // When this function is called from the Cli the Cli will be set, when it's
   // called from the Desktop app we have React set instead.
   if (Cli) {
-    Cli.log(Cli.output.labelValue('Label', uploadMbit, {unit: 'Mbit'}))
-  } else if (React) {
-    /*
-    XXX this is broken currently as it depends on react
-    const {
-      Container,
-      Heading
-    } = Components
-    return class extends React.Component {
-      render() {
-        return <Container>
-          <Heading h={1}>Results for NDT</Heading>
-          {uploadMbit}
-        </Container>
-      }
+    if (summary.foundMiddlebox === true) {
+      Cli.log(Cli.output.notok('Detected the presence of a Middle Box'))
+    } else {
+      Cli.log(Cli.output.ok('No Middle Box detected'))
     }
-    */
+  } else if (React) {
+    // XXX this is broken currently as it depends on react
   }
 }
 
@@ -31,6 +21,9 @@ export const renderHelp = () => {
 }
 
 export const makeSummary = ({test_keys}) => ({
+  foundMiddlebox: (test_keys.tampering.header_field_name ||
+                   test_keys.tampering.request_line_capitalization ||
+                   test_keys.tampering.total)
 })
 
 export const run = ({ooni, argv}) => {
@@ -39,7 +32,7 @@ export const run = ({ooni, argv}) => {
 
   httpHeaderFieldManipulation.on('begin', () => ooni.onProgress(0.0, 'starting http-header-field-manipulation'))
   httpHeaderFieldManipulation.on('progress', (percent, message) => {
-    ooni.onProgress(percent, message, persist)
+    ooni.onProgress(percent, message)
   })
   return ooni.run(httpHeaderFieldManipulation.run)
 }
diff --git a/src/nettests/http-invalid-request-line/index.js b/src/nettests/http-invalid-request-line/index.js
index 8681adc..82883e0 100644
--- a/src/nettests/http-invalid-request-line/index.js
+++ b/src/nettests/http-invalid-request-line/index.js
@@ -7,23 +7,13 @@ export const renderSummary = (measurements, {React, Cli, Components, chalk}) =>
   // When this function is called from the Cli the Cli will be set, when it's
   // called from the Desktop app we have React set instead.
   if (Cli) {
-    Cli.log(Cli.output.labelValue('Label', uploadMbit, {unit: 'Mbit'}))
-  } else if (React) {
-    /*
-    XXX this is broken currently as it depends on react
-    const {
-      Container,
-      Heading
-    } = Components
-    return class extends React.Component {
-      render() {
-        return <Container>
-          <Heading h={1}>Results for NDT</Heading>
-          {uploadMbit}
-        </Container>
-      }
+    if (summary.foundMiddlebox === true) {
+      Cli.log(Cli.output.notok('Detected the presence of a Middle Box'))
+    } else {
+      Cli.log(Cli.output.ok('No Middle Box detected'))
     }
-    */
+  } else if (React) {
+    // XXX this is broken currently as it depends on react
   }
 }
 
@@ -31,6 +21,7 @@ export const renderHelp = () => {
 }
 
 export const makeSummary = ({test_keys}) => ({
+  foundMiddlebox: test_keys.tampering
 })
 
 export const run = ({ooni, argv}) => {
@@ -39,7 +30,7 @@ export const run = ({ooni, argv}) => {
 
   httpInvalidRequestLine.on('begin', () => ooni.onProgress(0.0, 'starting http-invalid-request-line'))
   httpInvalidRequestLine.on('progress', (percent, message) => {
-    ooni.onProgress(percent, message, persist)
+    ooni.onProgress(percent, message)
   })
   return ooni.run(httpInvalidRequestLine.run)
 }
diff --git a/src/nettests/index.js b/src/nettests/index.js
index 5319eec..a3fde2d 100644
--- a/src/nettests/index.js
+++ b/src/nettests/index.js
@@ -81,6 +81,9 @@ export const makeOoni = (loader, geoip) => {
           uploaded = false
           reportId = `LOCAL-${entry.id}`
         }
+        debug('generating summary for ' + reportFile)
+        const summary = loader.nettest.makeSummary(entry)
+        debug('generated summary: ', summary)
         let measurement = Measurement.build({
           state: 'active',
           reportId: reportId,
@@ -92,7 +95,7 @@ export const makeOoni = (loader, geoip) => {
           reportFile: reportFile,
           // We append the Z to make moment understand it's UTC
           startTime: moment(entry['measurement_start_time'] + 'Z').toDate(),
-          summary: loader.nettest.makeSummary(entry)
+          summary
         })
         dbOperations.push(measurement.save())
         measurements.push(measurement)
@@ -224,10 +227,17 @@ export const nettestTypes = {
     help: 'No help for you',
     makeSummary: (measurements) => {
       return {
-        foundMiddlebox: true
+        foundMiddlebox: (measurements[0].summary.foundMiddlebox ||
+                         measurements[1].summary.foundMiddlebox)
       }
     },
-    renderSummary: (measurement, {Cli, chalk}) => {}
+    renderSummary: (result, {Cli, chalk}) => {
+      if (result.summary.foundMiddlebox === true) {
+        Cli.log(Cli.output.notok('Found Middle Box'))
+      } else {
+        Cli.log(Cli.output.ok('No Middle Box'))
+      }
+    }
   },
   imBlocking: {
     nettests: [
diff --git a/src/nettests/ndt/index.js b/src/nettests/ndt/index.js
index 74e42e8..528e856 100644
--- a/src/nettests/ndt/index.js
+++ b/src/nettests/ndt/index.js
@@ -23,21 +23,7 @@ export const renderSummary = (measurements, {React, Cli, Components, chalk}) =>
     Cli.log(Cli.output.labelValue('MSS', mss))
     Cli.log(Cli.output.labelValue('Timeouts', timeouts))
   } else if (React) {
-    /*
-    XXX this is broken currently as it depends on react
-    const {
-      Container,
-      Heading
-    } = Components
-    return class extends React.Component {
-      render() {
-        return <Container>
-          <Heading h={1}>Results for NDT</Heading>
-          {uploadMbit}
-        </Container>
-      }
-    }
-    */
+    // XXX this is broken currently as it depends on react
   }
 }
 
@@ -63,8 +49,8 @@ export const run = ({ooni, argv}) => {
 
   ndt.on('begin', () => ooni.onProgress(0.0, 'starting ndt'))
   ndt.on('progress', (percent, message) => {
-    const persist = !(message.startsWith('upload-speed') ||
-                      message.startsWith('download-speed'))
+    const persist = (message.startsWith('upload-speed') ||
+                     message.startsWith('download-speed'))
     ooni.onProgress(percent, message, persist)
   })
   return ooni.run(ndt.run)

From 4889e6b769ec8fea196bb5da4d5dc1edc47948de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 10 Jan 2018 15:49:17 +0100
Subject: [PATCH 11/17] Increase the width of the table

---
 src/cli/output/test-results.js | 8 ++++----
 src/nettests/whatsapp/index.js | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/cli/output/test-results.js b/src/cli/output/test-results.js
index 36cd932..db400ec 100644
--- a/src/cli/output/test-results.js
+++ b/src/cli/output/test-results.js
@@ -55,7 +55,7 @@ import labelValue from './label-value'
 */
 
 const testResults = async (results, getMeta) => {
-  const colWidth = 50
+  const colWidth = 76
   let o = '┏' + '━'.repeat(colWidth) + '┓\n'
   let totalDataUsage = 0
   let totalRows = 0
@@ -85,17 +85,17 @@ const testResults = async (results, getMeta) => {
     let firstRow = `${chalk.bold(`#${r.id}`)} - ${moment(meta.date).fromNow()}`
     firstRow += rightPad(firstRow, innerWidth)
     let secondRow = meta.name
-    secondRow += rightPad(secondRow, 26)
+    secondRow += rightPad(secondRow, colWidth/2)
     secondRow += meta.summary[0] || ''
     secondRow += rightPad(secondRow, innerWidth)
 
     let thirdRow = meta.network
-    thirdRow += rightPad(thirdRow, 26)
+    thirdRow += rightPad(thirdRow, colWidth/2)
     thirdRow += meta.summary[1] || ''
     thirdRow += rightPad(thirdRow, innerWidth)
 
     let fourthRow = `${chalk.cyan(meta.asn)} (${chalk.cyan(meta.country)})`
-    fourthRow += rightPad(fourthRow, 26)
+    fourthRow += rightPad(fourthRow, colWidth/2)
     fourthRow += meta.summary[2] || ''
     fourthRow += rightPad(fourthRow, innerWidth)
 
diff --git a/src/nettests/whatsapp/index.js b/src/nettests/whatsapp/index.js
index a16c90d..7affe53 100644
--- a/src/nettests/whatsapp/index.js
+++ b/src/nettests/whatsapp/index.js
@@ -20,9 +20,9 @@ export const renderSummary = (measurements, {React, Cli, Components, chalk}) =>
     }
 
     if (summary.registrationServerBlocked === true) {
-      Cli.log(Cli.output.notok('Whatapp registration server is blocked'))
+      Cli.log(Cli.output.notok('Registration server is blocked'))
     } else {
-      Cli.log(Cli.output.ok('Whatapp registration server is not blocked'))
+      Cli.log(Cli.output.ok('Registration server is not blocked'))
     }
   } else if (React) {
     // XXX this is broken currently as it depends on react

From 737539a7175130688f86f027509f4b8f81d729be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Wed, 10 Jan 2018 16:56:38 +0100
Subject: [PATCH 12/17] Add support for counting the data usage

---
 package.json                   |  4 ++--
 src/cli/output/test-results.js | 29 +++++++++++++++++++++--------
 src/commands/list.js           |  4 ++--
 src/commands/run.js            | 19 ++++++++++++++++---
 src/config/db.js               |  5 +++--
 src/config/geoip.js            |  6 ++++++
 src/nettests/index.js          | 24 +++++++++++++++---------
 src/nettests/ndt/index.js      |  6 ++++--
 yarn.lock                      |  6 +++---
 9 files changed, 72 insertions(+), 31 deletions(-)

diff --git a/package.json b/package.json
index 7149b26..8bf0b80 100644
--- a/package.json
+++ b/package.json
@@ -21,6 +21,7 @@
   },
   "devDependencies": {
     "ansi-escapes": "^3.0.0",
+    "axios": "^0.17.1",
     "babel-core": "^6.26.0",
     "babel-loader": "^7.1.2",
     "babel-preset-es2015": "^6.24.1",
@@ -53,7 +54,6 @@
     "zlib": "^1.0.5"
   },
   "dependencies": {
-    "axios": "^0.17.1",
-    "measurement-kit": "^0.1.0-alpha.6"
+    "measurement-kit": "^0.1.0-alpha.8"
   }
 }
diff --git a/src/cli/output/test-results.js b/src/cli/output/test-results.js
index db400ec..8a832e8 100644
--- a/src/cli/output/test-results.js
+++ b/src/cli/output/test-results.js
@@ -57,7 +57,8 @@ import labelValue from './label-value'
 const testResults = async (results, getMeta) => {
   const colWidth = 76
   let o = '┏' + '━'.repeat(colWidth) + '┓\n'
-  let totalDataUsage = 0
+  let totalDataUsageUp = 0
+  let totalDataUsageDown = 0
   let totalRows = 0
   let allAsns = []
   let allCountries = []
@@ -79,7 +80,10 @@ const testResults = async (results, getMeta) => {
     if (allAsns.indexOf(meta.asn) === -1) {
       allAsns.push(meta.asn)
     }
-    totalDataUsage += meta.dataUsage
+    if (meta.dataUsageUp && meta.dataUsageDown) {
+      totalDataUsageUp += meta.dataUsageUp
+      totalDataUsageDown += meta.dataUsageDown
+    }
     totalRows += 1
 
     let firstRow = `${chalk.bold(`#${r.id}`)} - ${moment(meta.date).fromNow()}`
@@ -110,8 +114,11 @@ const testResults = async (results, getMeta) => {
     results.map(getContentRow)
   )
 
-  let dataUsageCell = `${humanize.filesize(totalDataUsage)}`
-  dataUsageCell += rightPad(dataUsageCell, 12)
+  let dataUsageDownCell = `U ${humanize.filesize(totalDataUsageDown)}`
+  dataUsageDownCell += rightPad(dataUsageDownCell, 12)
+
+  let dataUsageUpCell = `D ${humanize.filesize(totalDataUsageUp)}`
+  dataUsageUpCell += rightPad(dataUsageUpCell, 12)
 
   let networksCell = `${allAsns.length} nets`
   networksCell += rightPad(networksCell, 12)
@@ -120,10 +127,16 @@ const testResults = async (results, getMeta) => {
   msmtsCell += rightPad(msmtsCell, 12)
 
   o += contentRows.join('┢' + '━'.repeat(colWidth) + '┪\n')
-  o += '└┬──────────────┬──────────────┬──────────────┬'+'─'.repeat(colWidth - 46)+'┘\n'
-  o += ` │ ${msmtsCell} │ ${networksCell} │ ${dataUsageCell} │
- ╰──────────────┴──────────────┴──────────────╯
-`
+
+  if (totalDataUsageDown && totalDataUsageUp) {
+    o += '└┬──────────────┬──────────────┬──────────────┬──────────────┬'+'─'.repeat(colWidth - 61)+'┘\n'
+    o += ` │ ${msmtsCell} │ ${networksCell} │ ${dataUsageDownCell} │ ${dataUsageUpCell} │\n`
+    o += ' ╰──────────────┴──────────────┴──────────────┴──────────────╯'
+  } else {
+    o += '└┬──────────────┬──────────────┬'+'─'.repeat(colWidth - 31)+'┘\n'
+    o += ` │ ${msmtsCell} │ ${networksCell} │\n`
+    o += ' ╰──────────────┴──────────────╯'
+  }
   return o
 }
 
diff --git a/src/commands/list.js b/src/commands/list.js
index 6711f06..539ada8 100644
--- a/src/commands/list.js
+++ b/src/commands/list.js
@@ -95,7 +95,6 @@ const listAction = async ctx => {
         network: measurement.asn,
         asn: measurement.asn,
         country: measurement.country,
-        dataUsage: measurement.dataUsage,
         date: measurement.startTime,
         summary: summary
       }
@@ -120,7 +119,8 @@ const listAction = async ctx => {
         network: measurements[0].asn,
         asn: measurements[0].asn,
         country: measurements[0].country,
-        dataUsage: measurements.map(m => m.dataUsage).reduce((a,b) => a += b),
+        dataUsageUp: result.dataUsageUp,
+        dataUsageDown: result.dataUsageDown,
         date: result.startTime,
         summary: summary
       }
diff --git a/src/commands/run.js b/src/commands/run.js
index 7117756..35c9b3a 100644
--- a/src/commands/run.js
+++ b/src/commands/run.js
@@ -72,20 +72,31 @@ const run = async ({camelName, argv}) => {
               chalk.bold(`${nettestType.nettests.length} ${nettestType.name} `) +
               `test${sOrNot}`))
 
+  let dataUsageUp = 0
+  let dataUsageDown = 0
   const geoip = await getGeoipPaths()
 
   for (const nettestLoader of nettestType.nettests) {
     const loader = nettestLoader()
     const { nettest, meta } = loader
     console.log(info(`${chalk.bold(meta.name)}`))
-    const measurement = await nettest.run({ooni: makeOoni(loader, geoip), argv})
-    nettest.renderSummary(measurement, {
+    const [measurements, dataUsage] = await nettest.run({
+      ooni: makeOoni(loader, geoip),
+      argv
+    })
+    debug('setting data usage', dataUsage)
+    dataUsageUp += dataUsage.up || 0
+    dataUsageDown += dataUsage.down || 0
+    nettest.renderSummary(measurements, {
       Cli: makeCli(),
       chalk: chalk,
       Components: null,
       React: null,
     })
-    dbOperations.push(result.addMeasurements(measurement))
+
+    for (const measurement of measurements) {
+      dbOperations.push(result.addMeasurements(measurement))
+    }
   }
   await Promise.all(dbOperations)
 
@@ -97,6 +108,8 @@ const run = async ({camelName, argv}) => {
   await result.update({
     endTime: moment.utc().toDate(),
     done: true,
+    dataUsageUp: dataUsageUp,
+    dataUsageDown: dataUsageDown,
     summary
   })
 }
diff --git a/src/config/db.js b/src/config/db.js
index f0bc44c..fd3eac0 100644
--- a/src/config/db.js
+++ b/src/config/db.js
@@ -31,7 +31,6 @@ export const Measurement = sequelize.define('measurement', {
   name: Sequelize.STRING,
   startTime: Sequelize.DATE,
   endTime: Sequelize.DATE,
-  dataUsage: Sequelize.INTEGER,
   // This is an opaque JSON that is test dependent
   summary: Sequelize.JSON,
 
@@ -66,7 +65,9 @@ export const Result = sequelize.define('result', {
   startTime: Sequelize.DATE,
   endTime: Sequelize.DATE,
   summary: Sequelize.JSON,
-  done: Sequelize.BOOLEAN
+  done: Sequelize.BOOLEAN,
+  dataUsageUp: Sequelize.INTEGER,
+  dataUsageDown: Sequelize.INTEGER
 })
 Result.hasMany(Measurement, { as: 'Measurements' })
 sequelize.sync()
diff --git a/src/config/geoip.js b/src/config/geoip.js
index fb4cf75..84ae304 100644
--- a/src/config/geoip.js
+++ b/src/config/geoip.js
@@ -4,6 +4,7 @@ import path from 'path'
 import axios from 'axios'
 import zlib from 'zlib'
 
+import wait from '../cli/output/wait'
 import { getOoniDir } from './global-path'
 
 const BASE_URL = 'https://github.com/OpenObservatory/ooni-resources/releases/download/'
@@ -35,6 +36,7 @@ const downloadFile = ({url, dst, uncompress}) => {
 }
 
 export const getGeoipPaths = async () => {
+  let progress
   const geoipDir = path.join(getOoniDir(), 'geoip')
 
   // XXX exception handling
@@ -46,21 +48,25 @@ export const getGeoipPaths = async () => {
   const geoipCountryPath = path.join(geoipDir, GEOIP_COUNTRY_FILENAME)
   const geoipCountryExists = await fs.pathExists(geoipCountryPath)
   if (!geoipCountryExists) {
+    progress = wait('downloading GeoIP country file')
     await downloadFile({
       url: `${BASE_URL}${LATEST_VERSION}/${GEOIP_COUNTRY_FILENAME}.gz`,
       dst: geoipCountryPath,
       uncompress: true
     })
+    progress()
   }
 
   const geoipAsnPath = path.join(geoipDir, GEOIP_ASN_FILENAME)
   const geoipAsnExists = await fs.pathExists(geoipAsnPath)
   if (!geoipAsnExists) {
+    progress = wait('downloading GeoIP ASN file')
     await downloadFile({
       url: `${BASE_URL}${LATEST_VERSION}/${GEOIP_ASN_FILENAME}.gz`,
       dst: geoipAsnPath,
       uncompress: true
     })
+    progress()
   }
   return {
     countryPath: geoipCountryPath,
diff --git a/src/nettests/index.js b/src/nettests/index.js
index a3fde2d..ad0365c 100644
--- a/src/nettests/index.js
+++ b/src/nettests/index.js
@@ -31,6 +31,10 @@ export const makeOoni = (loader, geoip) => {
       progress = null,
       reportId = null,
       reportFile = null,
+      dataUsage = {
+        up: 0,
+        down: 0
+      },
       measurementName = camelCase(loader.meta.name),
       uploaded = false,
       localReportId = null,
@@ -69,6 +73,10 @@ export const makeOoni = (loader, geoip) => {
       nt.on('end', () => {
         debug('ending test')
       })
+      nt.on('overall-data-usage', ({down, up}) => {
+        dataUsage.up = up
+        dataUsage.down = down
+      })
       nt.on('entry', entry => {
         // XXX This is a bit of a hack
         // When we don't have a reportId with the collector we set the
@@ -83,6 +91,8 @@ export const makeOoni = (loader, geoip) => {
         }
         debug('generating summary for ' + reportFile)
         const summary = loader.nettest.makeSummary(entry)
+        const startTime = moment(entry['measurement_start_time'] + 'Z').toDate()
+        const endTime = new Date(startTime.getTime() + entry['test_runtime'] * 1000)
         debug('generated summary: ', summary)
         let measurement = Measurement.build({
           state: 'active',
@@ -94,7 +104,8 @@ export const makeOoni = (loader, geoip) => {
           name: measurementName,
           reportFile: reportFile,
           // We append the Z to make moment understand it's UTC
-          startTime: moment(entry['measurement_start_time'] + 'Z').toDate(),
+          startTime,
+          endTime,
           summary
         })
         dbOperations.push(measurement.save())
@@ -108,31 +119,26 @@ export const makeOoni = (loader, geoip) => {
     progress = wait(`${percentage(percent)}: ${message}`, persist)
   }
 
-  const setSummary = (measurementId, summary) => {
-  }
-
   const run = async (runner) => {
     await runner()
 
     // XXX Here I make the assumption that either it all failed or not.
     // This is a lie.
+    // The endTime is also not correct
     for (const measurement of measurements) {
       dbOperations.push(measurement.update({
-        state: uploaded ? 'uploaded' : 'done',
-        endTime: moment().utc().toDate(),
-        dataUsage: 1024**2*randInt(1, 20) // XXX we currently fill this with some random data
+        state: uploaded ? 'uploaded' : 'done'
       }))
     }
 
     await Promise.all(dbOperations)
     progress && progress()
-    return measurements
+    return [measurements, dataUsage]
   }
 
   return {
     init,
     onProgress,
-    setSummary,
     mkOptions,
     run
   }
diff --git a/src/nettests/ndt/index.js b/src/nettests/ndt/index.js
index 528e856..d28f5dd 100644
--- a/src/nettests/ndt/index.js
+++ b/src/nettests/ndt/index.js
@@ -49,9 +49,11 @@ export const run = ({ooni, argv}) => {
 
   ndt.on('begin', () => ooni.onProgress(0.0, 'starting ndt'))
   ndt.on('progress', (percent, message) => {
-    const persist = (message.startsWith('upload-speed') ||
+    /*
+      const persist = (message.startsWith('upload-speed') ||
                      message.startsWith('download-speed'))
-    ooni.onProgress(percent, message, persist)
+    */
+    ooni.onProgress(percent, message)
   })
   return ooni.run(ndt.run)
 }
diff --git a/yarn.lock b/yarn.lock
index f10bbc2..6b34548 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2115,9 +2115,9 @@ md5.js@^1.3.4:
     hash-base "^3.0.0"
     inherits "^2.0.1"
 
-measurement-kit@^0.1.0-alpha.6:
-  version "0.1.0-alpha.6"
-  resolved "https://registry.yarnpkg.com/measurement-kit/-/measurement-kit-0.1.0-alpha.6.tgz#9c92534877ed6204d30435377169b72d666d7f60"
+measurement-kit@^0.1.0-alpha.8:
+  version "0.1.0-alpha.8"
+  resolved "https://registry.yarnpkg.com/measurement-kit/-/measurement-kit-0.1.0-alpha.8.tgz#27ac3040a08ecf0951084b449fed07680af988dd"
   dependencies:
     any-promise "^1.3.0"
     bindings "^1.3.0"

From adc34350acf9dce94f31fc8208578ef14ceeb7ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Thu, 11 Jan 2018 15:50:38 +0100
Subject: [PATCH 13/17] Remove node-ipc as we don't really need it

---
 package.json        |  1 -
 src/commands/run.js |  6 ++++++
 src/config/ipc.js   | 24 ++++++++++++++++++++++++
 src/ipc/config.js   | 21 ---------------------
 src/ipc/start.js    | 19 -------------------
 src/ooni.js         | 11 +++--------
 yarn.lock           | 26 --------------------------
 7 files changed, 33 insertions(+), 75 deletions(-)
 create mode 100644 src/config/ipc.js
 delete mode 100644 src/ipc/config.js
 delete mode 100644 src/ipc/start.js

diff --git a/package.json b/package.json
index 62ad04b..8bf0b80 100644
--- a/package.json
+++ b/package.json
@@ -39,7 +39,6 @@
     "lodash.range": "^3.2.0",
     "moment": "^2.19.1",
     "mri": "^1.1.0",
-    "node-ipc": "^9.1.1",
     "ora": "^1.3.0",
     "pkg": "^4.2.4",
     "prettyjson": "^1.2.1",
diff --git a/src/commands/run.js b/src/commands/run.js
index 35c9b3a..2a3ed6e 100644
--- a/src/commands/run.js
+++ b/src/commands/run.js
@@ -18,6 +18,10 @@ import {
   Result
 } from '../config/db'
 
+import {
+  notify
+} from '../config/ipc'
+
 import makeCli from '../cli/make-cli'
 
 import { getGeoipPaths } from '../config/geoip'
@@ -68,6 +72,8 @@ const run = async ({camelName, argv}) => {
   })
   dbOperations.push(result.save())
 
+  notify({key: 'starting-test', value: camelName})
+
   console.log(info('Running '+
               chalk.bold(`${nettestType.nettests.length} ${nettestType.name} `) +
               `test${sOrNot}`))
diff --git a/src/config/ipc.js b/src/config/ipc.js
new file mode 100644
index 0000000..5de0e45
--- /dev/null
+++ b/src/config/ipc.js
@@ -0,0 +1,24 @@
+let ipcEnabled = false
+
+export const enableIpc = () => {
+  if (ipcEnabled === true) {
+    throw Error('IPC cannot be enabled twice')
+  }
+  ipcEnabled = true
+  listenForMessages()
+}
+
+const listenForMessages = () => {
+  process.on('message', m => {
+    console.log('got message', m)
+  })
+}
+
+export const notify = ({key, value}) => {
+  if (ipcEnabled === true) {
+    process.send({
+      key,
+      value
+    })
+  }
+}
diff --git a/src/ipc/config.js b/src/ipc/config.js
deleted file mode 100644
index 00fca1b..0000000
--- a/src/ipc/config.js
+++ /dev/null
@@ -1,21 +0,0 @@
-import * as fs from 'fs-extra'
-import path from 'path'
-import { getOoniDir } from '../config/global-path'
-
-const configIpc = async (ipcId) => {
-  const ipc = require('node-ipc')
-  // XXX we should somewhere do some sanity checks on the permissions of the
-  // sockets directory to ensure the world cannot read and write to them.
-  const socketRoot = path.join(getOoniDir(), 'sockets')
-
-  await fs.ensureDir(socketRoot)
-
-  ipc.config.id = ipcId || ''+Date.now()
-  ipc.config.appspace = 'ooni.'
-  ipc.config.retry = 1500
-  ipc.config.socketRoot = socketRoot
-  // XXX check what this maxConnections does and ensure it's actually secure
-  ipc.config.maxConnections = 1
-  return ipc
-}
-export default configIpc
diff --git a/src/ipc/start.js b/src/ipc/start.js
deleted file mode 100644
index fc3eb33..0000000
--- a/src/ipc/start.js
+++ /dev/null
@@ -1,19 +0,0 @@
-const startIpc = (ipc) => {
-  return new Promise((resolve, reject) => {
-    ipc.serveNet(() => {
-      ip.server.on(
-        'something',
-        (data, socket) => {
-          ipc.log('got message', data)
-          ipc.server.emit(
-            socket,
-            'message',
-            data + 'world'
-          )
-        })
-    })
-    ipc.server.start()
-    resolve()
-  })
-}
-export default startIpc
diff --git a/src/ooni.js b/src/ooni.js
index 5c011dd..2cc15ef 100644
--- a/src/ooni.js
+++ b/src/ooni.js
@@ -16,8 +16,7 @@ import {
   writeToConfigFile
 } from './config/config-files'
 
-import configIpc from './ipc/config'
-import startIpc from './ipc/start'
+import { enableIpc } from './config/ipc'
 
 const OONI_DIR = getOoniDir()
 const OONI_CONFIG_PATH = getConfigFilePath()
@@ -138,10 +137,7 @@ const main = async (argv_) => {
   let ipc
   if (argv.ipc) {
     try {
-      ipc = await configIpc(argv.ipc)
-      // XXX do we maybe want to have an CLI option to have the process wait
-      // until the consumer connects to the IPC subsystem?
-      await startIpc(ipc)
+      enableIpc()
     } catch(err) {
       console.error(error('An error occurred while starting IPC subsystem ' +
         `ooni config file in "${OONI_CONFIG_PATH}"` + err.message
@@ -151,8 +147,7 @@ const main = async (argv_) => {
   }
 
   const ctx = {
-    argv: argv_,
-    ipc
+    argv: argv_
   }
 
   if (subcommand === 'help' && argv._[3]) {
diff --git a/yarn.lock b/yarn.lock
index d854245..806a4d4 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1195,10 +1195,6 @@ duplexify@^3.4.2, duplexify@^3.5.3:
     readable-stream "^2.0.0"
     stream-shift "^1.0.0"
 
-easy-stack@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/easy-stack/-/easy-stack-1.0.0.tgz#12c91b3085a37f0baa336e9486eac4bf94e3e788"
-
 ecc-jsbn@~0.1.1:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz#0fc73a9ed5f0d53c38193398523ef7e543777505"
@@ -1354,10 +1350,6 @@ event-emitter@~0.3.5:
     d "1"
     es5-ext "~0.10.14"
 
-event-pubsub@4.3.0:
-  version "4.3.0"
-  resolved "https://registry.yarnpkg.com/event-pubsub/-/event-pubsub-4.3.0.tgz#f68d816bc29f1ec02c539dc58c8dd40ce72cb36e"
-
 events@^1.0.0:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
@@ -2021,16 +2013,6 @@ isstream@~0.1.2:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a"
 
-js-message@1.0.5:
-  version "1.0.5"
-  resolved "https://registry.yarnpkg.com/js-message/-/js-message-1.0.5.tgz#2300d24b1af08e89dd095bc1a4c9c9cfcb892d15"
-
-js-queue@2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/js-queue/-/js-queue-2.0.0.tgz#362213cf860f468f0125fc6c96abc1742531f948"
-  dependencies:
-    easy-stack "^1.0.0"
-
 js-tokens@^3.0.0, js-tokens@^3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-3.0.2.tgz#9866df395102130e38f7f996bceb65443209c25b"
@@ -2391,14 +2373,6 @@ node-gyp@^3.6.2:
     tar "^2.0.0"
     which "1"
 
-node-ipc@^9.1.1:
-  version "9.1.1"
-  resolved "https://registry.yarnpkg.com/node-ipc/-/node-ipc-9.1.1.tgz#4e245ed6938e65100e595ebc5dc34b16e8dd5d69"
-  dependencies:
-    event-pubsub "4.3.0"
-    js-message "1.0.5"
-    js-queue "2.0.0"
-
 node-libs-browser@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/node-libs-browser/-/node-libs-browser-2.1.0.tgz#5f94263d404f6e44767d726901fff05478d600df"

From 2a85d4e425c914d5b521d0b45ca8b21a3c58414c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Thu, 11 Jan 2018 16:11:17 +0100
Subject: [PATCH 14/17] Fixup the IPC communication

---
 src/config/ipc.js | 4 +++-
 src/ooni.js       | 9 ++++++---
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/config/ipc.js b/src/config/ipc.js
index 5de0e45..02a58d5 100644
--- a/src/config/ipc.js
+++ b/src/config/ipc.js
@@ -1,3 +1,5 @@
+const process = require('process')
+
 let ipcEnabled = false
 
 export const enableIpc = () => {
@@ -15,7 +17,7 @@ const listenForMessages = () => {
 }
 
 export const notify = ({key, value}) => {
-  if (ipcEnabled === true) {
+  if (ipcEnabled === true && process.send) {
     process.send({
       key,
       value
diff --git a/src/ooni.js b/src/ooni.js
index 2cc15ef..3df9cd5 100644
--- a/src/ooni.js
+++ b/src/ooni.js
@@ -18,17 +18,20 @@ import {
 
 import { enableIpc } from './config/ipc'
 
+const debug = require('debug')('ooni')
+
 const OONI_DIR = getOoniDir()
 const OONI_CONFIG_PATH = getConfigFilePath()
 
 const main = async (argv_) => {
   const argv = mri(argv_, {
-    boolean: ['help', 'version', 'verbose'],
+    boolean: ['help', 'version', 'verbose', 'ipc'],
     string: [],
     alias: {
       help: 'h'
     }
   })
+  debug(argv)
 
   let subcommand = argv._[2]
 
@@ -134,9 +137,9 @@ const main = async (argv_) => {
     }
   }
 
-  let ipc
   if (argv.ipc) {
     try {
+      debug('enabling IPC')
       enableIpc()
     } catch(err) {
       console.error(error('An error occurred while starting IPC subsystem ' +
@@ -147,7 +150,7 @@ const main = async (argv_) => {
   }
 
   const ctx = {
-    argv: argv_
+    argv: argv._
   }
 
   if (subcommand === 'help' && argv._[3]) {

From 6002d6ede71af6c08b50dc25acdf0f7ff2123a01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Thu, 11 Jan 2018 21:50:12 +0100
Subject: [PATCH 15/17] Call exit(0) when we should exit

---
 src/commands/run.js | 1 +
 src/config/ipc.js   | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/commands/run.js b/src/commands/run.js
index 2a3ed6e..1b7e994 100644
--- a/src/commands/run.js
+++ b/src/commands/run.js
@@ -167,6 +167,7 @@ const main = async ctx => {
       await exit(0)
     } else {
       await run({camelName, argv})
+      await exit(0)
     }
   } catch(err) {
     if (err.usageError) {
diff --git a/src/config/ipc.js b/src/config/ipc.js
index 02a58d5..d9b48e3 100644
--- a/src/config/ipc.js
+++ b/src/config/ipc.js
@@ -16,7 +16,7 @@ const listenForMessages = () => {
   })
 }
 
-export const notify = ({key, value}) => {
+export const notify = ({key, value}) =>{
   if (ipcEnabled === true && process.send) {
     process.send({
       key,

From 35d78ea764ef7c62e75cb72853d03914ded75d91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Thu, 11 Jan 2018 23:31:19 +0100
Subject: [PATCH 16/17] Move all pack related functionality into a separate
 script

---
 pack.js      | 40 ++++++++++++++++++++++++++++++++++++++++
 package.json | 12 +-----------
 2 files changed, 41 insertions(+), 11 deletions(-)
 create mode 100644 pack.js

diff --git a/pack.js b/pack.js
new file mode 100644
index 0000000..9452f9b
--- /dev/null
+++ b/pack.js
@@ -0,0 +1,40 @@
+const cp = require('child_process')
+const os = require('os')
+const path = require('path')
+
+const pkg = require('pkg')
+
+const nodeTargets = {
+  darwin: "node8-macos-x64",
+  linux: "node8-linux-x64",
+  win32: "node8-win-x64",
+}
+
+const nativeModules = {
+  darwin: [
+    'node_modules/measurement-kit/build/Release/measurement-kit.node',
+    'node_modules/sqlite3/lib/binding/node-v57-darwin-x64/node_sqlite3.node'
+  ],
+  win32: [],
+  linux: []
+}
+
+const platform = os.platform()
+
+if (!nodeTargets[platform]) {
+  console.log('this platform is not supported')
+  process.exit(0)
+}
+
+console.log('- building packed/ooni for ' + nodeTargets[platform])
+pkg.exec([ 'dist/ooni.js', '--target', nodeTargets[platform], '-o', 'packed/ooni' ])
+.then(() => {
+  nativeModules[platform].forEach(dotNodePath => {
+    console.log('- copying ' + dotNodePath)
+    cp.spawnSync('cp', [
+      dotNodePath,
+      'packed/' + path.basename(dotNodePath)
+    ], {shell: true})
+  })
+  process.exit(0)
+})
diff --git a/package.json b/package.json
index 8bf0b80..b251e8b 100644
--- a/package.json
+++ b/package.json
@@ -6,19 +6,9 @@
   "license": "MIT",
   "scripts": {
     "dev": "webpack -w",
-    "pack": "webpack && pkg dist/ooni.js -c package.json -o packed/ooni",
+    "pack": "webpack && node pack.js",
     "clean-home": "rm -rf ~/.ooni/measurements && rm -rf ~/.ooni/*.ldb"
   },
-  "pkg": {
-    "scripts": [
-      "dist/*"
-    ],
-    "targets": [
-      "node7-linux-x64",
-      "node7-macos-x64",
-      "node7-win-x64"
-    ]
-  },
   "devDependencies": {
     "ansi-escapes": "^3.0.0",
     "axios": "^0.17.1",

From 760038e726d103d21a1447d058cfd364c3ef99d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Fri, 12 Jan 2018 17:06:46 +0100
Subject: [PATCH 17/17] Emit messages via notify

---
 src/commands/run.js                    | 4 ++--
 src/nettests/index.js                  | 4 +++-
 src/nettests/web-connectivity/index.js | 2 +-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/commands/run.js b/src/commands/run.js
index 1b7e994..a954eec 100644
--- a/src/commands/run.js
+++ b/src/commands/run.js
@@ -72,8 +72,7 @@ const run = async ({camelName, argv}) => {
   })
   dbOperations.push(result.save())
 
-  notify({key: 'starting-test', value: camelName})
-
+  notify({key: 'ooni.run.nettest.starting', value: camelName})
   console.log(info('Running '+
               chalk.bold(`${nettestType.nettests.length} ${nettestType.name} `) +
               `test${sOrNot}`))
@@ -85,6 +84,7 @@ const run = async ({camelName, argv}) => {
   for (const nettestLoader of nettestType.nettests) {
     const loader = nettestLoader()
     const { nettest, meta } = loader
+    notify({key: 'ooni.run.nettest.running', value: meta})
     console.log(info(`${chalk.bold(meta.name)}`))
     const [measurements, dataUsage] = await nettest.run({
       ooni: makeOoni(loader, geoip),
diff --git a/src/nettests/index.js b/src/nettests/index.js
index ad0365c..eaf5c4f 100644
--- a/src/nettests/index.js
+++ b/src/nettests/index.js
@@ -8,12 +8,12 @@ import wait from '../cli/output/wait'
 
 import { Measurement } from '../config/db'
 import { getOoniDir } from '../config/global-path'
+import { notify } from '../config/ipc'
 
 import iso8601 from '../util/iso8601'
 import randInt from '../util/randInt'
 
 const debug = require('debug')('nettests.index')
-
 const OONI_DIR = getOoniDir()
 
 const makeReportFile = (name) => {
@@ -117,6 +117,8 @@ export const makeOoni = (loader, geoip) => {
   const onProgress = (percent, message, persist) => {
     progress && progress()
     progress = wait(`${percentage(percent)}: ${message}`, persist)
+    notify({key: 'ooni.run.progress.percent', value: percent})
+    notify({key: 'ooni.run.progress.message', value: message})
   }
 
   const run = async (runner) => {
diff --git a/src/nettests/web-connectivity/index.js b/src/nettests/web-connectivity/index.js
index 01d6ed4..d4d1382 100644
--- a/src/nettests/web-connectivity/index.js
+++ b/src/nettests/web-connectivity/index.js
@@ -39,7 +39,7 @@ export const run = ({ooni, argv}) => {
 
   webConnectivity.on('begin', () => ooni.onProgress(0.0, 'starting web-connectivity'))
   webConnectivity.on('progress', (percent, message) => {
-    ooni.onProgress(percent, message, persist)
+    ooni.onProgress(percent, message)
   })
   return ooni.run(webConnectivity.run)
 }