Feature/mk 0.8.1

pack.js

@@ -0,0 +1,40 @@
+const cp = require('child_process')
+const os = require('os')
+const path = require('path')
+
+const pkg = require('pkg')
+
+const nodeTargets = {
+  darwin: "node8-macos-x64",
+  linux: "node8-linux-x64",
+  win32: "node8-win-x64",
+}
+
+const nativeModules = {
+  darwin: [
+    'node_modules/measurement-kit/build/Release/measurement-kit.node',
+    'node_modules/sqlite3/lib/binding/node-v57-darwin-x64/node_sqlite3.node'
+  ],
+  win32: [],
+  linux: []
+}
+
+const platform = os.platform()
+
+if (!nodeTargets[platform]) {
+  console.log('this platform is not supported')
+  process.exit(0)
+}
+
+console.log('- building packed/ooni for ' + nodeTargets[platform])
+pkg.exec([ 'dist/ooni.js', '--target', nodeTargets[platform], '-o', 'packed/ooni' ])
+.then(() => {
+  nativeModules[platform].forEach(dotNodePath => {
+    console.log('- copying ' + dotNodePath)
+    cp.spawnSync('cp', [
+      dotNodePath,
+      'packed/' + path.basename(dotNodePath)
+    ], {shell: true})
+  })
+  process.exit(0)
+})

package.json

@@ -6,21 +6,12 @@
   "license": "MIT",
   "scripts": {
     "dev": "webpack -w",
-    "pack": "webpack && pkg dist/ooni.js -c package.json -o packed/ooni",
+    "pack": "webpack && node pack.js",
     "clean-home": "rm -rf ~/.ooni/measurements && rm -rf ~/.ooni/*.ldb"
   },
-  "pkg": {
-    "scripts": [
-      "dist/*"
-    ],
-    "targets": [
-      "node7-linux-x64",
-      "node7-macos-x64",
-      "node7-win-x64"
-    ]
-  },
   "devDependencies": {
     "ansi-escapes": "^3.0.0",
+    "axios": "^0.17.1",
     "babel-core": "^6.26.0",
     "babel-loader": "^7.1.2",
     "babel-preset-es2015": "^6.24.1",
@@ -49,9 +40,10 @@
     "webpack": "^3.6.0",
     "webpack-node-externals": "^1.6.0",
     "window-size": "^1.1.0",
-    "wrap-ansi": "^3.0.1"
+    "wrap-ansi": "^3.0.1",
+    "zlib": "^1.0.5"
   },
   "dependencies": {
-    "measurement-kit": "^0.1.0-alpha.5"
+    "measurement-kit": "^0.1.0-alpha.8"
   }
 }

scripts/gen-nettests.py

@@ -0,0 +1,102 @@
+import os
+import pystache
+# pip install https://github.com/okunishinishi/python-stringcase/archive/cc7d5eb58ff4a959a508c3f2296459daf7c3a1f2.zip
+import stringcase
+
+PKG_TMPL = """
+{
+  "name": "ooni-{{nettest_dash_case}}",
+  "version": "0.3.0",
+  "description": "Official OONI test for running {{nettest_name}}",
+  "dependencies": {
+    "measurement-kit": "0.1.0"
+  }
+}
+"""
+
+INDEX_TMPL = """
+import { {{nettest_pascal_case}} } from 'measurement-kit'
+
+export const renderSummary = (measurements, {React, Cli, Components, chalk}) => {
+  const summary = measurements[0].summary
+
+  // When this function is called from the Cli the Cli will be set, when it's
+  // called from the Desktop app we have React set instead.
+  if (Cli) {
+    Cli.log(Cli.output.labelValue('Label', uploadMbit, {unit: 'Mbit'}))
+  } else if (React) {
+    /*
+    XXX this is broken currently as it depends on react
+    const {
+      Container,
+      Heading
+    } = Components
+    return class extends React.Component {
+      render() {
+        return <Container>
+          <Heading h={1}>Results for NDT</Heading>
+          {uploadMbit}
+        </Container>
+      }
+    }
+    */
+  }
+}
+
+export const renderHelp = () => {
+}
+
+export const makeSummary = ({test_keys}) => ({
+})
+
+export const run = ({ooni, argv}) => {
+  const {{nettest_camel_case}} = {{nettest_pascal_case}}(ooni.mkOptions)
+  ooni.init({{nettest_camel_case}})
+
+  {{nettest_camel_case}}.on('begin', () => ooni.onProgress(0.0, 'starting {{nettest_dash_case}}'))
+  {{nettest_camel_case}}.on('progress', (percent, message) => {
+    ooni.onProgress(percent, message, persist)
+  })
+  return ooni.run({{nettest_camel_case}}.run)
+}
+"""
+
+nettests = [
+    ['Web Connectivity', 'web-connectivity'],
+    ['HTTP Invalid Request Line', 'http-invalid-request-line'],
+    ['HTTP Header Field Manipulation', 'http-header-field-manipulation'],
+    #['NDT', 'ndt'],
+    ['Dash', 'dash'],
+    ['Facebook Messenger', 'facebook-messenger'],
+    ['Telegram', 'telegram'],
+    ['WhatsApp', 'whatsapp']
+]
+
+def gen():
+    for nettest_name, nettest_dash_case in nettests:
+        nettest_camel_case = stringcase.camelcase(nettest_dash_case)
+        nettest_pascal_case = stringcase.pascalcase(nettest_dash_case)
+        partials = dict(
+            nettest_dash_case=nettest_dash_case,
+            nettest_camel_case=nettest_camel_case,
+            nettest_pascal_case=nettest_pascal_case,
+            nettest_name=nettest_name
+        )
+        dst_path = os.path.join(os.path.dirname(__file__), '..', 'src', 'nettests', nettest_dash_case)
+        if not os.path.exists(dst_path):
+            os.mkdir(dst_path)
+
+        index_path = os.path.join(dst_path, 'index.js')
+        package_path = os.path.join(dst_path, 'package.json')
+        with open(index_path, 'w+') as out_file:
+            out_file.write(pystache.render(INDEX_TMPL, partials))
+        print('wrote {}'.format(index_path))
+        with open(package_path, 'w+') as out_file:
+            out_file.write(pystache.render(PKG_TMPL, partials))
+        print('wrote {}'.format(package_path))
+
+def main():
+    gen()
+
+if __name__ == '__main__':
+    main()

src/cli/make-cli.js

@@ -2,6 +2,8 @@ export const makeCli = (log = console.log) => ({
   output: {
     toMbit: require('./output/to-mbit').default,
     labelValue: require('./output/label-value').default,
+    ok: require('./output/ok'),
+    notok: require('./output/notok'),
   },
   log
 })

src/cli/output/test-results.js

@@ -55,9 +55,10 @@ import labelValue from './label-value'
 */
 
 const testResults = async (results, getMeta) => {
-  const colWidth = 50
+  const colWidth = 76
   let o = '┏' + '━'.repeat(colWidth) + '┓\n'
-  let totalDataUsage = 0
+  let totalDataUsageUp = 0
+  let totalDataUsageDown = 0
   let totalRows = 0
   let allAsns = []
   let allCountries = []
@@ -79,23 +80,26 @@ const testResults = async (results, getMeta) => {
     if (allAsns.indexOf(meta.asn) === -1) {
       allAsns.push(meta.asn)
     }
-    totalDataUsage += meta.dataUsage
+    if (meta.dataUsageUp && meta.dataUsageDown) {
+      totalDataUsageUp += meta.dataUsageUp
+      totalDataUsageDown += meta.dataUsageDown
+    }
     totalRows += 1
 
     let firstRow = `${chalk.bold(`#${r.id}`)} - ${moment(meta.date).fromNow()}`
     firstRow += rightPad(firstRow, innerWidth)
     let secondRow = meta.name
-    secondRow += rightPad(secondRow, 26)
+    secondRow += rightPad(secondRow, colWidth/2)
     secondRow += meta.summary[0] || ''
     secondRow += rightPad(secondRow, innerWidth)
 
     let thirdRow = meta.network
-    thirdRow += rightPad(thirdRow, 26)
+    thirdRow += rightPad(thirdRow, colWidth/2)
     thirdRow += meta.summary[1] || ''
     thirdRow += rightPad(thirdRow, innerWidth)
 
     let fourthRow = `${chalk.cyan(meta.asn)} (${chalk.cyan(meta.country)})`
-    fourthRow += rightPad(fourthRow, 26)
+    fourthRow += rightPad(fourthRow, colWidth/2)
     fourthRow += meta.summary[2] || ''
     fourthRow += rightPad(fourthRow, innerWidth)
 
@@ -110,8 +114,11 @@ const testResults = async (results, getMeta) => {
     results.map(getContentRow)
   )
 
-  let dataUsageCell = `${humanize.filesize(totalDataUsage)}`
-  dataUsageCell += rightPad(dataUsageCell, 12)
+  let dataUsageDownCell = `U ${humanize.filesize(totalDataUsageDown)}`
+  dataUsageDownCell += rightPad(dataUsageDownCell, 12)
+
+  let dataUsageUpCell = `D ${humanize.filesize(totalDataUsageUp)}`
+  dataUsageUpCell += rightPad(dataUsageUpCell, 12)
 
   let networksCell = `${allAsns.length} nets`
   networksCell += rightPad(networksCell, 12)
@@ -120,10 +127,16 @@ const testResults = async (results, getMeta) => {
   msmtsCell += rightPad(msmtsCell, 12)
 
   o += contentRows.join('┢' + '━'.repeat(colWidth) + '┪\n')
-  o += '└┬──────────────┬──────────────┬──────────────┬'+'─'.repeat(colWidth - 46)+'┘\n'
-  o += ` │ ${msmtsCell} │ ${networksCell} │ ${dataUsageCell} │
- ╰──────────────┴──────────────┴──────────────╯
-`
+
+  if (totalDataUsageDown && totalDataUsageUp) {
+    o += '└┬──────────────┬──────────────┬──────────────┬──────────────┬'+'─'.repeat(colWidth - 61)+'┘\n'
+    o += ` │ ${msmtsCell} │ ${networksCell} │ ${dataUsageDownCell} │ ${dataUsageUpCell} │\n`
+    o += ' ╰──────────────┴──────────────┴──────────────┴──────────────╯'
+  } else {
+    o += '└┬──────────────┬──────────────┬'+'─'.repeat(colWidth - 31)+'┘\n'
+    o += ` │ ${msmtsCell} │ ${networksCell} │\n`
+    o += ' ╰──────────────┴──────────────╯'
+  }
   return o
 }
 

src/commands/list.js

@@ -87,13 +87,14 @@ const listAction = async ctx => {
         summary.push(m)
       })
       // XXX we should figure out how this will work when we have many measurements
-      nettest.renderSummary([measurement], {Cli, chalk})
+      if (measurement.summary) {
+        nettest.renderSummary([measurement], {Cli, chalk})
+      }
       return {
         name: measurement.name,
         network: measurement.asn,
         asn: measurement.asn,
         country: measurement.country,
-        dataUsage: measurement.dataUsage,
         date: measurement.startTime,
         summary: summary
       }
@@ -118,7 +119,8 @@ const listAction = async ctx => {
         network: measurements[0].asn,
         asn: measurements[0].asn,
         country: measurements[0].country,
-        dataUsage: measurements.map(m => m.dataUsage).reduce((a,b) => a += b),
+        dataUsageUp: result.dataUsageUp,
+        dataUsageDown: result.dataUsageDown,
         date: result.startTime,
         summary: summary
       }

src/commands/nettest.js

@@ -118,7 +118,6 @@ const main = async ctx => {
     }
     await exit(1)
   }
-
 }
 
 export default main

src/commands/run.js

@@ -18,8 +18,14 @@ import {
   Result
 } from '../config/db'
 
+import {
+  notify
+} from '../config/ipc'
+
 import makeCli from '../cli/make-cli'
 
+import { getGeoipPaths } from '../config/geoip'
+
 const debug = require('debug')('commands.run')
 
 const help = () => {
@@ -66,30 +72,51 @@ const run = async ({camelName, argv}) => {
   })
   dbOperations.push(result.save())
 
+  notify({key: 'ooni.run.nettest.starting', value: camelName})
   console.log(info('Running '+
               chalk.bold(`${nettestType.nettests.length} ${nettestType.name} `) +
               `test${sOrNot}`))
 
+  let dataUsageUp = 0
+  let dataUsageDown = 0
+  const geoip = await getGeoipPaths()
+
   for (const nettestLoader of nettestType.nettests) {
     const loader = nettestLoader()
     const { nettest, meta } = loader
+    notify({key: 'ooni.run.nettest.running', value: meta})
     console.log(info(`${chalk.bold(meta.name)}`))
-    const measurements = await nettest.run({ooni: makeOoni(loader), argv})
+    const [measurements, dataUsage] = await nettest.run({
+      ooni: makeOoni(loader, geoip),
+      argv
+    })
+    debug('setting data usage', dataUsage)
+    dataUsageUp += dataUsage.up || 0
+    dataUsageDown += dataUsage.down || 0
     nettest.renderSummary(measurements, {
       Cli: makeCli(),
       chalk: chalk,
       Components: null,
       React: null,
     })
-    dbOperations.push(result.setMeasurements(measurements))
+
+    for (const measurement of measurements) {
+      dbOperations.push(result.addMeasurements(measurement))
+    }
   }
   await Promise.all(dbOperations)
 
   const measurements = await result.getMeasurements()
+  debug('updating the result table')
+  debug(measurements)
+  const summary = nettestType.makeSummary(measurements)
+  debug('summary: ', summary)
   await result.update({
-    summary: nettestType.makeSummary(measurements),
     endTime: moment.utc().toDate(),
-    done: true
+    done: true,
+    dataUsageUp: dataUsageUp,
+    dataUsageDown: dataUsageDown,
+    summary
   })
 }
 
@@ -140,6 +167,7 @@ const main = async ctx => {
       await exit(0)
     } else {
       await run({camelName, argv})
+      await exit(0)
     }
   } catch(err) {
     if (err.usageError) {