Merge pull request #27 from 1Hive/worker-improvements

Various improvements to worker

worker/src/classifiers/event.js

@@ -4,6 +4,7 @@ import topics from '../data/topics'
 
 export default function * () {
   yield takeEvery('daolist/eth/LOG', function * ({ payload: event }) {
+    const timestamp = event.timestamp
     switch (event.topics[0]) {
       case topics.NEW_APP_PROXY:
         const { proxy, appId } = abi.decodeLog(
@@ -27,7 +28,8 @@ export default function * () {
 
         const appInstall = {
           proxy,
-          appId
+          appId,
+          timestamp
         }
 
         yield put({
@@ -55,7 +57,8 @@ export default function * () {
         const newVersion = {
           id: versionId,
           semantic: semanticVersion,
-          repository: event.address
+          repository: event.address,
+          timestamp
         }
 
         yield put({

worker/src/classifiers/transaction.js

@@ -48,7 +48,8 @@ export default function * () {
         kit: transaction.to,
         creator: transaction.from,
         transaction: transaction.hash,
-        name: `${name}.aragonid.eth`
+        name: `${name}.aragonid.eth`,
+        timestamp: transaction.timestamp
       }
     })
   }

worker/src/fetchers/block.js

@@ -1,5 +1,4 @@
-import { delay } from 'redux-saga'
-import { getContext, put, all } from 'redux-saga/effects'
+import { getContext, put, all, delay } from 'redux-saga/effects'
 import _ from 'lodash'
 import { promisify } from 'util'
 

worker/src/fetchers/logs.js

@@ -8,6 +8,7 @@ export default function * () {
     const { logs } = yield web3.eth.getTransactionReceipt(transaction.hash)
 
     for (const log of logs) {
+      log.timestamp = transaction.timestamp
       yield put({
         type: 'daolist/eth/LOG',
         payload: log

worker/src/fetchers/transaction.js

@@ -3,10 +3,11 @@ import { take, put } from 'redux-saga/effects'
 export default function * transactionFetcher () {
   while (true) {
     const {
-      payload: { transactions }
+      payload: block
     } = yield take('daolist/eth/BLOCK')
 
-    for (const transaction of transactions) {
+    for (const transaction of block.transactions) {
+      transaction.timestamp = block.timestamp
       yield put({
         type: 'daolist/eth/TRANSACTION',
         payload: transaction

worker/src/persisters/app/install.js

@@ -27,6 +27,9 @@ export default function * () {
             id: appInstall.appId,
             address: appInstall.proxy
           }
+        },
+        $min: {
+          created_at: appInstall.timestamp
         }
       }
     )

worker/src/persisters/app/metadata.js

@@ -1,4 +1,4 @@
-import { getContext, takeEvery } from 'redux-saga/effects'
+import { getContext, takeEvery, retry } from 'redux-saga/effects'
 import abi from 'web3-eth-abi'
 import got from 'got'
 import path from 'path'
@@ -35,6 +35,8 @@ const KNOWN_APPS_BY_URI = {
   }
 }
 
+const NULL_ADDRESS = '0x0000000000000000000000000000000000000000'
+
 export async function fetchVersion (web3, repository, versionId) {
   const call = abi.encodeFunctionCall(
     {
@@ -73,8 +75,15 @@ export async function fetchVersion (web3, repository, versionId) {
     result
   )
 
-  const [type, uri] = web3.utils.hexToAscii(contentURI).split(':')
+  const content = web3.utils.hexToAscii(contentURI)
+  if (!content.includes(':') && contractAddress === NULL_ADDRESS) {
+    // If the content URI is malformed and there is no implementation address,
+    // then we skip this app as it is possibly another type of content published
+    // over APM.
+    return null
+  }
 
+  const [type, uri] = content.split(':')
   const isKnownApp = !!KNOWN_APPS_BY_URI[uri]
 
   let manifest = {}
@@ -131,20 +140,26 @@ export default function * () {
     payload: newVersion
   }) {
     log.info('New version for app', newVersion)
-
-    const { app, version } = yield fetchVersion(
+    const versionMeta = yield retry(
+      3,
+      3000,
+      fetchVersion,
       web3,
       newVersion.repository,
       newVersion.id
     )
 
+    if (versionMeta === null) {
+      return
+    }
+
     // Add app to org
     yield apps.updateOne(
       { address: newVersion.repository },
       {
-        $set: app,
+        $set: versionMeta.app,
         $addToSet: {
-          versions: version
+          versions: versionMeta.version
         }
       },
       { upsert: true }

worker/src/persisters/dao/name.js

@@ -1,4 +1,4 @@
-import { getContext, take } from 'redux-saga/effects'
+import { getContext, take, retry } from 'redux-saga/effects'
 import { safeUpsert } from '../../utils/index'
 
 export default function * () {
@@ -12,9 +12,18 @@ export default function * () {
 
     log.info('DAO name set', { dao })
 
-    const address = yield web3.eth.ens.getAddress(dao.name)
+    const address = yield retry(3, 3000, () => web3.eth.ens.getAddress(dao.name))
     dao.address = address
 
-    yield safeUpsert(orgs, { address }, { $set: dao })
+    yield safeUpsert(
+      orgs,
+      { address },
+      {
+        $set: dao,
+        $min: {
+          created_at: dao.timestamp
+        }
+      }
+    )
   }
 }