server.js

"use strict";
const stream      = require('stream');
const async       = require('async');
const util        = require('util');
const path        = require('path');
const co          = require('co');
const _           = require('underscore');
const Q           = require('q');
const archiver    = require('archiver');
const unzip       = require('unzip2');
const fs          = require('fs');
const parsers     = require('./app/lib/streams/parsers');
const constants   = require('./app/lib/constants');
const fileDAL     = require('./app/lib/dal/fileDAL');
const jsonpckg    = require('./package.json');
const router      = require('./app/lib/streams/router');
const base58      = require('./app/lib/crypto/base58');
const keyring      = require('./app/lib/crypto/keyring');
const directory   = require('./app/lib/system/directory');
const dos2unix    = require('./app/lib/system/dos2unix');
const Synchroniser = require('./app/lib/sync');
const multicaster = require('./app/lib/streams/multicaster');
const upnp        = require('./app/lib/system/upnp');
const bma         = require('./app/lib/streams/bma');
const rawer       = require('./app/lib/ucp/rawer');

function Server (dbConf, overrideConf) {

  stream.Duplex.call(this, { objectMode: true });

  const home = directory.getHome(dbConf.name, dbConf.home);
  const paramsP = directory.getHomeParams(dbConf && dbConf.memory, home);
  const logger = require('./app/lib/logger')('server');
  const that = this;
  that.home = home;
  that.conf = null;
  that.dal = null;
  that.version = jsonpckg.version;

  that.MerkleService       = require("./app/lib/helpers/merkle");
  that.ParametersService   = require("./app/lib/helpers/parameters")();
  that.IdentityService     = require('./app/service/IdentityService')();
  that.MembershipService   = require('./app/service/MembershipService')();
  that.PeeringService      = require('./app/service/PeeringService')(that);
  that.BlockchainService   = require('./app/service/BlockchainService')(that);
  that.TransactionsService = require('./app/service/TransactionsService')();

  // Create document mapping
  const documentsMapping = {
    'identity':      { action: that.IdentityService.submitIdentity,                                               parser: parsers.parseIdentity },
    'certification': { action: that.IdentityService.submitCertification,                                          parser: parsers.parseCertification},
    'revocation':    { action: that.IdentityService.submitRevocation,                                             parser: parsers.parseRevocation },
    'membership':    { action: that.MembershipService.submitMembership,                                           parser: parsers.parseMembership },
    'peer':          { action: that.PeeringService.submitP,                                                       parser: parsers.parsePeer },
    'transaction':   { action: that.TransactionsService.processTx,                                                parser: parsers.parseTransaction },
    'block':         { action: _.partial(that.BlockchainService.submitBlock, _, true, constants.NO_FORK_ALLOWED), parser: parsers.parseBlock }
  };

  // Unused, but made mandatory by Duplex interface
  this._read = () => null;

  this._write = (obj, enc, writeDone) => that.submit(obj, false, () => writeDone);

  /**
   * Facade method to control what is pushed to the stream (we don't want it to be closed)
   * @param obj An object to be pushed to the stream.
   */
  this.streamPush = (obj) => {
    if (obj) {
      that.push(obj);
    }
  };

  this.plugFileSystem = () => co(function *() {
    logger.debug('Plugging file system...');
    const params = yield paramsP;
    that.dal = fileDAL(params);
  });

  this.unplugFileSystem = () => co(function *() {
    logger.debug('Unplugging file system...');
    yield that.dal.close();
  });

  this.loadConf = (useDefaultConf) => co(function *() {
    logger.debug('Loading conf...');
    that.conf = yield that.dal.loadConf(overrideConf, useDefaultConf);
    // Default values
    const defaultValues = {
      remoteipv6:         that.conf.ipv6,
      remoteport:         that.conf.port,
      cpu:                constants.DEFAULT_CPU,
      c:                  constants.CONTRACT.DEFAULT.C,
      dt:                 constants.CONTRACT.DEFAULT.DT,
      ud0:                constants.CONTRACT.DEFAULT.UD0,
      stepMax:            constants.CONTRACT.DEFAULT.STEPMAX,
      sigPeriod:          constants.CONTRACT.DEFAULT.SIGPERIOD,
      sigStock:           constants.CONTRACT.DEFAULT.SIGSTOCK,
      sigWindow:          constants.CONTRACT.DEFAULT.SIGWINDOW,
      sigValidity:        constants.CONTRACT.DEFAULT.SIGVALIDITY,
      msValidity:         constants.CONTRACT.DEFAULT.MSVALIDITY,
      sigQty:             constants.CONTRACT.DEFAULT.SIGQTY,
      idtyWindow:         constants.CONTRACT.DEFAULT.IDTYWINDOW,
      msWindow:           constants.CONTRACT.DEFAULT.MSWINDOW,
      xpercent:           constants.CONTRACT.DEFAULT.X_PERCENT,
      percentRot:         constants.CONTRACT.DEFAULT.PERCENTROT,
      blocksRot:          constants.CONTRACT.DEFAULT.BLOCKSROT,
      powDelay:           constants.CONTRACT.DEFAULT.POWDELAY,
      avgGenTime:         constants.CONTRACT.DEFAULT.AVGGENTIME,
      dtDiffEval:         constants.CONTRACT.DEFAULT.DTDIFFEVAL,
      medianTimeBlocks:   constants.CONTRACT.DEFAULT.MEDIANTIMEBLOCKS,
      rootoffset:         0,
      forksize:           constants.BRANCHES.DEFAULT_WINDOW_SIZE
    };
    _.keys(defaultValues).forEach(function(key){
      if (that.conf[key] == undefined) {
        that.conf[key] = defaultValues[key];
      }
    });
    logger.debug('Loading crypto functions...');
    // Extract key pair
    let keyPair = null;
    const keypairOverriden = overrideConf && (overrideConf.salt || overrideConf.passwd);
    if (!keypairOverriden && that.conf.pair) {
      keyPair = keyring.Key(that.conf.pair.pub, that.conf.pair.sec);
    }
    else if (that.conf.passwd || that.conf.salt) {
      keyPair = yield keyring.scryptKeyPair(that.conf.salt, that.conf.passwd);
    }
    else {
      keyPair = keyring.Key(constants.CRYPTO.DEFAULT_KEYPAIR.pub,
          constants.CRYPTO.DEFAULT_KEYPAIR.sec);
    }
    if (!keyPair) {
      throw Error('This node does not have a keypair. Use `duniter wizard key` to fix this.');
    }
    that.keyPair = keyPair;
    that.sign = keyPair.sign;
    // Update services
    [that.IdentityService, that.MembershipService, that.PeeringService, that.BlockchainService, that.TransactionsService].map((service) => {
      service.setConfDAL(that.conf, that.dal, that.keyPair);
    });
    that.router().setConfDAL(that.conf, that.dal);
    return that.conf;
  });

  this.initWithDAL = () => co(function *() {
    yield that.plugFileSystem();
    yield that.loadConf();
    yield that.initDAL();
    return that;
  });

  this.submit = (obj, isInnerWrite, done) => {
    return co(function *() {
      if (!obj.documentType) {
        throw 'Document type not given';
      }
      try {
        const action = documentsMapping[obj.documentType].action;
        let res;
        if (typeof action == 'function') {
          // Handle the incoming object
          res = yield action(obj);
        } else {
          throw 'Unknown document type \'' + obj.documentType + '\'';
        }
        if (res) {
          // Only emit valid documents
          that.emit(obj.documentType, _.clone(res));
          that.streamPush(_.clone(res));
        }
        if (done) {
          isInnerWrite ? done(null, res) : done();
        }
        return res;
      } catch (err) {
        logger.debug('Document write error: ', err);
        if (done) {
          isInnerWrite ? done(err, null) : done();
        } else {
          throw err;
        }
      }
    });
  };

  this.submitP = (obj, isInnerWrite) => Q.nbind(this.submit, this)(obj, isInnerWrite);

  this.initDAL = () => this.dal.init();

  this.start = () => co(function*(){
    yield that.checkConfig();
    // Add signing & public key functions to PeeringService
    logger.info('Node version: ' + that.version);
    logger.info('Node pubkey: ' + that.PeeringService.pubkey);
    return that.initPeer();
  });

  this.stop = () => {
    that.BlockchainService.stopCleanMemory();
    return that.PeeringService.stopRegular();
  };

  this.recomputeSelfPeer = () => that.PeeringService.generateSelfPeer(that.conf, 0);

  this.initPeer = () => co(function*(){
      yield that.checkConfig();
      yield Q.nbind(that.PeeringService.regularCrawlPeers, that.PeeringService);
      logger.info('Storing self peer...');
      yield that.PeeringService.regularPeerSignal();
      yield Q.nbind(that.PeeringService.regularTestPeers, that.PeeringService);
      yield Q.nbind(that.PeeringService.regularSyncBlock, that.PeeringService);
      yield Q.nbind(that.BlockchainService.regularCleanMemory, that.BlockchainService);
  });

  let shouldContinue = false;

  this.stopBlockComputation = () => {
    shouldContinue = false;
    that.BlockchainService.stopPoWThenProcessAndRestartPoW();
  };
  
  this.getCountOfSelfMadePoW = () => this.BlockchainService.getCountOfSelfMadePoW();
  this.isServerMember = () => this.BlockchainService.isMember();

  this.isPoWPaused = true;
  this._blockComputation = () => co(function *() {
    while (shouldContinue) {
      try {
        const block = yield that.BlockchainService.startGeneration();
        if (block && shouldContinue) {
          try {
            const obj = parsers.parseBlock.syncWrite(dos2unix(block.getRawSigned()));
            yield that.singleWritePromise(obj);
          } catch (err) {
            logger.warn('Proof-of-work self-submission: %s', err.message || err);
          }
        }
      }
      catch (e) {
        that.isPoWPaused = true;
        logger.error(e);
        shouldContinue = true;
      }
    }
    logger.info('Proof-of-work computation STOPPED.');
  });

  this.startBlockComputation = () => {
    shouldContinue = true;
    return that._blockComputation();
  };

  this.checkConfig = () => {
    return that.checkPeeringConf(that.conf);
  };

  this.checkPeeringConf = (conf) => co(function*() {
      if (!conf.pair && conf.passwd == null) {
        throw new Error('No key password was given.');
      }
      if (!conf.pair && conf.salt == null) {
        throw new Error('No key salt was given.');
      }
      if (!conf.currency) {
        throw new Error('No currency name was given.');
      }
      if(!conf.ipv4 && !conf.ipv6){
        throw new Error("No interface to listen to.");
      }
      if(!conf.remoteipv4 && !conf.remoteipv6 && !conf.remotehost){
        throw new Error('No interface for remote contact.');
      }
      if (!conf.remoteport) {
        throw new Error('No port for remote contact.');
      }
  });

  this.resetHome = () => co(function *() {
    const params = yield paramsP;
    const myFS = params.fs;
    const rootPath = params.home;
    const existsDir = yield myFS.exists(rootPath);
    if (existsDir) {
      yield myFS.removeTree(rootPath);
    }
  });

  this.resetAll = (done) => co(function*() {
    const files = ['stats', 'cores', 'current', directory.DUNITER_DB_NAME, directory.DUNITER_DB_NAME + '.db', directory.DUNITER_DB_NAME + '.log', directory.WOTB_FILE, 'export.zip', 'import.zip', 'conf'];
    const dirs  = ['blocks', 'ud_history', 'branches', 'certs', 'txs', 'cores', 'sources', 'links', 'ms', 'identities', 'peers', 'indicators', 'leveldb'];
    return resetFiles(files, dirs, done);
  });

  this.resetData = (done) => co(function*(){
    const files = ['stats', 'cores', 'current', directory.DUNITER_DB_NAME, directory.DUNITER_DB_NAME + '.db', directory.DUNITER_DB_NAME + '.log', directory.WOTB_FILE];
    const dirs  = ['blocks', 'ud_history', 'branches', 'certs', 'txs', 'cores', 'sources', 'links', 'ms', 'identities', 'peers', 'indicators', 'leveldb'];
    yield resetFiles(files, dirs, done);
  });

  this.resetConf = (done) => {
    const files = ['conf'];
    const dirs  = [];
    return resetFiles(files, dirs, done);
  };

  this.resetStats = (done) => {
    const files = ['stats'];
    const dirs  = ['ud_history'];
    return resetFiles(files, dirs, done);
  };

  this.resetPeers = (done) => {
    return that.dal.resetPeers(done);
  };

  this.exportAllDataAsZIP = () => co(function *() {
    const params = yield paramsP;
    const rootPath = params.home;
    const myFS = params.fs;
    const archive = archiver('zip');
    if (yield myFS.exists(path.join(rootPath, 'indicators'))) {
      archive.directory(path.join(rootPath, 'indicators'), '/indicators', undefined, { name: 'indicators'});
    }
    const files = ['duniter.db', 'stats.json', 'wotb.bin'];
    for (const file of files) {
      if (yield myFS.exists(path.join(rootPath, file))) {
        archive.file(path.join(rootPath, file), { name: file });
      }
    }
    archive.finalize();
    return archive;
  });

  this.importAllDataFromZIP = (zipFile) => co(function *() {
    const params = yield paramsP;
    yield that.resetData();
    const output = unzip.Extract({ path: params.home });
    fs.createReadStream(zipFile).pipe(output);
    return new Promise((resolve, reject) => {
      output.on('error', reject);
      output.on('close', resolve);
    });
  });

  this.cleanDBData = () => co(function *() {
    yield that.dal.cleanCaches();
    that.dal.wotb.resetWoT();
    const files = ['stats', 'cores', 'current', directory.DUNITER_DB_NAME, directory.DUNITER_DB_NAME + '.db', directory.DUNITER_DB_NAME + '.log'];
    const dirs  = ['blocks', 'ud_history', 'branches', 'certs', 'txs', 'cores', 'sources', 'links', 'ms', 'identities', 'peers', 'indicators', 'leveldb'];
    return resetFiles(files, dirs);
  });

  function resetFiles(files, dirs, done) {
    return co(function *() {
      const params = yield paramsP;
      const myFS = params.fs;
      const rootPath = params.home;
      for (const fName of files) {
        // JSON file?
        const existsJSON = yield myFS.exists(rootPath + '/' + fName + '.json');
        if (existsJSON) {
          const theFilePath = rootPath + '/' + fName + '.json';
          yield myFS.remove(theFilePath);
          if (yield myFS.exists(theFilePath)) {
            throw Error('Failed to delete file "' + theFilePath + '"');
          }
        } else {
          // Normal file?
          const normalFile = path.join(rootPath, fName);
          const existsFile = yield myFS.exists(normalFile);
          if (existsFile) {
            yield myFS.remove(normalFile);
            if (yield myFS.exists(normalFile)) {
              throw Error('Failed to delete file "' + normalFile + '"');
            }
          }
        }
      }
      for (const dirName of dirs) {
        const existsDir = yield myFS.exists(rootPath + '/' + dirName);
        if (existsDir) {
          yield myFS.removeTree(rootPath + '/' + dirName);
          if (yield myFS.exists(rootPath + '/' + dirName)) {
            throw Error('Failed to delete folder "' + rootPath + '/' + dirName + '"');
          }
        }
      }
      done && done();
    })
        .catch((err) => {
          done && done(err);
          throw err;
        });
  }

  this.disconnect = () => Promise.resolve(that.dal && that.dal.close());

  this.pullBlocks = that.PeeringService.pullBlocks;

  this.doMakeNextBlock = (manualValues) => that.BlockchainService.makeNextBlock(null, null, manualValues);

  this.doCheckBlock = (block) => {
    const parsed = parsers.parseBlock.syncWrite(block.getRawSigned());
    return that.BlockchainService.checkBlock(parsed, false);
  };

  this.revert = () => this.BlockchainService.revertCurrentBlock();

  this.revertTo = (number) => co(function *() {
    const current = yield that.BlockchainService.current();
    for (let i = 0, count = current.number - number; i < count; i++) {
      yield that.BlockchainService.revertCurrentBlock();
    }
    if (current.number <= number) {
      logger.warn('Already reached');
    }
  });

  this.singleWritePromise = (obj) => that.submit(obj);

  let theRouter;

  this.router = (active) => {
    if (!theRouter) {
      theRouter = router(that.PeeringService, that.conf, that.dal);
    }
    theRouter.setActive(active !== false);
    return theRouter;
  };

  /**
   * Synchronize the server with another server.
   *
   * If local server's blockchain is empty, process a fast sync: **no block is verified in such a case**, unless
   * you force value `askedCautious` to true.
   *
   * @param onHost Syncs on given host.
   * @param onPort Syncs on given port.
   * @param upTo Sync up to this number, if `upTo` value is a positive integer.
   * @param chunkLength Length of each chunk of blocks to download. Kind of buffer size.
   * @param interactive Tell if the loading bars should be used for console output.
   * @param askedCautious If true, force the verification of each downloaded block. This is the right way to have a valid blockchain for sure.
   * @param nopeers If true, sync will omit to retrieve peer documents.
   */
  this.synchronize = (onHost, onPort, upTo, chunkLength, interactive, askedCautious, nopeers) => {
    const remote = new Synchroniser(that, onHost, onPort, that.conf, interactive === true);
    const syncPromise = remote.sync(upTo, chunkLength, askedCautious, nopeers);
    return {
      flow: remote,
      syncPromise: syncPromise
    };
  };
  
  this.testForSync = (onHost, onPort) => {
    const remote = new Synchroniser(that, onHost, onPort);
    return remote.test();
  };

  /**
   * Enable routing features:
   *   - The server will try to send documents to the network
   *   - The server will eventually be notified of network failures
   */
  this.routing = () => {
    // The router asks for multicasting of documents
    this.pipe(this.router())
      // The documents get sent to peers
      .pipe(multicaster(this.conf))
      // The multicaster may answer 'unreachable peer'
      .pipe(this.router());
  };

  this.upnp = () => co(function *() {
    const upnpAPI = yield upnp(that.conf.port, that.conf.remoteport);
    that.upnpAPI = upnpAPI;
    return upnpAPI;
  });
  
  this.listenToTheWeb = (showLogs) => co(function *() {
    const bmapi = yield bma(that, [{
      ip: that.conf.ipv4,
      port: that.conf.port
    }], showLogs);
    return bmapi.openConnections();
  });

  this.rawer = rawer;

  this.writeRaw = (raw, type) => co(function *() {
    const parser = documentsMapping[type] && documentsMapping[type].parser;
    const obj = parser.syncWrite(raw);
    return yield that.singleWritePromise(obj);
  });

  /**
   * Retrieve the last linesQuantity lines from the log file.
   * @param linesQuantity
   */
  this.getLastLogLines = (linesQuantity) => this.dal.getLogContent(linesQuantity);
}

util.inherits(Server, stream.Duplex);

module.exports = Server;