Big bang

.cargo/config

@@ -0,0 +1,20 @@
+[build]
+rustflags = ["-D", "warnings"]
+
+# Using 'cfg` is broken, see https://github.com/rust-lang/cargo/issues/6858
+# [target.'cfg(target_arch = "x86_64")']
+# rustflags = ["-D", "warnings", "-C", "target-cpu=skylake"]
+
+# ...so instead we list all target triples (Tier 1 64-bit platforms)
+
+[target.x86_64-unknown-linux-gnu]
+rustflags = ["-D", "warnings", "-C", "target-cpu=skylake"]
+
+[target.x86_64-pc-windows-gnu]
+rustflags = ["-D", "warnings", "-C", "target-cpu=skylake"]
+
+[target.x86_64-pc-windows-msvc]
+rustflags = ["-D", "warnings", "-C", "target-cpu=skylake"]
+
+[target.x86_64-apple-darwin]
+rustflags = ["-D", "warnings", "-C", "target-cpu=skylake"]

.circleci/config.yml

@@ -0,0 +1,282 @@
+# vim: tabstop=2 softtabstop=2 shiftwidth=2 expandtab:
+
+version: 2.1
+
+defaults:
+  builder-install: &builder-install gcr.io/mobilenode-211420/builder-install:1_8
+
+executors:
+  build-executor:
+    docker:
+      - image: *builder-install
+    resource_class: xlarge
+
+  test-executor:
+    docker:
+      - image: *builder-install
+    resource_class: large
+
+commands:
+  print_versions:
+    description: Version Info
+    steps:
+      - run:
+          name: Version Info
+          command: |
+            rustc --version
+            cargo --version
+            rustup --version
+            sccache --version
+
+  env_setup:
+    description: Environment Setup
+    steps:
+      - run:
+          name: Setup Env
+          command: |
+            echo 'export IAS_MODE=DEV' >> $BASH_ENV
+            echo 'export SGX_MODE=SW' >> $BASH_ENV
+            echo 'export RUST_BACKTRACE=1' >> $BASH_ENV
+            echo 'export SKIP_SLOW_TESTS=1' >> $BASH_ENV
+      - run:
+          name: Configure cargo to use git cli
+          command: |
+            mkdir -p .cargo
+            echo '[net]' >> .cargo/config
+            echo 'git-fetch-with-cli = true' >> .cargo/config
+            if [ -f ~/.gitconfig ]; then
+              sed -i 's/github/git-non-exist-hub/g' ~/.gitconfig # https://github.com/rust-lang/cargo/issues/3900
+            fi
+
+  enable_sccache:
+    description: Enabling sccache
+    steps:
+      - run:
+          name: Enable sccache
+          command: |
+            echo 'export RUSTC_WRAPPER=sccache' >> $BASH_ENV
+            echo 'export SCCACHE_IDLE_TIMEOUT=1200' >> $BASH_ENV
+            echo 'export SCCACHE_CACHE_SIZE=1G' >> $BASH_ENV
+            echo 'export SCCACHE_ERROR_LOG=/tmp/sccache.log' >> $BASH_ENV
+
+
+  restore-sccache-cache:
+    steps:
+      - restore_cache:
+          name: Restore sccache cache
+          key: sccache-cache-stable-{{ arch }}
+
+  save-sccache-cache:
+    steps:
+      - save_cache:
+          name: Save sccache cache
+          # We use {{ epoch }} to always upload a fresh cache:
+          # Of course, restore_cache will not find this exact key,
+          # but it will fall back to the closest key (aka the most recent).
+          # See https://discuss.circleci.com/t/add-mechanism-to-update-existing-cache-key/9014/13
+          key: sccache-cache-stable-{{ arch }}-{{ epoch }}
+          paths:
+            - "~/.cache/sccache"
+      - store-sccache-logs
+
+  store-sccache-logs:
+    steps:
+      - store_artifacts:
+          path: /tmp/sccache.log
+          destination: logs/sccache.log
+
+  restore-cargo-cache:
+    steps:
+      - restore_cache:
+          name: Restore cargo cache
+          key: cargo-cache-stable-{{ arch }}
+
+  save-cargo-cache:
+    steps:
+      - save_cache:
+          name: Save cargo cache
+          # We use {{ epoch }} to always upload a fresh cache:
+          # Of course, restore_cache will not find this exact key,
+          # but it will fall back to the closest key (aka the most recent).
+          # See https://discuss.circleci.com/t/add-mechanism-to-update-existing-cache-key/9014/13
+          key: cargo-cache-stable-{{ arch }}-{{ epoch }}
+          paths:
+            - "~/.cargo"
+
+  build_setup:
+    steps:
+      - checkout
+      - print_versions
+      - env_setup
+
+  check-dirty-git:
+    steps:
+      - run:
+          name: Checking dirty git
+          command: |
+            # We expect this file to be modified (we modified it in the `env_setup` step)
+            git update-index --assume-unchanged .cargo/config
+
+            if [[ -n $(git status --porcelain) ]]; then
+              echo "repo is dirty"
+              git status
+              exit 1
+
+jobs:
+  # A job that builds all the tests in the workspace, and stores them in a test-bins/ directory.
+  # This directory then gets uploaded to the CircleCI workflow's workspace to be consumed
+  # by the run-tests jobs
+  build-parallel-tests:
+    executor: build-executor
+    steps:
+      - build_setup
+      - enable_sccache
+      - restore-cargo-cache
+      - restore-sccache-cache
+
+      - run:
+          name: Build/prepare unit tests
+          command: |
+            # TODO where do put this?
+            cargo install -Z install-upgrade cargo2junit
+
+            # Get a list of package names to test and the binaries that need to be executed in order to run them.
+            # It's possible for a package to produce multiple test binaries. This also compiles the tests.
+            rm -rf test-bins && mkdir test-bins && mkdir test-bins/bins-per-pkg
+
+            echo "Discovering tests..."
+            cargo test --locked --no-run --message-format=json | jq -r "select(.profile.test == true) | (.package_id | split(\" \")[0]) + \" \" + .filenames[]" | sort | uniq | tee test-list
+
+            while read line; do
+              PKG=$(echo $line | cut -f 1 -d ' ')
+              TEST_BIN=$(echo $line | cut -f 2 -d ' ')
+              echo "Discovered package $PKG with binary $TEST_BIN"
+              echo $PKG >> test-bins/all-pkgs.txt.unsorted
+              echo $TEST_BIN >> test-bins/bins-per-pkg/$PKG
+            done < test-list
+
+            # De-dupe packages in all-pkgs.txt
+            cat test-bins/all-pkgs.txt.unsorted | sort | uniq > test-bins/all-pkgs.txt
+
+            # Collect all test bins into test-bins/
+            for PKG in $(cat test-bins/all-pkgs.txt); do
+              for TEST_BIN in $(cat test-bins/bins-per-pkg/$PKG); do
+                cp $TEST_BIN test-bins
+              done
+            done
+
+            # Some logging
+            echo 'inside test-bins/:'
+            ls -lRa test-bins
+            echo "inside all-pkgs.txt:"
+            cat test-bins/all-pkgs.txt
+
+      - run:
+          name: Linting
+          command: ./tools/lint.sh
+
+      - check-dirty-git
+
+      # Save the test binaries and helper text files into the workspace so that they could be consumed by the `run-tests` job.
+      - persist_to_workspace:
+          root: test-bins
+          paths:
+            - .
+      - save-cargo-cache
+      - save-sccache-cache
+
+  # Runs tests from a test-bin directory previously generated by the build-tests job
+  run-tests:
+    executor: test-executor
+    parallelism: 2
+    steps:
+      - restore-cargo-cache
+      - env_setup
+      - attach_workspace:
+          at: test-bins
+
+      - run:
+          name: Run unit tests
+          command: |
+            ls -lRa test-bins
+
+            # Crates that define macros (e.g. `digestible_derive`) link dynamically against libtest*.so, which sits here.
+            export LD_LIBRARY_PATH="$HOME/.rustup/toolchains/$(rustup show active-toolchain | awk '{print $1}')/lib"
+            source /opt/intel/sgxsdk/environment
+            echo "LD_LIBRARY_PATH = $LD_LIBRARY_PATH"
+
+            # Run the test binaries for all the packages we're assigned by CircleCI's test splitting mechanism,
+            rm -rf test-results && mkdir test-results
+            echo "About to run: $(cat test-bins/all-pkgs.txt | circleci tests split --split-by=timings --timings-type=classname | sort | uniq)"
+            for PKG in $(cat test-bins/all-pkgs.txt | circleci tests split --split-by=timings --timings-type=classname); do
+              echo "Running tests for $PKG"
+              for TEST_BIN in $(cat test-bins/bins-per-pkg/$PKG); do
+                echo "Running bin $TEST_BIN for pkg $PKG..."
+                JSON_OUT="results-$PKG.json"
+                # Run the test and also store the output into a JSON file
+                ./test-bins/$(basename $TEST_BIN) -Z unstable-options --report-time --format json | tee $JSON_OUT
+
+                # For CircleCI's time-based splitting to work, we need to convert Cargo's JSON output into a JUNIT xml file.
+                # Sadly, cargo does not output the name of the package it is testing. This search-and-replace is a hack around that.
+                cat $JSON_OUT | cargo2junit | sed -e 's/name="\(.*\)" package/name="'"$PKG"'" package/' > test-results/$(basename $TEST_BIN).xml
+              done
+            done
+
+      - check-dirty-git
+
+      - store_test_results:
+          path: test-results
+      - store_artifacts:
+          path: test-results
+
+
+  # Run all tests on a single container
+  run-all-tests:
+    executor: build-executor
+    parallelism: 1
+    steps:
+      - build_setup
+      - enable_sccache
+      - restore-cargo-cache
+      - restore-sccache-cache
+      - run:
+          name: Run all unit tests
+          command: |
+            openssl genrsa -out $(pwd)/Enclave_private.pem -3 3072
+            export CONSENSUS_ENCLAVE_PRIVKEY=$(pwd)/Enclave_private.pem
+            export INGEST_ENCLAVE_PRIVKEY=$(pwd)/Enclave_private.pem
+            export LEDGER_ENCLAVE_PRIVKEY=$(pwd)/Enclave_private.pem
+            export VIEW_ENCLAVE_PRIVKEY=$(pwd)/Enclave_private.pem
+
+            cargo test
+
+            rm $(pwd)/Enclave_private.pem
+      - run:
+          name: Lint/fmt
+          command: |
+            openssl genrsa -out $(pwd)/Enclave_private.pem -3 3072
+            export CONSENSUS_ENCLAVE_PRIVKEY=$(pwd)/Enclave_private.pem
+            export INGEST_ENCLAVE_PRIVKEY=$(pwd)/Enclave_private.pem
+            export LEDGER_ENCLAVE_PRIVKEY=$(pwd)/Enclave_private.pem
+            export VIEW_ENCLAVE_PRIVKEY=$(pwd)/Enclave_private.pem
+
+            ./tools/lint.sh
+
+            rm $(pwd)/Enclave_private.pem
+      - save-cargo-cache
+      - save-sccache-cache
+
+workflows:
+  version: 2
+  # Build and run tests on a single container
+  run-all-tests:
+    jobs:
+      - run-all-tests
+
+  # Build and run tests in parallel - not needed at the moment since the test suite is fast enough.
+  # build-and-run-tests:
+  #   jobs:
+  #     - build-parallel-tests
+  #     - run-parallel-tests:
+  #         requires:
+  #           - build-parallel-tests

.gitignore

@@ -0,0 +1,74 @@
+# Local-users generated keys for development of enclaves
+**/Enclave_private.pem
+
+# IntelliJ IDEA
+.idea/
+*.iml
+*.ipr
+*.iws
+
+# Eclipse
+.settings/
+.classpath
+.project
+
+nb-configuration.xml
+atlassian-ide-plugin.xml
+
+logs
+
+# Rust
+# Generated by Cargo
+# will have compiled files and executables
+/target/
+**/target/
+/cargo/
+**/cargo/
+**/cmake-build-debug/
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+# Generated binary libraries
+*.so
+*.a
+
+# grpc
+attest_api/src/attest.rs
+attest_api/src/attest_grpc.rs
+grpc_util/src/health_api.rs
+grpc_util/src/health_api_grpc.rs
+consensus/api/src/blockchain.rs
+consensus/api/src/blockchain_grpc.rs
+consensus/api/src/consensus_client.rs
+consensus/api/src/consensus_client_grpc.rs
+consensus/api/src/consensus_common.rs
+consensus/api/src/consensus_peer.rs
+consensus/api/src/consensus_peer_grpc.rs
+consensus/api/src/external.rs
+consensus/api/src/ledger_enclave_server.rs
+consensus/api/src/ledger_enclave_server_grpc.rs
+consensus/api/src/ledger_server.rs
+consensus/api/src/ledger_server_grpc.rs
+mobilecoind/api/src/mobilecoind_api.rs
+mobilecoind/api/src/mobilecoind_api_grpc.rs
+
+# Random junk
+.DS_Store
+.factorypath
+.vscode
+*.swp
+*.sublime-workspace
+*.sublime-project
+
+# Dockerfile-build output
+dist/
+/keyrings/live/pubring.gpg~
+/keyrings/live/pubring.kbx~
+/keyrings/live/secring.gpg
+/docker/nginx/nginx.key
+/docker/nginx/nginx.crt
+
+# Per-crate lock files
+Cargo.lock
+!/Cargo.lock

.mobconf

@@ -0,0 +1,4 @@
+[image]
+dockerfile = docker/Dockerfile
+target = builder-install
+repository = gcr.io/mobilenode-211420/