Merge pull request #40 from ShipChain/feature/docker-image-cleanup

Docker Build Improvements

.circleci/config.yml

@@ -6,7 +6,9 @@ version: 2
 jobs:
   build:
     docker:
-      - image: docker/compose:1.19.0
+      - image: docker/compose:1.21.0
+    environment:
+      ROLE: circleci
 
     working_directory: ~/repo
 
@@ -22,42 +24,57 @@ jobs:
             docker network ls | grep portal > /dev/null || docker network create portal
 
       - run:
-          name: Build application Docker image
+          name: Install dependencies
           command: |
-            docker-compose -f compose/base-services.yml -f compose/circleci.yml build
+            apk add --no-cache py-pip bash
 
       - run:
-          name: Install dependencies
+          name: Build application Docker images
           command: |
-            apk add --no-cache py-pip
-            pip install awscli
+            docker build --target test   --tag engine-node-test .
+            docker build --target deploy --tag engine-node-deploy --cache-from engine-node-test .
 
       - run:
           name: Launch the docker containers
           command: |
-            $(aws ecr get-login --no-include-email)
-            docker-compose -f compose/base-services.yml -f compose/circleci.yml up -d
+            bin/dc up -d rpc
+
+      - run:
+          name: Check yarn.lock integrity
+          command: |
+            bin/dc exec -T rpc yarn check --integrity
+            bin/dc exec -T rpc yarn check --verify-tree
+
+      - run:
+          name: Run lint for JUnit
+          command: |
+            bin/dc exec -T rpc yarn lint-junit
+
+      - run:
+          name: Run lint for HTML
+          command: |
+            bin/dc exec -T rpc yarn lint-html
 
       - run:
           name: Run unit tests
           command: |
-            docker-compose -f compose/base-services.yml -f compose/circleci.yml exec -T rpc yarn test
+            bin/dc exec -T rpc yarn test
 
       - run:
           name: Copy artifacts from Docker
           command: |
-            docker cp $(docker-compose -f compose/base-services.yml -f compose/circleci.yml ps -q rpc):/app/reports reports
+            docker cp $(bin/dc ps -q rpc):/app/reports/. reports
 
       - store_test_results:
-          path: reports
+          path: reports/junit
       - store_artifacts:
           path: reports
 
       - run:
           name: Save docker image
           command: |
             mkdir -p docker-cache
-            docker save -o docker-cache/built-image.tar engine-node
+            docker save -o docker-cache/built-image.tar engine-node-deploy
 
       - save_cache:
           key: docker-cache-{{ .Branch }}-{{ .Revision }}
@@ -96,8 +113,8 @@ jobs:
           name: Push image to ECR
           command: |
             source $BASH_ENV
-            docker tag engine-node $ECR_ENDPOINT/engine-node:$SHORT_GIT_HASH
-            docker tag engine-node $ECR_ENDPOINT/engine-node:latest
+            docker tag engine-node-deploy $ECR_ENDPOINT/engine-node:$SHORT_GIT_HASH
+            docker tag engine-node-deploy $ECR_ENDPOINT/engine-node:latest
             $(aws ecr get-login --no-include-email)
             docker push $ECR_ENDPOINT/engine-node:$SHORT_GIT_HASH
             docker push $ECR_ENDPOINT/engine-node:latest

.dockerignore

@@ -1,2 +1,18 @@
-node_modules
+# Directories
+# -----------
+.circleci
 .git
+.githooks
+.idea
+bin
+compose/nginx
+node_modules
+reports
+
+
+# Specific Files
+# --------------
+compose/*.yml
+.dockerignore
+.env*
+*Dockerfile*

.eslintignore

@@ -0,0 +1,2 @@
+__tests__
+src/entity/migration

.eslintrc.json

@@ -0,0 +1,21 @@
+{
+  "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "project": "./tsconfig.json",
+    "ecmaVersion": 6,
+    "sourceType": "module",
+    "ecmaFeatures": {
+      "modules": true
+    }
+  },
+  "plugins": ["@typescript-eslint"],
+//  "rules": {
+//    "@typescript-eslint/indent": "error",
+//    "@typescript-eslint/camelcase": "error",
+//    "@typescript-eslint/no-unused-vars": "warn",
+//    "@typescript-eslint/no-var-requires": "off",
+//    "@typescript-eslint/explicit-function-return-type": "error"
+//  },
+  "extends": ["plugin:prettier/recommended"]
+  //  "extends": ["plugin:@typescript-eslint/recommended", "plugin:prettier/recommended"]
+}

.githooks/pre-push

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+BRANCH=`git rev-parse --abbrev-ref HEAD`
+
+if [[ "$BRANCH" == "master" ]]; then
+  bin/docker_tests
+fi
+
+exit 0

.prettierignore

@@ -0,0 +1,2 @@
+__tests__
+src/entity/migration

.yarnclean

@@ -0,0 +1,47 @@
+# test directories
+__tests__
+test
+tests
+powered-test
+
+# asset directories
+docs
+doc
+website
+images
+
+# examples
+example
+examples
+
+# code coverage directories
+coverage
+.nyc_output
+
+# build scripts
+Makefile
+Gulpfile.js
+Gruntfile.js
+
+# configs
+appveyor.yml
+circle.yml
+codeship-services.yml
+codeship-steps.yml
+wercker.yml
+.tern-project
+.gitattributes
+.editorconfig
+.*ignore
+.eslintrc
+.jshintrc
+.flowconfig
+.documentup.json
+.yarn-metadata.json
+.travis.yml
+
+# misc
+*.md
+
+# remove aws-sdk from winston-cloudwatch.  It's already included in this project and isn't yet a peer dependency in that package
+winston-cloudwatch/node_modules/aws-sdk

Dockerfile

@@ -1,32 +1,87 @@
-FROM node:10.14.0-stretch
+## Base image with node and entrypoint scripts ##
+## =========================================== ##
+FROM node:10.15.0-alpine AS base
 
 LABEL maintainer="Lucas Clay <[email protected]>"
 
 ENV LANG C.UTF-8
-ENV PYTHONUNBUFFERED 1
 
-# SUPPORT SSH FOR IAM USERS #
-RUN apt-get update && apt-get -y install openssh-server python3-pip jq
-RUN mkdir /var/run/sshd /etc/cron.d
-RUN pip3 install keymaker
-RUN keymaker install
-
-# Configure public key SSH
-RUN echo "AllowAgentForwarding yes" >> /etc/ssh/sshd_config
-RUN echo "PasswordAuthentication no" >> /etc/ssh/sshd_config
-# ------------------------- #
-
-RUN pip3 install awscli
+RUN apk add --no-cache bash
 
 RUN mkdir /app
 WORKDIR /app
 
-ADD ./package.json /app/
-ADD ./yarn.lock /app/
-RUN yarn
-
-COPY . /app/
-
 COPY ./compose/scripts/*.sh /
 RUN chmod +x /*.sh
 ENTRYPOINT ["/entrypoint.sh"]
+
+
+## Image with system dependencies for building ##
+## =========================================== ##
+FROM base AS build
+
+RUN apk add --no-cache \
+    libc6-compat \
+    # git, python, make, g++ are for installing/building several npm modules
+    git \
+    python \
+    make \
+    g++
+
+
+## Image with dev-dependencies ##
+## =========================== ##
+FROM build AS test
+
+COPY package.json /app/
+COPY yarn.lock /app/
+COPY .yarnclean /app/
+
+RUN yarn --frozen-lockfile && yarn cache clean
+
+COPY . /app/
+
+
+## Image only used for production building ##
+## ======================================= ##
+FROM build AS prod
+
+COPY package.json /app/
+COPY yarn.lock /app/
+COPY .yarnclean /app/
+
+RUN yarn --prod --frozen-lockfile && yarn cache clean
+
+COPY . /app/
+
+
+## Image to be deployed to ECS with additional utils and no build tools ##
+## ==================================================================== ##
+FROM base AS deploy
+
+# Install openssh for ECS management container
+RUN apk add --no-cache \
+    openssh-server-pam \
+    python3 \
+    jq \
+    openssl \
+    shadow \
+    nano
+
+RUN mkdir /var/run/sshd /etc/cron.d && touch /etc/pam.d/sshd
+RUN sed -i 's/^CREATE_MAIL_SPOOL=yes/CREATE_MAIL_SPOOL=no/' /etc/default/useradd
+
+# Keymaker for SSH auth via IAM
+RUN pip3 install \
+    keymaker==1.0.8 \
+    awscli==1.16.95 && \
+    rm -rf /root/.cache/*
+
+# Configure public key SSH
+RUN echo "PubkeyAuthentication yes" >> /etc/ssh/sshd_config
+RUN echo "UsePAM yes" >> /etc/ssh/sshd_config
+RUN echo "AllowAgentForwarding yes" >> /etc/ssh/sshd_config
+RUN echo "PasswordAuthentication no" >> /etc/ssh/sshd_config
+
+# Copy production node_modules without having to install packages in build
+COPY --from=prod /app /app

README.md

@@ -28,7 +28,7 @@ We developed Engine using an array of Docker containers.  Deployment of these co
 
 See the official Docker documentation for installation information:
 
- - [Install Docker](https://docs.docker.com/engine/installation/)
+ - [Install Docker](https://docs.docker.com/engine/installation/) version > 17.09.0
  - [Install Docker Compose](https://docs.docker.com/compose/install/) version > 1.21.0
 
 Once Docker is installed, you will need a Docker "network" named `portal`:
@@ -75,18 +75,12 @@ git clone https://github.com/ShipChain/engine.git shipchain-engine
 
 In the cloned repository there are scripts provided in the `bin` directory for Docker container management.  Using these to interact with yarn will ensure you are using the correct version of Node.js (This was developed using LTS v10.14.0).
 
-Install the required Node packages:
+Install the required Node packages locally, using the build-tools from the docker image.  The first time you run this command it will build the prerequisite docker images for local Engine development.  These images will contain the startup scripts as well as tools required for compiling node packages during installation.
 
 ```
 bin/ddo yarn
 ```
 
-Build the Docker image:
-
-```
-bin/dc build
-```
-
 ### Scripts
 
 The scripts provided in the `bin` directory allow for easier interaction with the Docker compose services and containers.  All scripts use `base-services.yml` as a baseline for service definitions and are extended by override files.  By default, the override used is the `dev-lite.yml` compose file.  This can be changed to any configuration file by setting the `ROLE` environment variable.  For example if you want to override `base-services.yml` with settings from `my_settings.yml`, you would only need to set `ROLE=my_settings` in your environment.
@@ -96,6 +90,10 @@ The scripts provided in the `bin` directory allow for easier interaction with th
  - `bin/dcleanup` Single command to kill, remove, restart, and tail the new logs of a container.
  - `bin/docker_tests` This executes the unit tests with the `circleci.yml` configuration file.  The RPC service is launched using `sleep infinity` to prevent the full server from launching for the tests.
 
+Local development (with `dev` or `dev-lite` roles) use the `base` stage present in the [Dockerfile](Dockerfile); please note, this file *doesn't* use the docker `COPY` directive to copy the project code into the container, instead the code is mounted as a volume (so that as you save files, they update inside the container).
+
+A deployed environment should use the `prod` or `deploy` stage in the Dockerfile.  These will install node modules passing the `--production` flag to Yarn.  Additionally, the `deploy` stage includes prerequisites for connecting to a separate management container when running in an AWS environment.
+
 ### Configuration
 
 Before you can begin using Engine, you may need to do some configuration depending on your specific requirements.
@@ -143,6 +141,10 @@ If you want to also log messages to ElasticSearch, add the following variable po
  - `ELASTICSEARCH_LEVEL`
  	- Defaults to the value set in `LOGGING_LEVEL`
 
+Log messages will be sent automatically to AWS CloudWatch when `ENV` is set to `DEV`, `STAGE`, `DEMO`, or `PROD`).  The log level of the messages send to CloudWatch can be controlled via:
+ - `CLOUDWATCH_LEVEL`
+ 	- Defaults to the value set in `LOGGING_LEVEL`
+
 ##### Metrics
 Engine supports the reporting of application metrics to an InfluxDB instance. We use this internally in combination with
 Graphana to make a real-time dashboard of our application use. In order to use this, set:
@@ -173,6 +175,8 @@ Run the unit tests via the provided script
 bin/docker_tests
 ```
 
+Generated reports (Jest and Coverage) will be copied locally to a new `reports` directory after the successful completion of unit tests.
+
 ## Launching Engine
 
 To start Engine, use the provided script to bring the stack online:

bin/dc

@@ -2,9 +2,13 @@
 BIN="$( cd "$( dirname "${BASH_SOURCE[0]}"  )" && pwd  )"
 PROJECT_DIR="$( cd $BIN/.. && pwd  )"
 
+# Install git hooks
+cp ${PROJECT_DIR}/.githooks/* ${PROJECT_DIR}/.git/hooks/
+chmod ug+x ${PROJECT_DIR}/.git/hooks/*
+
 [[ -f ../.envrc ]] && source ../.envrc
 [[ -f ../.env ]] && source ../.env
-[[ -z "$COMPOSE_PROJECT" ]] && COMPOSE_PROJECT=$(basename $PROJECT_DIR)
+[[ -z "$COMPOSE_PROJECT" ]] && COMPOSE_PROJECT=$(basename ${PROJECT_DIR})
 [[ -z "$ROLE" ]] && ROLE=dev-lite
 
-docker-compose -p $COMPOSE_PROJECT -f compose/base-services.yml -f compose/$ROLE.yml $*
+docker-compose -p ${COMPOSE_PROJECT} -f compose/base-services.yml -f compose/${ROLE}.yml $*

bin/docker_tests

@@ -18,6 +18,14 @@ bin/dc build
 
 bin/dc up -d rpc
 
+bin/dc exec -T rpc yarn check --integrity
+
+bin/dc exec -T rpc yarn check --verify-tree
+
+bin/dc exec -T rpc yarn run lint
+
 bin/dc exec -T rpc yarn run test
 
+docker cp $(bin/dc ps -q rpc):/app/reports/. reports/
+
 bin/dc down