Rer 5 - Dataset env filtering

.gitignore

@@ -5,3 +5,4 @@ npm-debug.*
 coverage
 .nyc_output
 /dev.env
+dist

.travis.yml

@@ -2,8 +2,7 @@ language: node_js
 services:
   - mongodb
 node_js:
-  - "11.8"
-  - "11"
+  - "12"
 cache:
   directories:
   - "node_modules"
@@ -12,18 +11,19 @@ env:
     - CC_TEST_REPORTER_ID=dabdd92208e41ebbec13b01f5e943a0dcf7ec40f4a98d6d0a8f811e8cca9b443
     - NODE_ENV=test
     - PORT=5037
-    - API_VERSION=v1
-    - LOCAL_URL=http://127.0.0.1:3001
-    - CT_URL=http://127.0.0.1:9000
+    - GATEWAY_URL=http://127.0.0.1:9000
     - MONGO_PORT_27017_TCP_ADDR=127.0.0.1
-    - CT_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6Im1pY3Jvc2VydmljZSIsImNyZWF0ZWRBdCI6IjIwMTYtMDktMTQifQ.IRCIRm1nfIQTfda_Wb6Pg-341zhV8soAgzw7dd5HxxQ
+    - MICROSERVICE_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6Im1pY3Jvc2VydmljZSIsImNyZWF0ZWRBdCI6IjIwMTYtMDktMTQifQ.IRCIRm1nfIQTfda_Wb6Pg-341zhV8soAgzw7dd5HxxQ
+    - S3_ACCESS_KEY_ID=foo
+    - S3_SECRET_ACCESS_KEY=bar
+    - FASTLY_ENABLED=false
 
 before_script:
   - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
   - chmod +x ./cc-test-reporter
   - ./cc-test-reporter before-build
 script:
-  - npm test
-  - npm run coverage
+  - yarn test
+  - yarn run coverage
 after_script:
   - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT

CHANGELOG.md

@@ -0,0 +1,99 @@
+## 17/06/2021
+
+- Update `rw-api-microservice-node` to remove CT support.
+
+## 09/06/2021
+
+- Add support for hosts from `x-rw-domain` header when generating pagination links.
+
+## 28/05/2021
+
+- Update `rw-api-microservice-node` to add CORS support.
+
+## 25/05/2021
+
+- Remove body from `DELETE` requests to other microservices
+
+## 21/05/2021
+
+- Add support for hosts from `referer` header when generating pagination links.
+
+## 03/05/2021
+
+- Fix issue preventing updating `dataLastUpdated` with `null`.
+
+## 22/02/2021
+
+- Update `rw-api-microservice-node` to fix issue with Fastly headers.
+
+## 12/02/2021
+
+- Remove dependency on CT's `authenticated` functionality
+
+## 18/12/2020
+
+- Initial migration to Typescript
+
+## 04/12/2020
+
+- Replace CT integration library
+
+# v1.3.3
+
+## 17/11/2020
+
+- Security update of NPM package `lodash` from 4.17.15 to 4.17.20.
+- Security update of NPM package `yargs-parser` from 13.1.1 to 13.1.2.
+- Security update of NPM package `bl` from 2.2.0 to 2.2.1.
+
+# v1.3.2
+
+## 13/07/2020
+
+- Minor logging improvements.
+- Security updates to the `handlebars` and `websocket-extensions` NPM packages.
+
+# v1.3.1
+
+## 19/05/2020
+
+- Use secondary mongo nodes for read operations.
+- Disable mongo unified topology.
+
+# v1.3.0
+
+## 09/04/2020
+
+- Add node affinity to kubernetes configuration.
+
+## 24/03/2020
+
+- Remove blockchain and verified fields and associated blockchain signature logic.
+
+# v1.2.0
+
+## 23/03/2020
+
+- Fix issue where deleting wms datasets would fail
+- Fix issue where dataset upload would fail.
+- Add support for nested filters in applicationConfig object.
+- Fix issue with sorting for ascending order.
+- Remove `usersRole` query param which generated huge pagination links.
+
+# v1.1.0
+
+## 27/01/2020
+
+- Add possibility of sorting datasets by user fields (such as name or role).
+- Fix bug patching dataset without changing dataset apps.
+
+# v1.0.0
+
+## 14/01/2020
+
+- Add filter for querying subscribable datasets.
+- Fix bug removing dataset applications by admins who can manage the removed applications.
+
+# Previous
+
+- Allow microservices to update the dataset's `errorMessage` field.

Dockerfile

@@ -1,26 +1,27 @@
-FROM mhart/alpine-node:11.8
-MAINTAINER sergio.gordillo@vizzuality.com
+FROM node:12-alpine
+MAINTAINER info@vizzuality.com
 
 ENV NAME dataset
-ENV USER microservice
+ENV USER dataset
 
 RUN apk update && apk upgrade && \
     apk add --no-cache --update bash git openssh python alpine-sdk
 
 RUN addgroup $USER && adduser -s /bin/bash -D -G $USER $USER
 
-RUN npm install -g grunt-cli bunyan
-
 RUN mkdir -p /opt/$NAME
 COPY package.json /opt/$NAME/package.json
-RUN cd /opt/$NAME && npm install
+COPY yarn.lock /opt/$NAME/yarn.lock
+RUN cd /opt/$NAME && yarn
 
 COPY entrypoint.sh /opt/$NAME/entrypoint.sh
+COPY tsconfig.json /opt/$NAME/tsconfig.json
 COPY config /opt/$NAME/config
+COPY ./src /opt/$NAME/src
+COPY ./test opt/$NAME/test
 
 WORKDIR /opt/$NAME
 
-COPY ./app /opt/$NAME/app
 RUN chown -R $USER:$USER /opt/$NAME
 
 # Tell Docker we are going to use this ports

Jenkinsfile

@@ -2,24 +2,10 @@
 
 node {
 
-  // Actions
-  def forceCompleteDeploy = false
-  try {
-    timeout(time: 15, unit: 'SECONDS') {
-      forceCompleteDeploy = input(
-        id: 'Proceed0', message: 'Force COMPLETE Deployment', parameters: [
-        [$class: 'BooleanParameterDefinition', defaultValue: true, description: '', name: 'Please confirm you want to recreate services and deployments']
-      ])
-    }
-  }
-  catch(err) { // timeout reached or input false
-      // nothing
-  }
-
   // Variables
   def tokens = "${env.JOB_NAME}".tokenize('/')
   def appName = tokens[0]
-  def dockerUsername = "${DOCKER_USERNAME}"
+  def dockerUsername = "${DOCKER_WRI_USERNAME}"
   def imageTag = "${dockerUsername}/${appName}:${env.BRANCH_NAME}.${env.BUILD_NUMBER}"
 
   currentBuild.result = "SUCCESS"
@@ -41,8 +27,8 @@ node {
     }
 
     stage('Push Docker') {
-      withCredentials([usernamePassword(credentialsId: 'Vizzuality Docker Hub', usernameVariable: 'DOCKER_HUB_USERNAME', passwordVariable: 'DOCKER_HUB_PASSWORD')]) {
-        sh("docker -H :2375 login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD}")
+      withCredentials([usernamePassword(credentialsId: 'WRI Docker Hub', usernameVariable: 'DOCKER_HUB_USERNAME', passwordVariable: 'DOCKER_HUB_PASSWORD')]) {
+        sh("docker -H :2375 login -u ${DOCKER_HUB_USERNAME} -p '${DOCKER_HUB_PASSWORD}'")
         sh("docker -H :2375 push ${imageTag}")
         sh("docker -H :2375 push ${dockerUsername}/${appName}:latest")
         sh("docker -H :2375 rmi ${imageTag}")
@@ -52,50 +38,49 @@ node {
     stage ("Deploy Application") {
       switch ("${env.BRANCH_NAME}") {
 
+        // Roll out to dev
+        case "dev":
+          sh("echo Deploying to DEV cluster")
+          sh("kubectl config use-context ${KUBECTL_CONTEXT_PREFIX}_${CLOUD_PROJECT_NAME}_${CLOUD_PROJECT_ZONE}_${KUBE_DEV_CLUSTER}")
+          sh("kubectl apply -f k8s/dev/")
+          sh("kubectl set image deployment ${appName} ${appName}=${imageTag} --record")
+          break
+
         // Roll out to staging
-        case "develop":
+        case "staging":
           sh("echo Deploying to STAGING cluster")
-          sh("kubectl config use-context gke_${GCLOUD_PROJECT}_${GCLOUD_GCE_ZONE}_${KUBE_STAGING_CLUSTER}")
-          def service = sh([returnStdout: true, script: "kubectl get deploy ${appName} || echo NotFound"]).trim()
-          if ((service && service.indexOf("NotFound") > -1) || (forceCompleteDeploy)){
-            sh("sed -i -e 's/{name}/${appName}/g' k8s/services/*.yaml")
-            sh("sed -i -e 's/{name}/${appName}/g' k8s/staging/*.yaml")
-            sh("kubectl apply -f k8s/services/")
-            sh("kubectl apply -f k8s/staging/")
-          }
+          sh("kubectl config use-context ${KUBECTL_CONTEXT_PREFIX}_${CLOUD_PROJECT_NAME}_${CLOUD_PROJECT_ZONE}_${KUBE_STAGING_CLUSTER}")
+          sh("kubectl apply -f k8s/staging/")
           sh("kubectl set image deployment ${appName} ${appName}=${imageTag} --record")
           break
 
         // Roll out to production
-        case "master":
+        case "production":
           def userInput = true
           def didTimeout = false
-          try {
-            timeout(time: 60, unit: 'SECONDS') {
-              userInput = input(
-                id: 'Proceed1', message: 'Confirm deployment', parameters: [
-                [$class: 'BooleanParameterDefinition', defaultValue: true, description: '', name: 'Please confirm you agree with this deployment']
-              ])
-            }
-          }
-          catch(err) { // timeout reached or input false
-              sh("echo Aborted by user or timeout")
-              if('SYSTEM' == user.toString()) { // SYSTEM means timeout.
-                  didTimeout = true
-              } else {
-                  userInput = false
+
+          if ("${SKIP_DEPLOYMENT_CONFIRMATION}" != "true") {
+              try {
+                timeout(time: 60, unit: 'SECONDS') {
+                  userInput = input(
+                    id: 'Proceed1', message: 'Confirm deployment', parameters: [
+                    [$class: 'BooleanParameterDefinition', defaultValue: true, description: '', name: 'Please confirm you agree with this deployment']
+                  ])
+                }
+              }
+              catch(err) { // timeout reached or input false
+                  sh("echo Aborted by user or timeout")
+                  if('SYSTEM' == user.toString()) { // SYSTEM means timeout.
+                      didTimeout = true
+                  } else {
+                      userInput = false
+                  }
               }
           }
-          if (userInput == true && !didTimeout){
+          if ((userInput == true && !didTimeout) || "${SKIP_DEPLOYMENT_CONFIRMATION}" != "true") {
             sh("echo Deploying to PROD cluster")
-            sh("kubectl config use-context gke_${GCLOUD_PROJECT}_${GCLOUD_GCE_ZONE}_${KUBE_PROD_CLUSTER}")
-            def service = sh([returnStdout: true, script: "kubectl get deploy ${appName} || echo NotFound"]).trim()
-            if ((service && service.indexOf("NotFound") > -1) || (forceCompleteDeploy)){
-              sh("sed -i -e 's/{name}/${appName}/g' k8s/services/*.yaml")
-              sh("sed -i -e 's/{name}/${appName}/g' k8s/production/*.yaml")
-              sh("kubectl apply -f k8s/services/")
-              sh("kubectl apply -f k8s/production/")
-            }
+            sh("kubectl config use-context ${KUBECTL_CONTEXT_PREFIX}_${CLOUD_PROJECT_NAME}_${CLOUD_PROJECT_ZONE}_${KUBE_PROD_CLUSTER}")
+            sh("kubectl apply -f k8s/production/")
             sh("kubectl set image deployment ${appName} ${appName}=${imageTag} --record")
           } else {
             sh("echo NOT DEPLOYED")
@@ -109,28 +94,9 @@ node {
           currentBuild.result = "SUCCESS"
       }
     }
-
-    // Notify Success
-    slackSend (color: '#00FF00', channel: '#the-new-api', message: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})")
-    emailext (
-      subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
-      body: """<p>SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]':</p>
-        <p>Check console output at "<a href="${env.BUILD_URL}">${env.JOB_NAME} [${env.BUILD_NUMBER}]</a>"</p>""",
-      recipientProviders: [[$class: 'DevelopersRecipientProvider']]
-    )
-
-
   } catch (err) {
 
     currentBuild.result = "FAILURE"
-    // Notify Error
-    slackSend (color: '#FF0000', channel: '#the-new-api', message: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})")
-    emailext (
-      subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
-      body: """<p>FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]':</p>
-        <p>Check console output at "<a href="${env.BUILD_URL}">${env.JOB_NAME} [${env.BUILD_NUMBER}]</a>"</p>""",
-      recipientProviders: [[$class: 'DevelopersRecipientProvider']]
-    )
     throw err
   }