Merge of all branches with latest dev

Flipkart · Jan 31, 2022 · bfc764f · bfc764f
2 parents e27fd42 + f45f4d5
commit bfc764f
Show file tree

Hide file tree

Showing 134 changed files with 4,351 additions and 504 deletions.
diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml
@@ -0,0 +1,27 @@
+on:
+    push:
+        branches-ignore:
+            - 'master'
+    pull_request:
+        branches:
+            - 'master'
+jobs:
+    test_and_package:
+        runs-on: ubuntu-latest
+        continue-on-error: false
+        steps:
+            - name: Checking out code base
+              uses: actions/checkout@v2
+
+            - name: Set up JDK 1.8
+              uses: actions/setup-java@v1
+              with:
+                java-version: 1.8
+
+            - name: Pull ryuk
+              run: docker pull quay.io/testcontainers/ryuk:0.2.2
+              continue-on-error: false
+
+            - name: Verification
+              run: mvn clean verify -Dmaven.test.failure.ignore=false
+              continue-on-error: false
diff --git a/.gitignore b/.gitignore
@@ -40,4 +40,3 @@ target/*
 config/test.yml
 gauth-creds-env.sh
 idman-creds-env.sh
-
diff --git a/Dockerfile b/Dockerfile
@@ -3,7 +3,7 @@ MAINTAINER Santanu Sinha <santanu DOT sinha [at] gmail.com>
 
 RUN apt-get clean && apt-get update && apt-get install -y --no-install-recommends software-properties-common
 RUN add-apt-repository ppa:openjdk-r/ppa && apt-get update
-RUN apt-get install -y --no-install-recommends openjdk-8-jdk ca-certificates && apt-get install -y --no-install-recommends ca-certificates-java bash curl tzdata iproute2 zip unzip wget
+RUN apt-get install -y --no-install-recommends openjdk-8-jdk ca-certificates ca-certificates-java bash curl tzdata iproute2 zip unzip wget
 
 
 EXPOSE 17000
@@ -12,9 +12,10 @@ EXPOSE 5701
 
 VOLUME /var/log/foxtrot-server
 
-ADD config/docker.yml docker.yml
+ADD config/docker* config/
 ADD foxtrot-server/target/foxtrot-server*.jar server.jar
 ADD scripts/local_es_setup.sh local_es_setup.sh
+ADD startup.sh startup.sh
 
-CMD sh -c "sleep 15 ; java -Dfile.encoding=utf-8 -XX:+${GC_ALGO-UseG1GC} -Xms${JAVA_PROCESS_MIN_HEAP-1g} -Xmx${JAVA_PROCESS_MAX_HEAP-1g} ${JAVA_OPTS} -jar server.jar server docker.yml"
+CMD ./startup.sh
 
diff --git a/README.md b/README.md
@@ -31,8 +31,15 @@ Check the [Wiki](https://github.com/Flipkart/foxtrot/wiki/Introduction) for deta
 
 Version
 ----
+6.3.1-9
 
-0.1
+Docker
+------
+Docker can be found on [DockerHub](https://hub.docker.com/layers/santanusinha/foxtrot/6.3.1-9/images/sha256-01cb327eb0353d31874681ee9ece4df8993b2152dfd3fa7279a31e3d7e32ee7e?context=explore)
+
+Docker can be customised using environment variables. Refer [here](https://github.com/Flipkart/foxtrot/blob/simple_auth/config/docker.yml) for the variables that need to be passed.
+
+Volume mount the config file in a docker. And pass in the full path to file `CONFIG_PATH` environment variable. 
 
 Tech
 -----------

diff --git a/build.properties b/build.properties
@@ -1 +1 @@
-BUILD_PROJECT_VERSION=6.3.1-8
+BUILD_PROJECT_VERSION=6.3.1-9
diff --git a/config/docker-gauth.yml b/config/docker-gauth.yml
@@ -0,0 +1,89 @@
+server:
+  applicationConnectors:
+    - type: http
+      port: 17000
+  adminConnectors:
+    - type: http
+      port: 17001
+
+swaggerScheme: ${SWAGGER_SCHEME}
+
+elasticsearch:
+  hosts:
+  - ${ELASTICSEARCH_HOST}
+  cluster: ${ELASTICSEARCH_CLUSTER_NAME}
+  tableNamePrefix: ${ELASTICSEARCH_TABLE_NAME_PREFIX}
+  port: ${ELASTICSEARCH_PORT}
+  connectionType: ${ELASTICSEARCH_PROTOCOL}
+
+hbase:
+  secure : false
+  tableName: ${HBASE_TABLE_NAME}
+  hbaseZookeeperQuorum: ${HBASE_ZOOKEEPER_QUORUM}
+  hbaseZookeeperClientPort: ${HBASE_ZOOKEEPER_CLIENT_PORT}
+
+cluster:
+  name: foxtrot
+  discovery:
+    type: ${DISCOVERY_TYPE}
+    disableMulticast: true
+    members: ["localhost:5701"]
+
+cacheConfig:
+  maxIdleSeconds: 15
+  timeToLiveSeconds: 15
+
+logging:
+  level: INFO
+  loggers:
+    com.flipkart.foxtrot.core.querystore.actions: DEBUG
+    org.apache.hadoop.hbase.zookeeper: WARN
+    org.apache.zookeeper: WARN
+    org.apache.hadoop.hbase.client: WARN
+  appenders:
+    - type: console
+      threshold: TRACE
+      timeZone: IST
+      logFormat: "%(%-5level) [%date] %X{TRACE-ID} [%thread] [%logger{0}]: %message%n"
+
+cardinality:
+  enabled: true
+  batchSize: 10
+  active: true
+  interval: 86400
+  initialDelay: 6
+  maxCardinality: 20000
+
+deletionconfig:
+  active: true
+  interval: 86400
+  initialdelay: 60
+
+esIndexOptimizationConfig:
+  active: true
+  interval: 86400
+  initialDelay: 6
+
+consoleHistoryConfig:
+  active: true
+  interval: 86400
+  initialDelay: 10
+
+sessionCleanupConfig:
+  active: true
+  interval: 86400
+  initialDelay: 10
+
+auth:
+  enabled: true
+  jwt:
+    issuerId: foxtrot-server
+    privateKey: ${PRIVATE_KEY}
+    sessionDuration: 15d
+
+  provider:
+   type: OAUTH_GOOGLE
+   clientId: ${GOOGLE_CLIENT_ID}
+   clientSecret: ${GOOGLE_CLIENT_SECRET}
+   server: ${GOOGLE_CALLBACK_HOST_PORT}
+   secureEndpoint: ${GOOGLE_SECURE_ENDPOINT}
diff --git a/config/docker-idman.yml b/config/docker-idman.yml
@@ -0,0 +1,89 @@
+server:
+  applicationConnectors:
+    - type: http
+      port: 17000
+  adminConnectors:
+    - type: http
+      port: 17001
+
+swaggerScheme: ${SWAGGER_SCHEME}
+
+elasticsearch:
+  hosts:
+  - ${ELASTICSEARCH_HOST}
+  cluster: ${ELASTICSEARCH_CLUSTER_NAME}
+  tableNamePrefix: ${ELASTICSEARCH_TABLE_NAME_PREFIX}
+  port: ${ELASTICSEARCH_PORT}
+  connectionType: ${ELASTICSEARCH_PROTOCOL}
+
+hbase:
+  secure : false
+  tableName: ${HBASE_TABLE_NAME}
+  hbaseZookeeperQuorum: ${HBASE_ZOOKEEPER_QUORUM}
+  hbaseZookeeperClientPort: ${HBASE_ZOOKEEPER_CLIENT_PORT}
+
+cluster:
+  name: foxtrot
+  discovery:
+    type: ${DISCOVERY_TYPE}
+    disableMulticast: true
+    members: ["localhost:5701"]
+
+cacheConfig:
+  maxIdleSeconds: 15
+  timeToLiveSeconds: 15
+
+logging:
+  level: INFO
+  loggers:
+    com.flipkart.foxtrot.core.querystore.actions: DEBUG
+    org.apache.hadoop.hbase.zookeeper: WARN
+    org.apache.zookeeper: WARN
+    org.apache.hadoop.hbase.client: WARN
+  appenders:
+    - type: console
+      threshold: TRACE
+      timeZone: IST
+      logFormat: "%(%-5level) [%date] %X{TRACE-ID} [%thread] [%logger{0}]: %message%n"
+
+cardinality:
+  enabled: true
+  batchSize: 10
+  active: true
+  interval: 86400
+  initialDelay: 6
+  maxCardinality: 20000
+
+deletionconfig:
+  active: true
+  interval: 86400
+  initialdelay: 60
+
+esIndexOptimizationConfig:
+  active: true
+  interval: 86400
+  initialDelay: 6
+
+consoleHistoryConfig:
+  active: true
+  interval: 86400
+  initialDelay: 10
+
+sessionCleanupConfig:
+  active: true
+  interval: 86400
+  initialDelay: 10
+
+auth:
+  enabled: true
+  jwt:
+    issuerId: foxtrot-server
+    privateKey: ${PRIVATE_KEY}
+    sessionDuration: 15d
+
+  provider:
+    type: OAUTH_IDMAN
+    idmanEndpoint: ${IDMAN_ENDPOINT}
+    clientId: FOXTROT
+    clientSecret: 8f4bcf45-2909-42e0-9bba-f46351bb0e6d
+    serverEndpoint: "http://localhost:17000"
diff --git a/config/docker.yml b/config/docker.yml
@@ -6,6 +6,8 @@ server:
     - type: http
       port: 17001
 
+swaggerScheme: ${SWAGGER_SCHEME}
+
 elasticsearch:
   hosts:
   - ${ELASTICSEARCH_HOST}
@@ -25,7 +27,7 @@ cluster:
   discovery:
     type: ${DISCOVERY_TYPE}
     disableMulticast: true
-    members: ["localhost:5701"]
+    members: ["${HOSTNAME}:5701"]
 
 cacheConfig:
   maxIdleSeconds: 15
@@ -67,3 +69,8 @@ consoleHistoryConfig:
   interval: 86400
   initialDelay: 10
 
+sessionCleanupConfig:
+  active: true
+  interval: 86400
+  initialDelay: 10
+
diff --git a/config/local.yml b/config/local.yml
@@ -21,7 +21,7 @@ hbase:
   hbaseZookeeperQuorum: localhost:2181
   hbaseZookeeperClientPort: 2181
   seggregatedTablePrefix: foxtrot-
-  hbaseZookeeperZnodeParent: /hbase-unsecure
+  hbaseZookeeperZnodeParent: /hbase-test
 
 cluster:
   name: foxtrot
@@ -50,6 +50,7 @@ elasticsearchTuningConfig:
   aggregationSize: 10000
   scrollSize: 100
   scrollTimeInSeconds: 120
+  documentsLimitAllowed: 10000
 
 esIndexOptimizationConfig:
   active: true
@@ -71,4 +72,23 @@ clusterRerouteConfig:
   active: false
   interval: 86400
   initialDelay: 10
-  thresholdShardCountPercentage: 10
+  thresholdShardCountPercentage: 10
+
+sessionCleanupConfig:
+  active: true
+  interval: 86400
+  initialDelay: 10
+
+auth:
+  enabled: false
+  jwt:
+    issuerId: foxtrot-server
+    privateKey: ${PRIVATE_KEY}
+    sessionDuration: 15d
+
+  provider:
+    type: OAUTH_GOOGLE
+    clientId: ${GOOGLE_CLIENT_ID}
+    clientSecret: ${GOOGLE_CLIENT_SECRET}
+    server: ${GOOGLE_CALLBACK_HOST_PORT}
+    secureEndpoint: false
diff --git a/docker-compose-auth.yml b/docker-compose-auth.yml
@@ -0,0 +1,66 @@
+version: '2'
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.8
+    container_name: elasticsearch
+    hostname: elasticsearch
+    environment:
+          - cluster.name=elasticsearch
+          - bootstrap.memory_lock=true
+          - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+          memlock:
+            soft: -1
+            hard: -1
+    command: elasticsearch -Ehttp.cors.enabled=true
+    volumes:
+          - /elasticsearch/data
+    ports:
+     - "9200:9200"
+     - "9300:9300"
+
+  hbase:
+    image: hyness/hbase-rest-standalone
+    hostname: hbase
+    ports:
+     - "2181:2181"
+     - "8080:8080"
+     - "16000:16000"
+     - "16010:16010"
+     - "16020:16020"
+     - "16030:16030"
+    volumes:
+     - /data/hbase
+     - /data/zookeeper
+
+  foxtrot-server:
+    depends_on:
+     - elasticsearch
+     - hbase
+    container_name: foxtrot_server
+    build: .
+    ports:
+     - "17000:17000"
+     - "17001:17001"
+     - "5701:5701"
+    volumes:
+     - /var/log/foxtrot-server
+    environment:
+     - HOST=localhost
+     - ELASTICSEARCH_HOST=elasticsearch
+     - ELASTICSEARCH_CLUSTER_NAME=elasticsearch
+     - ELASTICSEARCH_TABLE_NAME_PREFIX=foxtrot
+     - ELASTICSEARCH_PORT="9200"
+     - ELASTICSEARCH_PROTOCOL=http
+     - HBASE_TABLE_NAME=foxtrot
+     - HBASE_ZOOKEEPER_QUORUM=hbase
+     - HBASE_ZOOKEEPER_CLIENT_PORT=2181
+     - HAZELCAST_CLUSTER_NAME=foxtrot
+     - AUTH_ENABLED=true
+     - GOOGLE_CALLBACK_HOST_PORT=localhost:17000
+     - GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID}
+     - GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET}
+     - GOOGLE_SECURE_ENDPOINT=false
+     - DISCOVERY_TYPE=foxtrot_simple
+     - SWAGGER_SCHEME="http"
+     - PRIVATE_KEY="6/Qh1eLvF7cGJEgH1wIMYErIpi0d8Kqt5wHKw/o5iFbhfziKiux03HfFihO2+f5FKpsjk1D7UmgdAWQsY966WkyAsyo22m4g+C+rNCwMUD/UM2qop7JZBDEkgWpwK8jpxOTpYUnaNsiBaBsadoZCQ+Ns/d8RrS7QPAW+1PETbdFfo+fPW0/Sm8RUmMKkc9jJ6bhuSpFH3q/bWoyPZnj7geYQQIkgmQv9jXaszw=="