Merge pull request #18 from MZC-CSC/main

Completed the first round of docker-compose.yaml modernization for testing

.gitignore

@@ -24,5 +24,8 @@ yarn.lock
 src/mayfly
 docker-compose-mode-files/data/*
 
+# test data
+conf/docker/data/*
+
 # IDE's
 *.idea

Dockerfile

@@ -6,6 +6,8 @@
 # 베이스 이미지 선택
 FROM ubuntu:latest as prod
 
+RUN apt-get update && apt-get install -y iputils-ping curl
+
 # 컨테이너 내의 작업 디렉토리 설정(RUN / CMD / ENTRYPOINT 명령이 실행될 기본 경로)
 WORKDIR /app
 

conf/docker/conf/cm-beetle/config.yaml

@@ -0,0 +1,60 @@
+## Set system endpoints
+beetle:
+  root: # To be set in runtime
+
+  cbstore:
+    root: # To be set in runtime (based on beetle.root)
+
+  cblog:
+    root: # To be set in runtime (based on beetle.root)
+
+  tumblebug:
+    rest:
+      url: http://localhost:1323/tumblebug
+
+  ## Logger configuration
+  logfile:
+    # Set log file path (default logfile path: ./beetle.log)
+    path: ./beetle.log
+    maxsize: 10
+    maxbackups: 3
+    maxage: 30
+    compress: false
+
+  # Set log level, such as trace, debug info, warn, error, fatal, and panic
+  loglevel: debug
+
+  # Set log writer, such as file, stdout, or both
+  logwriter: both
+
+  # Set execution environment, such as development or production
+  node:
+    env: development
+
+  ## Set internal DB config (SQLlite)
+  db:
+    url: localhost:3306
+    database: cm_beetle
+    user: cm_beetle
+    password: cm_beetle
+
+  ## Set API access config
+  api:
+    # Set API_ALLOW_ORIGINS (ex: https://cloud-barista.org,http://localhost:8080 or * for all)
+    allow:
+      origins: "*"
+
+    # Set API_AUTH_ENABLED=true currently for basic auth for all routes (i.e., url or path)
+    auth:
+      enabled: true
+
+    username: default
+    password: default
+
+  ## Set period for auto control goroutine invocation
+  autocontrol:
+    duration_ms: 10000
+
+  ## Set SELF_ENDPOINT, to access Swagger API dashboard outside (Ex: export SELF_ENDPOINT=x.x.x.x:8056)
+  self:
+    endpoint: localhost:8056

conf/docker/conf/cm-beetle/log_conf.yaml

@@ -0,0 +1,18 @@
+#### Config for CB-Log Lib. ####
+
+cblog:
+  ## true | false
+  loopcheck: true # This temp method for development is busy wait. cf) cblogger.go:levelSetupLoop().
+
+  ## debug | info | warn | error
+  loglevel: error # If loopcheck is true, You can set this online.
+
+  ## true | false
+  logfile: true
+
+## Config for File Output ##
+logfileinfo:
+  filename: $BEETLE_ROOT/log/cblogs.log
+  maxsize: 10 # megabytes
+  maxbackups: 50
+  maxage: 31 # days

conf/docker/conf/cm-beetle/setup.env

@@ -0,0 +1,45 @@
+## Set system endpoints
+# Set BEETLE_ROOT based on path of setup.env relatively
+SCRIPT_DIR=`dirname ${BASH_SOURCE[0]-$0}`
+export BEETLE_ROOT=`cd $SCRIPT_DIR && cd .. && pwd`
+export BEETLE_CBSTORE_ROOT=$BEETLE_ROOT
+export BEETLE_CBLOG_ROOT=$BEETLE_ROOT
+
+## Set API access config
+# BEETLE_API_ALLOW_ORIGINS (ex: https://cloud-barista.org,http://localhost:8080 or * for all)
+export BEETLE_API_ALLOW_ORIGINS=*
+# Set BEETLE_API_AUTH_ENABLED=true currently for basic auth for all routes (i.e., url or path)
+export BEETLE_API_AUTH_ENABLED=true
+export BEETLE_API_USERNAME=default
+export BEETLE_API_PASSWORD=default
+
+## Set BEETLE_SELF_ENDPOINT, to access Swagger API dashboard outside (Ex: export SELF_ENDPOINT=x.x.x.x:8056)
+export BEETLE_SELF_ENDPOINT=localhost:8056
+
+## Set API access config
+export BEETLE_TUMBLEBUG_REST_URL=http://localhost:1323/tumblebug
+
+
+## Set internal DB config (SQLlite)
+export BEETLE_SQLITE_URL=localhost:3306
+export BEETLE_SQLITE_DATABASE=cm_beetle
+export BEETLE_SQLITE_USER=cm_beetle
+export BEETLE_SQLITE_PASSWORD=cm_beetle
+
+## Logger configuration
+# Set log file path (default logfile path: ./beetle.log) 
+export BEETLE_LOGFILE_PATH=beetle.log
+export BEETLE_LOGFILE_MAXSIZE=10
+export BEETLE_LOGFILE_MAXBACKUPS=3
+export BEETLE_LOGFILE_MAXAGE=30
+export BEETLE_LOGFILE_COMPRESS=false
+# Set log level, such as trace, debug info, warn, error, fatal, and panic
+export BEETLE_LOGLEVEL=debug
+# Set log writer, such as file, stdout, or both
+export BEETLE_LOGWRITER=both
+# Set execution environment, such as development or production
+export BEETLE_NODE_ENV=development
+
+
+## Set period for auto control goroutine invocation
+export BEETLE_AUTOCONTROL_DURATION_MS=10000

conf/docker/conf/cm-beetle/store_conf.yaml

@@ -0,0 +1,15 @@
+#### Config for CB-Store ####
+
+# server type: NUTSDB | ETCD
+# NUTSDB: embedded Key-Value Store on Local Filesystem
+storetype: NUTSDB
+#storetype: ETCD
+
+nutsdb:
+  dbpath: "$BEETLE_ROOT/meta_db/dat"
+  segmentsize: 1048576  # 1048576 1024*1024 (1MB)
+  #segmentsize: 10485760  # 10485760 10*1024*1024 (10MB)
+
+etcd:
+  # etcd server, when ETCD typpe
+  etcdserverport: "129.254.175.43:2379"

conf/docker/conf/cm-beetle/template-config.yaml

@@ -0,0 +1,61 @@
+## Set system endpoints
+beetle:
+  root: # To be set in runtime
+
+  cbstore:
+    root: # To be set in runtime (based on beetle.root)
+
+  cblog:
+    root: # To be set in runtime (based on beetle.root)
+
+  ## Set Tumblebug API endpoint
+  tumblebug:
+    rest:
+      url: http://localhost:1323/tumblebug
+
+  ## Logger configuration
+  logfile:
+    # Set log file path (default logfile path: ./beetle.log)
+    path: ./beetle.log
+    maxsize: 10
+    maxbackups: 3
+    maxage: 30
+    compress: false
+
+  # Set log level, such as trace, debug info, warn, error, fatal, and panic
+  loglevel: debug
+
+  # Set log writer, such as file, stdout, or both
+  logwriter: both
+
+  # Set execution environment, such as development or production
+  node:
+    env: development
+
+  ## Set internal DB config (SQLlite)
+  db:
+    url: localhost:3306
+    database: cm_beetle
+    user: cm_beetle
+    password: cm_beetle
+
+  ## Set API access config
+  api:
+    # Set API_ALLOW_ORIGINS (ex: https://cloud-barista.org,http://localhost:8080 or * for all)
+    allow:
+      origins: "*"
+
+    # Set API_AUTH_ENABLED=true currently for basic auth for all routes (i.e., url or path)
+    auth:
+      enabled: true
+
+    username: default
+    password: default
+
+  ## Set period for auto control goroutine invocation
+  autocontrol:
+    duration_ms: 10000
+
+  ## Set SELF_ENDPOINT, to access Swagger API dashboard outside (Ex: export SELF_ENDPOINT=x.x.x.x:8056)
+  self:
+    endpoint: localhost:8056

conf/docker/conf/cm-beetle/template-setup.env

@@ -0,0 +1,43 @@
+## Set system endpoints
+# Set BEETLE_ROOT based on path of setup.env relatively
+SCRIPT_DIR=`dirname ${BASH_SOURCE[0]-$0}`
+export BEETLE_ROOT=`cd $SCRIPT_DIR && cd .. && pwd`
+export BEETLE_CBSTORE_ROOT=$BEETLE_ROOT
+export BEETLE_CBLOG_ROOT=$BEETLE_ROOT
+
+## Set API access config
+export BEETLE_TUMBLEBUG_REST_URL=http://localhost:1323/tumblebug
+
+## Logger configuration
+# Set log file path (default logfile path: ./beetle.log) 
+export BEETLE_LOGFILE_PATH=beetle.log
+export BEETLE_LOGFILE_MAXSIZE=10
+export BEETLE_LOGFILE_MAXBACKUPS=3
+export BEETLE_LOGFILE_MAXAGE=30
+export BEETLE_LOGFILE_COMPRESS=false
+# Set log level, such as trace, debug info, warn, error, fatal, and panic
+export BEETLE_LOGLEVEL=debug
+# Set log writer, such as file, stdout, or both
+export BEETLE_LOGWRITER=both
+# Set execution environment, such as development or production
+export BEETLE_NODE_ENV=development
+
+## Set internal DB config (SQLlite)
+export BEETLE_SQLITE_URL=localhost:3306
+export BEETLE_SQLITE_DATABASE=cm_beetle
+export BEETLE_SQLITE_USER=cm_beetle
+export BEETLE_SQLITE_PASSWORD=cm_beetle
+
+## Set API access config
+# BEETLE_API_ALLOW_ORIGINS (ex: https://cloud-barista.org,http://localhost:8080 or * for all)
+export BEETLE_API_ALLOW_ORIGINS=*
+# Set BEETLE_API_AUTH_ENABLED=true currently for basic auth for all routes (i.e., url or path)
+export BEETLE_API_AUTH_ENABLED=true
+export BEETLE_API_USERNAME=default
+export BEETLE_API_PASSWORD=default
+
+## Set period for auto control goroutine invocation
+export BEETLE_AUTOCONTROL_DURATION_MS=10000
+
+## Set BEETLE_SELF_ENDPOINT, to access Swagger API dashboard outside (Ex: export SELF_ENDPOINT=x.x.x.x:8056)
+export BEETLE_SELF_ENDPOINT=localhost:8056

conf/docker/conf/cm-butterfly/conf/setup.env

@@ -0,0 +1,24 @@
+## for dynamic target, using ${CBWS_TARGET}
+# common WIP env for test/demo set export CBWS_TARGET=
+#export CBWS_TARGET=localhost
+export CBWS_TARGET=localhost
+
+## localhost
+export SPIDER_URL=${SPIDER_URL:-http://$CBWS_TARGET:1024/spider}
+export TUMBLE_URL=${TUMBLE_URL:-http://$CBWS_TARGET:1323/tumblebug}
+export DRAGONFLY_URL=${DRAGONFLY_URL:-http://$CBWS_TARGET:9090/dragonfly}
+export CICADA_URL=${CICADA_URL:-http://$CBWS_TARGET:8083/cicada}
+export HONEYBEE_URL=${HONEYBEE_URL:-http://$CBWS_TARGET:8081/honeybee}
+export BEETLE_URL=${BEETLE_URL:-http://$CBWS_TARGET:8056/beetle}
+
+export API_GW=${API_GW:-http://$CBWS_TARGET:8000}
+export LADYBUG_URL=${LADYBUG_URL:-http://$CBWS_TARGET:1470/ladybug}
+
+export LoginUser=${LoginUser:-admin}
+export LoginEmail=${LoginEmail:-admin}
+export LoginPassword=${LoginPassword:-admin}
+export LoginAccessSecret=${LoginAccessSecret:-admin}
+export LoginRefreshSecret=${LoginRefreshSecret:-admin}
+
+export API_USERNAME=${API_USERNAME:-default}
+export API_PASSWORD=${API_PASSWORD:-default}

conf/docker/conf/cm-cicada/_airflow/.gitignore

@@ -0,0 +1 @@
+db_data/

conf/docker/conf/cm-cicada/_airflow/Dockerfile

@@ -0,0 +1,21 @@
+FROM python:3.11.9-slim-bookworm
+USER root
+
+# MySQL Requirements
+RUN apt-get update && apt-get install -y python3-dev default-libmysqlclient-dev build-essential pkg-config && apt-get clean
+
+# Python Requirements
+ADD requirements.txt .
+RUN pip3 install -r requirements.txt
+RUN pip3 install connexion
+RUN pip3 install "connexion[swagger-ui]"
+RUN pip3 cache purge
+
+# Airflow Env Vars
+ENV AIRFLOW_HOME='/usr/local/airflow'
+
+# Set wd
+WORKDIR /usr/local/airflow
+
+# Sleep forever
+CMD sleep infinity

conf/docker/conf/cm-cicada/_airflow/airflow-home/.gitignore

@@ -0,0 +1,6 @@
+/logs/
+/dags/**/
+*_test.py
+
+/*.pid
+/*.py