Merge pull request #218 from cisagov/v6.3.0_merge_cisagov

* New Features
    * Support remote OpenSearch instance/cluster as alternative to local containerized instance (idaholab#110)
    * Documentation and convenience scripts for configuring ingestion of third-party logs, and basic parsing/normalizing of Fluent-Bit's Windows event logs
    * S7comm Plus support and replaced Amazon S7comm parser with icsnpp-s7comm (idaholab#99)

* Version Bumps
    * OpenSearch and OpenSearch Dashboards to v2.2.1
    * Zeek to v5.0.1
    * Spicy to v1.5.1
    * spicy-plugin to v1.3.17
    * YARA to v4.2.3
    * Capa to v4.0.1

* Improvements
    * Major improvements to OPC UA Binary parser and supporting dashboards
    * Ensure that all containers are provided the same information about trusted CA certificates
    * changed list of "sensitive countries" to match U.S. Department of Energy Sensitive Country List
    * Increased maximum fields from 3,000 to 5,000
    * Standardized configuration and authentication for primary and secondary remote OpenSearch instances, and make sure that index templates are created on secondary remote OpenSearch instances
    * Expand and fix normalization of [network.direction](https://www.elastic.co/guide/en/ecs/current/ecs-network.html#field-network-direction) in lieu of using tags
    * Various tweaks and improvements to the `install.py` script for enabling/disabling some features

* Bugs Fixed
    * fields could be missing in Arkime due to a large number of concurrent requests (idaholab#115)
    * mapper_parsing_exception, TCP flag parsing problem (#214)

.dockerignore

@@ -10,8 +10,9 @@
 **/.ldap_config_defaults
 **/htpasswd
 **/malcolm_*images.tar.gz
-.envrc
-.direnv
+**/.envrc
+**/.direnv
+**/.opensearch*.curlrc
 .trigger_workflow_build
 .tmp
 docker-compose*yml
@@ -23,5 +24,6 @@ arkime-raw
 nginx/nginx_ldap.conf
 pcap
 scripts
+!scripts/malcolm_common.py
 zeek-logs
 suricata-logs

.gitignore

@@ -7,6 +7,7 @@
 # auth-related
 /.ldap_config_defaults
 /auth.env
+/.opensearch*.curlrc
 /logstash/certs
 /filebeat/certs
 /nginx/certs

Dockerfiles/api.Dockerfile

@@ -12,10 +12,11 @@ RUN    apt-get update -q \
     && python3 -m pip install flake8
 
 COPY ./api /usr/src/app/
+COPY scripts/malcolm_common.py /usr/src/app/
 WORKDIR /usr/src/app
 
 RUN python3 -m pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheels -r requirements.txt \
-    && flake8 --ignore=E501,F401
+    && flake8 --ignore=E501,F401,W503
 
 FROM python:3-slim
 
@@ -48,6 +49,7 @@ ARG ARKIME_INDEX_PATTERN="arkime_sessions3-*"
 ARG ARKIME_INDEX_TIME_FIELD="firstPacket"
 ARG DASHBOARDS_URL="http://dashboards:5601/dashboards"
 ARG OPENSEARCH_URL="http://opensearch:9200"
+ARG OPENSEARCH_LOCAL=true
 ARG RESULT_SET_LIMIT="500"
 
 ENV HOME=/malcolm
@@ -60,13 +62,15 @@ ENV ARKIME_INDEX_PATTERN $ARKIME_INDEX_PATTERN
 ENV ARKIME_INDEX_TIME_FIELD $ARKIME_INDEX_TIME_FIELD
 ENV DASHBOARDS_URL $DASHBOARDS_URL
 ENV OPENSEARCH_URL $OPENSEARCH_URL
+ENV OPENSEARCH_LOCAL $OPENSEARCH_LOCAL
 ENV RESULT_SET_LIMIT $RESULT_SET_LIMIT
 
 WORKDIR "${APP_HOME}"
 
 COPY --from=builder /usr/src/app/wheels /wheels
 COPY --from=builder /usr/src/app/requirements.txt .
 COPY ./api "${APP_HOME}"
+COPY scripts/malcolm_common.py "${APP_HOME}"/
 COPY shared/bin/opensearch_status.sh "${APP_HOME}"/
 
 ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/

Dockerfiles/arkime.Dockerfile

@@ -5,16 +5,16 @@ FROM debian:11-slim AS build
 ENV DEBIAN_FRONTEND noninteractive
 
 ENV ARKIME_VERSION "3.4.2"
-ENV ARKIMEDIR "/opt/arkime"
+ENV ARKIME_DIR "/opt/arkime"
 ENV ARKIME_URL "https://github.com/arkime/arkime.git"
 ENV ARKIME_LOCALELASTICSEARCH no
 ENV ARKIME_INET yes
 
 ADD arkime/scripts/bs4_remove_div.py /opt/
 ADD arkime/patch/* /opt/patches/
-ADD README.md $ARKIMEDIR/doc/
-ADD docs/doc.css $ARKIMEDIR/doc/
-ADD docs/images $ARKIMEDIR/doc/images/
+ADD README.md $ARKIME_DIR/doc/
+ADD docs/doc.css $ARKIME_DIR/doc/
+ADD docs/images $ARKIME_DIR/doc/images/
 
 RUN apt-get -q update && \
     apt-get -y -q --no-install-recommends upgrade && \
@@ -56,24 +56,24 @@ RUN apt-get -q update && \
         wget \
         zlib1g-dev && \
   pip3 install --no-cache-dir beautifulsoup4 && \
-  cd $ARKIMEDIR/doc/images && \
+  cd $ARKIME_DIR/doc/images && \
     find . -name "*.png" -exec bash -c 'convert "{}" -fuzz 2% -transparent white -background white -alpha remove -strip -interlace Plane -quality 85% "{}.jpg" && rename "s/\.png//" "{}.jpg"' \; && \
-    cd $ARKIMEDIR/doc && \
+    cd $ARKIME_DIR/doc && \
     sed -i "s/^# Malcolm$//" README.md && \
     sed -i '/./,$!d' README.md && \
     sed -i "s/.png/.jpg/g" README.md && \
     sed -i "s@docs/images@images@g" README.md && \
     sed -i 's/\!\[.*\](.*\/badge.svg)//g' README.md && \
-    pandoc -s --self-contained --metadata title="Malcolm README" --css $ARKIMEDIR/doc/doc.css -o $ARKIMEDIR/doc/README.html $ARKIMEDIR/doc/README.md && \
+    pandoc -s --self-contained --metadata title="Malcolm README" --css $ARKIME_DIR/doc/doc.css -o $ARKIME_DIR/doc/README.html $ARKIME_DIR/doc/README.md && \
   cd /opt && \
     git clone --depth=1 --single-branch --recurse-submodules --shallow-submodules --no-tags --branch="v$ARKIME_VERSION" "$ARKIME_URL" "./arkime-"$ARKIME_VERSION && \
     cd "./arkime-"$ARKIME_VERSION && \
     bash -c 'for i in /opt/patches/*; do patch -p 1 -r - --no-backup-if-mismatch < $i || true; done' && \
-    find $ARKIMEDIR/doc/images/screenshots -name "*.png" -delete && \
-    export PATH="$ARKIMEDIR/bin:${PATH}" && \
-    ln -sfr $ARKIMEDIR/bin/npm /usr/local/bin/npm && \
-    ln -sfr $ARKIMEDIR/bin/node /usr/local/bin/node && \
-    ln -sfr $ARKIMEDIR/bin/npx /usr/local/bin/npx && \
+    find $ARKIME_DIR/doc/images/screenshots -name "*.png" -delete && \
+    export PATH="$ARKIME_DIR/bin:${PATH}" && \
+    ln -sfr $ARKIME_DIR/bin/npm /usr/local/bin/npm && \
+    ln -sfr $ARKIME_DIR/bin/node /usr/local/bin/node && \
+    ln -sfr $ARKIME_DIR/bin/npx /usr/local/bin/npx && \
     python3 /opt/bs4_remove_div.py -i ./viewer/vueapp/src/components/users/Users.vue -o ./viewer/vueapp/src/components/users/Users.new -c "new-user-form" && \
     mv -vf ./viewer/vueapp/src/components/users/Users.new ./viewer/vueapp/src/components/users/Users.vue && \
     sed -i 's/v-if.*password.*"/v-if="false"/g' ./viewer/vueapp/src/components/settings/Settings.vue && \
@@ -84,8 +84,8 @@ RUN apt-get -q update && \
     npm -g config set user root && \
     make install && \
     npm cache clean --force && \
-    rm -f ${ARKIMEDIR}/wiseService/source.* && \
-    bash -c "file ${ARKIMEDIR}/bin/* ${ARKIMEDIR}/node-v*/bin/* | grep 'ELF 64-bit' | sed 's/:.*//' | xargs -l -r strip -v --strip-unneeded"
+    rm -f ${ARKIME_DIR}/wiseService/source.* && \
+    bash -c "file ${ARKIME_DIR}/bin/* ${ARKIME_DIR}/node-v*/bin/* | grep 'ELF 64-bit' | sed 's/:.*//' | xargs -l -r strip -v --strip-unneeded"
 
 FROM debian:11-slim
 
@@ -109,8 +109,8 @@ ENV PUSER_PRIV_DROP true
 ENV DEBIAN_FRONTEND noninteractive
 ENV TERM xterm
 
-ARG OS_HOST=opensearch
-ARG OS_PORT=9200
+ARG OPENSEARCH_URL="http://opensearch:9200"
+ARG OPENSEARCH_LOCAL=true
 ARG MALCOLM_USERNAME=admin
 ARG ARKIME_ECS_PROVIDER=arkime
 ARG ARKIME_ECS_DATASET=session
@@ -128,16 +128,15 @@ ARG PCAP_MONITOR_HOST=pcap-monitor
 ARG MAXMIND_GEOIP_DB_LICENSE_KEY=""
 
 # Declare envs vars for each arg
-ENV OS_HOST $OS_HOST
-ENV OS_PORT $OS_PORT
-ENV ARKIME_ELASTICSEARCH "http://"$OS_HOST":"$OS_PORT
+ENV OPENSEARCH_URL $OPENSEARCH_URL
+ENV OPENSEARCH_LOCAL $OPENSEARCH_LOCAL
 ENV ARKIME_INTERFACE $ARKIME_INTERFACE
 ENV MALCOLM_USERNAME $MALCOLM_USERNAME
 # this needs to be present, but is unused as nginx is going to handle auth for us
 ENV ARKIME_PASSWORD "ignored"
 ENV ARKIME_ECS_PROVIDER $ARKIME_ECS_PROVIDER
 ENV ARKIME_ECS_DATASET $ARKIME_ECS_DATASET
-ENV ARKIMEDIR "/opt/arkime"
+ENV ARKIME_DIR "/opt/arkime"
 ENV ARKIME_ANALYZE_PCAP_THREADS $ARKIME_ANALYZE_PCAP_THREADS
 ENV WISE $WISE
 ENV VIEWER $VIEWER
@@ -147,7 +146,7 @@ ENV PCAP_PIPELINE_DEBUG $PCAP_PIPELINE_DEBUG
 ENV PCAP_PIPELINE_DEBUG_EXTRA $PCAP_PIPELINE_DEBUG_EXTRA
 ENV PCAP_MONITOR_HOST $PCAP_MONITOR_HOST
 
-COPY --from=build $ARKIMEDIR $ARKIMEDIR
+COPY --from=build $ARKIME_DIR $ARKIME_DIR
 
 RUN sed -i "s/bullseye main/bullseye main contrib non-free/g" /etc/apt/sources.list && \
     apt-get -q update && \
@@ -181,9 +180,9 @@ RUN sed -i "s/bullseye main/bullseye main contrib non-free/g" /etc/apt/sources.l
       wget \
       tar gzip unzip cpio bzip2 lzma xz-utils p7zip-full unrar zlib1g && \
     pip3 install --no-cache-dir beautifulsoup4 pyzmq && \
-    ln -sfr $ARKIMEDIR/bin/npm /usr/local/bin/npm && \
-      ln -sfr $ARKIMEDIR/bin/node /usr/local/bin/node && \
-      ln -sfr $ARKIMEDIR/bin/npx /usr/local/bin/npx && \
+    ln -sfr $ARKIME_DIR/bin/npm /usr/local/bin/npm && \
+      ln -sfr $ARKIME_DIR/bin/node /usr/local/bin/node && \
+      ln -sfr $ARKIME_DIR/bin/npx /usr/local/bin/npx && \
     apt-get -q -y --purge remove gcc gcc-10 cpp cpp-10 libssl-dev && \
       apt-get -q -y autoremove && \
       apt-get clean && \
@@ -194,9 +193,10 @@ ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD arkime/scripts /opt/
 ADD shared/bin/pcap_processor.py /opt/
 ADD shared/bin/pcap_utils.py /opt/
+ADD scripts/malcolm_common.py /opt/
 ADD shared/bin/opensearch_status.sh /opt/
-ADD arkime/etc $ARKIMEDIR/etc/
-ADD arkime/wise/source.*.js $ARKIMEDIR/wiseService/
+ADD arkime/etc $ARKIME_DIR/etc/
+ADD arkime/wise/source.*.js $ARKIME_DIR/wiseService/
 ADD arkime/supervisord.conf /etc/supervisord.conf
 
 # MaxMind now requires a (free) license key to download the free versions of
@@ -208,29 +208,30 @@ RUN [ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ] && for DB in ASN Country City; do
       cd /tmp && \
       curl -s -S -L -o "GeoLite2-$DB.mmdb.tar.gz" "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-$DB&license_key=$MAXMIND_GEOIP_DB_LICENSE_KEY&suffix=tar.gz" && \
       tar xf "GeoLite2-$DB.mmdb.tar.gz" --wildcards --no-anchored '*.mmdb' --strip=1 && \
-      mkdir -p $ARKIMEDIR/etc/ $ARKIMEDIR/logs/ && \
-      mv -v "GeoLite2-$DB.mmdb" $ARKIMEDIR/etc/; \
+      mkdir -p $ARKIME_DIR/etc/ $ARKIME_DIR/logs/ && \
+      mv -v "GeoLite2-$DB.mmdb" $ARKIME_DIR/etc/; \
       rm -f "GeoLite2-$DB*"; \
     done; \
-  curl -s -S -L -o $ARKIMEDIR/etc/ipv4-address-space.csv "https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv" && \
-  curl -s -S -L -o $ARKIMEDIR/etc/oui.txt "https://raw.githubusercontent.com/wireshark/wireshark/master/manuf"
+  curl -s -S -L -o $ARKIME_DIR/etc/ipv4-address-space.csv "https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv" && \
+  curl -s -S -L -o $ARKIME_DIR/etc/oui.txt "https://raw.githubusercontent.com/wireshark/wireshark/master/manuf"
 
 RUN groupadd --gid $DEFAULT_GID $PGROUP && \
-    useradd -M --uid $DEFAULT_UID --gid $DEFAULT_GID --home $ARKIMEDIR $PUSER && \
+    useradd -M --uid $DEFAULT_UID --gid $DEFAULT_GID --home $ARKIME_DIR $PUSER && \
       usermod -a -G tty $PUSER && \
     chmod 755 /opt/*.sh && \
     ln -sfr /opt/pcap_processor.py /opt/pcap_arkime_processor.py && \
-    cp -f /opt/arkime_update_geo.sh $ARKIMEDIR/bin/arkime_update_geo.sh && \
-    chmod u+s $ARKIMEDIR/bin/capture && \
+    cp -f /opt/arkime_update_geo.sh $ARKIME_DIR/bin/arkime_update_geo.sh && \
+    mv $ARKIME_DIR/etc/config.ini $ARKIME_DIR/etc/config.orig.ini && \
+    chmod u+s $ARKIME_DIR/bin/capture && \
     mkdir -p /var/run/arkime && \
-    chown -R $PUSER:$PGROUP $ARKIMEDIR/etc $ARKIMEDIR/logs /var/run/arkime
+    chown -R $PUSER:$PGROUP $ARKIME_DIR/etc $ARKIME_DIR/logs /var/run/arkime
 #Update Path
-ENV PATH="/opt:$ARKIMEDIR/bin:${PATH}"
+ENV PATH="/opt:$ARKIME_DIR/bin:${PATH}"
 
 EXPOSE 8000 8005 8081
-WORKDIR $ARKIMEDIR
+WORKDIR $ARKIME_DIR
 
-ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
+ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh", "/opt/docker_entrypoint.sh"]
 
 CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf", "-n"]
 

Dockerfiles/dashboards-helper.Dockerfile

@@ -25,6 +25,7 @@ ARG ARKIME_INDEX_PATTERN_ID="arkime_sessions3-*"
 ARG ARKIME_INDEX_TIME_FIELD="firstPacket"
 ARG CREATE_OS_ARKIME_SESSION_INDEX="true"
 ARG OPENSEARCH_URL="http://opensearch:9200"
+ARG OPENSEARCH_LOCAL=true
 ARG ISM_SNAPSHOT_COMPRESSED=false
 ARG ISM_SNAPSHOT_REPO=logs
 ARG OFFLINE_REGION_MAPS_PORT="28991"
@@ -36,6 +37,7 @@ ENV ARKIME_INDEX_PATTERN_ID $ARKIME_INDEX_PATTERN_ID
 ENV ARKIME_INDEX_TIME_FIELD $ARKIME_INDEX_TIME_FIELD
 ENV CREATE_OS_ARKIME_SESSION_INDEX $CREATE_OS_ARKIME_SESSION_INDEX
 ENV OPENSEARCH_URL $OPENSEARCH_URL
+ENV OPENSEARCH_LOCAL $OPENSEARCH_LOCAL
 ENV ISM_SNAPSHOT_COMPRESSED $ISM_SNAPSHOT_COMPRESSED
 ENV ISM_SNAPSHOT_REPO $ISM_SNAPSHOT_REPO
 ENV OFFLINE_REGION_MAPS_PORT $OFFLINE_REGION_MAPS_PORT
@@ -63,10 +65,11 @@ ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD shared/bin/opensearch_status.sh /data/
 ADD shared/bin/opensearch_index_size_prune.py /data/
 ADD shared/bin/opensearch_read_only.py /data/
+ADD scripts/malcolm_common.py /data/
 
 RUN apk update --no-cache && \
     apk upgrade --no-cache && \
-    apk --no-cache add bash python3 py3-pip curl procps psmisc npm shadow jq && \
+    apk --no-cache add bash python3 py3-pip curl openssl procps psmisc npm shadow jq && \
     npm install -g http-server && \
     pip3 install supervisor humanfriendly requests && \
     curl -fsSLO "$SUPERCRONIC_URL" && \

Dockerfiles/dashboards.Dockerfile

@@ -14,10 +14,10 @@ ENV PGROUP "dashboarder"
 
 ENV TERM xterm
 
-ARG OPENSEARCH_VERSION="2.1.0"
+ARG OPENSEARCH_VERSION="2.2.1"
 ENV OPENSEARCH_VERSION $OPENSEARCH_VERSION
 
-ARG OPENSEARCH_DASHBOARDS_VERSION="2.1.0"
+ARG OPENSEARCH_DASHBOARDS_VERSION="2.2.1"
 ENV OPENSEARCH_DASHBOARDS_VERSION $OPENSEARCH_DASHBOARDS_VERSION
 
 # base system dependencies for checking out and building plugins
@@ -68,7 +68,7 @@ RUN eval "$(nodenv init -)" && \
 
 # runtime ##################################################################
 
-FROM opensearchproject/opensearch-dashboards:2.1.0
+FROM opensearchproject/opensearch-dashboards:2.2.1
 
 LABEL maintainer="[email protected]"
 LABEL org.opencontainers.image.authors='[email protected]'
@@ -90,6 +90,7 @@ ENV PUSER_PRIV_DROP true
 ENV TERM xterm
 
 ARG OPENSEARCH_URL="http://opensearch:9200"
+ARG OPENSEARCH_LOCAL="true"
 ARG CREATE_OS_ARKIME_SESSION_INDEX="true"
 ARG ARKIME_INDEX_PATTERN="arkime_sessions3-*"
 ARG ARKIME_INDEX_PATTERN_ID="arkime_sessions3-*"
@@ -104,14 +105,15 @@ ENV ARKIME_INDEX_TIME_FIELD $ARKIME_INDEX_TIME_FIELD
 ENV OPENSEARCH_DEFAULT_DASHBOARD $OPENSEARCH_DEFAULT_DASHBOARD
 ENV PATH="/data:${PATH}"
 ENV OPENSEARCH_URL $OPENSEARCH_URL
+ENV OPENSEARCH_LOCAL $OPENSEARCH_LOCAL
 ENV NODE_OPTIONS $NODE_OPTIONS
 
 USER root
 
 COPY --from=build /usr/share/opensearch-dashboards/plugins/sankey_vis/build/kbnSankeyVis.zip /tmp/kbnSankeyVis.zip
 
 RUN yum upgrade -y && \
-    yum install -y curl psmisc util-linux zip unzip && \
+    yum install -y curl psmisc util-linux openssl python3 zip unzip && \
     usermod -a -G tty ${PUSER} && \
     # Malcolm manages authentication and encryption via NGINX reverse proxy
     /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove securityDashboards --allow-root && \
@@ -121,8 +123,10 @@ RUN yum upgrade -y && \
     yum clean all && \
     rm -rf /var/cache/yum
 
-ADD dashboards/opensearch_dashboards.yml /usr/share/opensearch-dashboards/config/opensearch_dashboards.yml
+ADD dashboards/opensearch_dashboards.yml /usr/share/opensearch-dashboards/config/opensearch_dashboards.orig.yml
 ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
+ADD dashboards/scripts/docker_entrypoint.sh /usr/local/bin/
+ADD scripts/malcolm_common.py /usr/local/bin/
 
 # Yeah, I know about https://opensearch.org/docs/latest/dashboards/branding ... but I can't figure out a way
 # to specify the entries in the opensearch_dashboards.yml such that they are valid BOTH from the
@@ -138,7 +142,7 @@ ADD docs/images/favicon/favicon32.png /usr/share/opensearch-dashboards/src/core/
 ADD docs/images/favicon/apple-touch-icon-precomposed.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/apple-touch-icon.png
 
 
-ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
+ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh", "/usr/local/bin/docker_entrypoint.sh"]
 
 CMD ["/usr/share/opensearch-dashboards/opensearch-dashboards-docker-entrypoint.sh"]
 

Dockerfiles/file-monitor.Dockerfile

@@ -75,12 +75,12 @@ ENV EXTRACTED_FILE_ENABLE_CAPA $EXTRACTED_FILE_ENABLE_CAPA
 ENV EXTRACTED_FILE_CAPA_VERBOSE $EXTRACTED_FILE_CAPA_VERBOSE
 ENV SRC_BASE_DIR "/usr/local/src"
 ENV CLAMAV_RULES_DIR "/var/lib/clamav"
-ENV YARA_VERSION "4.2.2"
+ENV YARA_VERSION "4.2.3"
 ENV YARA_URL "https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.tar.gz"
 ENV YARA_RULES_URL "https://github.com/Neo23x0/signature-base"
 ENV YARA_RULES_DIR "/yara-rules"
 ENV YARA_RULES_SRC_DIR "$SRC_BASE_DIR/signature-base"
-ENV CAPA_VERSION "3.2.0"
+ENV CAPA_VERSION "4.0.1"
 ENV CAPA_URL "https://github.com/fireeye/capa/releases/download/v${CAPA_VERSION}/capa-v${CAPA_VERSION}-linux.zip"
 ENV CAPA_DIR "/opt/capa"
 ENV CAPA_BIN "${CAPA_DIR}/capa"

Dockerfiles/filebeat.Dockerfile

@@ -39,6 +39,8 @@ ARG FILEBEAT_NGINX_LOG_PATH="/nginx"
 ARG LOG_CLEANUP_MINUTES=0
 ARG ZIP_CLEANUP_MINUTES=0
 ARG NGINX_LOG_ACCESS_AND_ERRORS=false
+ARG OPENSEARCH_URL="http://opensearch:9200"
+ARG OPENSEARCH_LOCAL=true
 ARG FILEBEAT_TCP_LISTEN=false
 ARG FILEBEAT_TCP_PORT=5045
 ARG FILEBEAT_TCP_LOG_FORMAT="raw"
@@ -64,7 +66,7 @@ USER root
 
 RUN yum install -y epel-release && \
         yum upgrade -y && \
-        yum install -y curl inotify-tools file psmisc tar gzip unzip cpio bzip2 lzma xz p7zip p7zip-plugins unar python3-setuptools python3-pip && \
+        yum install -y curl inotify-tools file psmisc tar gzip unzip cpio bzip2 lzma xz openssl p7zip p7zip-plugins unar python3-setuptools python3-pip && \
         yum clean all && \
     ln -sr /usr/sbin/fuser /bin/fuser && \
     python3 -m pip install patool entrypoint2 pyunpack python-magic ordered-set supervisor && \
@@ -79,6 +81,7 @@ ADD filebeat/filebeat.yml /usr/share/filebeat/filebeat.yml
 ADD filebeat/filebeat-nginx.yml /usr/share/filebeat-nginx/filebeat-nginx.yml
 ADD filebeat/filebeat-tcp.yml /usr/share/filebeat-tcp/filebeat-tcp.yml
 ADD filebeat/scripts /usr/local/bin/
+ADD scripts/malcolm_common.py /usr/local/bin/
 ADD shared/bin/opensearch_status.sh /usr/local/bin/
 ADD filebeat/supervisord.conf /etc/supervisord.conf
 RUN for INPUT in nginx tcp; do \
@@ -107,6 +110,8 @@ ENV FILEBEAT_ZEEK_LOG_LIVE_PATH $FILEBEAT_ZEEK_LOG_LIVE_PATH
 ENV FILEBEAT_SURICATA_LOG_PATH $FILEBEAT_SURICATA_LOG_PATH
 ENV FILEBEAT_NGINX_LOG_PATH $FILEBEAT_NGINX_LOG_PATH
 ENV NGINX_LOG_ACCESS_AND_ERRORS $NGINX_LOG_ACCESS_AND_ERRORS
+ENV OPENSEARCH_URL $OPENSEARCH_URL
+ENV OPENSEARCH_LOCAL $OPENSEARCH_LOCAL
 ENV FILEBEAT_TCP_LISTEN $FILEBEAT_TCP_LISTEN
 ENV FILEBEAT_TCP_PORT $FILEBEAT_TCP_PORT
 ENV FILEBEAT_TCP_LOG_FORMAT $FILEBEAT_TCP_LOG_FORMAT

Dockerfiles/logstash.Dockerfile

@@ -105,6 +105,7 @@ ADD logstash/pipelines/ /usr/share/logstash/malcolm-pipelines/
 ADD logstash/patterns/ /usr/share/logstash/malcolm-patterns/
 ADD logstash/ruby/ /usr/share/logstash/malcolm-ruby/
 ADD logstash/scripts /usr/local/bin/
+ADD scripts/malcolm_common.py /usr/local/bin/
 ADD logstash/supervisord.conf /etc/supervisord.conf
 
 RUN bash -c "chmod --silent 755 /usr/local/bin/*.sh /usr/local/bin/*.py || true" && \