From 3851592ef5a7ee6c0478f52b8e6d2380aaa37586 Mon Sep 17 00:00:00 2001 From: SG <13872653+mmguero@users.noreply.github.com> Date: Fri, 28 Jun 2019 09:22:30 -0600 Subject: [PATCH] reduce moloch image size by 1/2 by using multistage build (#30) * reduce size of moloch image by 50% by using multi-stage --- Dockerfiles/moloch.Dockerfile | 289 +++++++++++++--------------- Dockerfiles/pcap-capture.Dockerfile | 2 +- moloch/scripts/zeek-process-pcap.py | 3 +- 3 files changed, 135 insertions(+), 159 deletions(-) diff --git a/Dockerfiles/moloch.Dockerfile b/Dockerfiles/moloch.Dockerfile index fbabd67d7..f252f2fe7 100644 --- a/Dockerfiles/moloch.Dockerfile +++ b/Dockerfiles/moloch.Dockerfile @@ -1,79 +1,118 @@ -FROM debian:stretch-slim +FROM debian:stretch-slim AS build # Copyright (c) 2019 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="Seth.Grover@inl.gov" ENV DEBIAN_FRONTEND noninteractive +ENV MOLOCH_VERSION "1.8.0" +ENV MOLOCHDIR "/data/moloch" +ENV ZEEK_VERSION "2.6.2" +ENV ZEEK_DIR "/opt/bro" +ENV CYBERCHEF_VERSION "8.30.1" + +ADD moloch/patch/* /data/patches/ +ADD README.md $MOLOCHDIR/doc/ +ADD doc.css $MOLOCHDIR/doc/ +ADD docs/images $MOLOCHDIR/doc/images/ +ADD https://github.com/aol/moloch/archive/v$MOLOCH_VERSION.tar.gz /data/moloch.tar.gz +ADD https://github.com/gchq/CyberChef/releases/download/v$CYBERCHEF_VERSION/cyberchef.htm $MOLOCHDIR/doc/cyberchef.htm +ADD https://www.zeek.org/downloads/bro-$ZEEK_VERSION.tar.gz /data/bro.tar.gz + RUN sed -i "s/stretch main/stretch main contrib non-free/" /etc/apt/sources.list && \ apt-get -q update && \ - bash -c "echo 'localepurge localepurge/nopurge multiselect en,en_US.UTF-8' | debconf-set-selections" && \ apt-get install -q -y --no-install-recommends \ bison \ - cgdb \ cmake \ - cron \ curl \ - ethtool \ - file \ flex \ g++ \ gcc \ - gdb \ - geoip-bin \ + gettext \ git \ groff \ groff-base \ imagemagick \ - inotify-tools \ + libcap-dev \ libgoogle-perftools-dev \ - libgoogle-perftools4 \ libjson-perl \ - libkrb5-3 \ libkrb5-dev \ libmaxminddb-dev \ - libmaxminddb0 \ - libpcap0.8 \ libpcap0.8-dev \ - libssl1.0 \ libssl1.0-dev \ libtool \ libwww-perl \ libyaml-dev \ - localepurge \ make \ ninja-build \ pandoc \ patch \ - psmisc \ - python \ python-dev \ - python3 \ python3-dev \ - python3-pip \ - python3-setuptools \ - python3-wheel \ rename \ sudo \ - supervisor \ swig \ - tshark \ - vim-tiny \ wget \ - zlib1g-dev \ - tar gzip unzip cpio bzip2 lzma xz-utils p7zip-full unrar zlib1g && \ - dpkg-reconfigure localepurge && \ - localepurge && \ - pip3 install --no-cache-dir elasticsearch manuf geoip2 patool entrypoint2 pyunpack && \ - apt-get -q -y --purge remove python3-dev && \ - apt-get -q -y autoremove && \ + zlib1g-dev && \ + cd /data && \ + tar -xvf "bro.tar.gz" && \ + rm -f "bro.tar.gz" && \ + cd "./bro-"$ZEEK_VERSION && \ + ./configure --prefix=$ZEEK_DIR --generator=Ninja && \ + cd build && \ + ninja && \ + ninja install && \ + strip --strip-unneeded \ + $ZEEK_DIR/bin/bro \ + $ZEEK_DIR/bin/bro-cut \ + $ZEEK_DIR/bin/binpac \ + $ZEEK_DIR/lib/libbroker.so.. \ + $ZEEK_DIR/lib/libcaf_core.so.0.16.2 \ + $ZEEK_DIR/lib/libcaf_io.so.0.16.2 \ + $ZEEK_DIR/lib/libcaf_openssl.so.0.16.2 && \ + git clone --depth 1 https://github.com/salesforce/ja3 /tmp/ja3 && \ + mkdir -p $ZEEK_DIR/share/bro/site/ja3 && \ + cp -v /tmp/ja3/bro/* $ZEEK_DIR/share/bro/site/ja3 && \ + rm -rf /tmp/ja3 && \ + cd $MOLOCHDIR/doc/images && \ + find . -name "*.png" -exec bash -c 'convert "{}" -fuzz 2% -transparent white -background white -alpha remove -strip -interlace Plane -quality 85% "{}.jpg" && rename "s/\.png//" "{}.jpg"' \; && \ + cd $MOLOCHDIR/doc && \ + sed -i "s/^# Malcolm$//" README.md && \ + sed -i '/./,$!d' README.md && \ + sed -i "s/.png/.jpg/g" README.md && \ + sed -i "s@docs/images@images@g" README.md && \ + pandoc -s --self-contained --metadata title="Malcolm README" --css $MOLOCHDIR/doc/doc.css -o $MOLOCHDIR/doc/README.html $MOLOCHDIR/doc/README.md && \ + cd /data && \ + tar -xvf "moloch.tar.gz" && \ + rm -f "moloch.tar.gz" && \ + cd "./moloch-"$MOLOCH_VERSION && \ + bash -c 'for i in /data/patches/*; do patch -p1 < $i; done' && \ + cp -v $MOLOCHDIR/doc/images/moloch/moloch_155.png ./viewer/public/moloch_155.png && \ + cp -v $MOLOCHDIR/doc/images/moloch/moloch_77.png ./viewer/public/moloch_77.png && \ + cp -v $MOLOCHDIR/doc/images/moloch/header_logo.png ./parliament/vueapp/src/assets/header_logo.png && \ + cp -v $MOLOCHDIR/doc/images/moloch/header_logo.png ./viewer/public/header_logo.png && \ + cp -v $MOLOCHDIR/doc/images/moloch/header_logo.png ./viewer/vueapp/src/assets/logo.png && \ + find $MOLOCHDIR/doc/images/screenshots -name "*.png" -delete && \ + cp -v $MOLOCHDIR/doc/cyberchef.htm ./viewer/public/cyberchef.htm && \ + rm -f ./viewer/public/cyberchef.htm.gz && \ + gzip ./viewer/public/cyberchef.htm && \ + chmod 664 ./viewer/public/cyberchef.htm.gz $MOLOCHDIR/doc/cyberchef.htm && \ + export PATH="$MOLOCHDIR/bin:${PATH}" && \ + ln -sf $MOLOCHDIR/bin/npm /usr/local/bin/npm && \ + ln -sf $MOLOCHDIR/bin/node /usr/local/bin/node && \ + ln -sf $MOLOCHDIR/bin/npx /usr/local/bin/npx && \ + ./easybutton-build.sh --install && \ + npm cache clean --force && \ apt-get clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + rm -rf $MOLOCHDIR"-"$MOLOCH_VERSION "/data/bro-"$ZEEK_VERSION /var/lib/apt/lists/* /tmp/* /var/tmp/* + +FROM debian:stretch-slim AS runtime + +# Copyright (c) 2019 Battelle Energy Alliance, LLC. All rights reserved. +LABEL maintainer="Seth.Grover@inl.gov" + +ENV DEBIAN_FRONTEND noninteractive -# Declare args -ARG MOLOCH_VERSION=1.8.0 -ARG ZEEK_VERSION=2.6.2 -ARG CYBERCHEF_VERSION=8.30.1 ARG ES_HOST=elasticsearch ARG ES_PORT=9200 ARG MALCOLM_USERNAME=admin @@ -116,152 +155,88 @@ ENV INITIALIZEDB $INITIALIZEDB ENV WIPEDB $WIPEDB ENV MANAGE_PCAP_FILES $MANAGE_PCAP_FILES ENV AUTO_TAG $AUTO_TAG +ENV ZEEK_DIR "/opt/bro" ENV ZEEK_AUTO_ANALYZE_PCAP_FILES $ZEEK_AUTO_ANALYZE_PCAP_FILES ENV ZEEK_AUTO_ANALYZE_PCAP_THREADS $ZEEK_AUTO_ANALYZE_PCAP_THREADS ENV ZEEK_EXTRACTOR_MODE $ZEEK_EXTRACTOR_MODE ENV ZEEK_EXTRACTOR_PATH $ZEEK_EXTRACTOR_PATH -# we're now building moloch and bro source rather than installing the .deb -ADD moloch/patch/* /data/patches/ -ADD README.md /data/moloch/doc/ -ADD doc.css /data/moloch/doc/ -ADD docs/images /data/moloch/doc/images/ -ADD https://github.com/aol/moloch/archive/v$MOLOCH_VERSION.tar.gz /data/moloch.tar.gz -ADD https://github.com/gchq/CyberChef/releases/download/v$CYBERCHEF_VERSION/cyberchef.htm /data/moloch/doc/cyberchef.htm -ADD https://www.zeek.org/downloads/bro-$ZEEK_VERSION.tar.gz /data/bro.tar.gz -RUN apt-get -q update && \ - cd /data/moloch/doc/images && \ - find . -name "*.png" -exec bash -c 'convert "{}" -fuzz 2% -transparent white -background white -alpha remove -strip -interlace Plane -quality 85% "{}.jpg" && rename "s/\.png//" "{}.jpg"' \; && \ - cd /data/moloch/doc && \ - sed -i "s/^# Malcolm$//" README.md && \ - sed -i '/./,$!d' README.md && \ - sed -i "s/.png/.jpg/g" README.md && \ - sed -i "s@docs/images@images@g" README.md && \ - pandoc -s --self-contained --metadata title="Malcolm README" --css /data/moloch/doc/doc.css -o /data/moloch/doc/README.html /data/moloch/doc/README.md && \ - groupadd --gid 1000 $MOLOCHUSER && \ - useradd -M --uid 1000 --gid 1000 --home $MOLOCHDIR $MOLOCHUSER && \ - cd /data && \ - tar -xvf "bro.tar.gz" && \ - rm -f "bro.tar.gz" && \ - cd "./bro-"$ZEEK_VERSION && \ - ./configure --prefix=/usr --generator=Ninja && \ - cd build && \ - ninja && \ - ninja install && \ - strip --strip-unneeded \ - /usr/bin/bro \ - /usr/bin/bro-cut \ - /usr/bin/binpac \ - /usr/lib/libbroker.so.. \ - /usr/lib/libcaf_core.so.0.16.2 \ - /usr/lib/libcaf_io.so.0.16.2 \ - /usr/lib/libcaf_openssl.so.0.16.2 && \ - git clone --depth 1 https://github.com/salesforce/ja3 /tmp/ja3 && \ - mkdir -p /usr/share/bro/site/ja3 && \ - cp -v /tmp/ja3/bro/* /usr/share/bro/site/ja3 && \ - rm -rf /tmp/ja3 && \ - cd /data && \ - tar -xvf "moloch.tar.gz" && \ - rm -f "moloch.tar.gz" && \ - cd "./moloch-"$MOLOCH_VERSION && \ - bash -c 'for i in /data/patches/*; do patch -p1 < $i; done' && \ - cp -v /data/moloch/doc/images/moloch/moloch_155.png ./viewer/public/moloch_155.png && \ - cp -v /data/moloch/doc/images/moloch/moloch_77.png ./viewer/public/moloch_77.png && \ - cp -v /data/moloch/doc/images/moloch/header_logo.png ./parliament/vueapp/src/assets/header_logo.png && \ - cp -v /data/moloch/doc/images/moloch/header_logo.png ./viewer/public/header_logo.png && \ - cp -v /data/moloch/doc/images/moloch/header_logo.png ./viewer/vueapp/src/assets/logo.png && \ - find /data/moloch/doc/images/screenshots -name "*.png" -delete && \ - cp -v /data/moloch/doc/cyberchef.htm ./viewer/public/cyberchef.htm && \ - rm -f ./viewer/public/cyberchef.htm.gz && \ - gzip ./viewer/public/cyberchef.htm && \ - chmod 664 ./viewer/public/cyberchef.htm.gz /data/moloch/doc/cyberchef.htm && \ - export PATH="/data/moloch/bin:${PATH}" && \ - ln -sf /data/moloch/bin/npm /usr/local/bin/npm && \ - ln -sf /data/moloch/bin/node /usr/local/bin/node && \ - ln -sf /data/moloch/bin/npx /usr/local/bin/npx && \ - ./easybutton-build.sh --install && \ - npm cache clean --force && \ - apt-get -q -y remove --purge \ - autoconf \ - automake \ - autopoint \ - autotools-dev \ - bison \ - bsdmainutils \ - bzip2-doc \ - cmake \ - debhelper \ - dh-autoreconf \ - dh-strip-nondeterminism \ - dwz \ - flex \ - g++ \ - gcc \ - git \ - imagemagick \ - intltool-debian \ - libbison-dev \ - libbz2-dev \ - libffi-dev \ - libfl-dev \ - libgeoip-dev \ - libgoogle-perftools-dev \ - libkrb5-dev \ - libltdl-dev \ - libmagic-dev \ - libmaxminddb-dev \ - libncurses-dev \ - libpcap0.8-dev \ - libpcre3-dev \ - libpng-dev \ - libreadline-dev \ - libssl1.0-dev \ - m4 \ - make \ - man-db \ - ninja-build \ - pandoc \ - pkg-config \ - po-debconf \ - python-dev \ - rename \ - uuid-dev \ - zlib1g-dev && \ +COPY --from=build $MOLOCHDIR $MOLOCHDIR +COPY --from=build $ZEEK_DIR $ZEEK_DIR + +RUN sed -i "s/stretch main/stretch main contrib non-free/" /etc/apt/sources.list && \ + apt-get -q update && \ + apt-get install -q -y --no-install-recommends \ + cron \ + curl \ + file \ + geoip-bin \ + gettext \ + inotify-tools \ + libcap2-bin \ + libgoogle-perftools4 \ + libjson-perl \ + libkrb5-3 \ + libmaxminddb0 \ + libpcap0.8 \ + libssl1.0 \ + libtool \ + libwww-perl \ + libyaml-0-2 \ + psmisc \ + python \ + python3 \ + python3-pip \ + python3-setuptools \ + python3-wheel \ + rename \ + sudo \ + supervisor \ + vim-tiny \ + wget \ + tar gzip unzip cpio bzip2 lzma xz-utils p7zip-full unrar zlib1g && \ + pip3 install --no-cache-dir elasticsearch manuf geoip2 patool entrypoint2 pyunpack && \ + ln -sf $MOLOCHDIR/bin/npm /usr/local/bin/npm && \ + ln -sf $MOLOCHDIR/bin/node /usr/local/bin/node && \ + ln -sf $MOLOCHDIR/bin/npx /usr/local/bin/npx && \ apt-get -q -y autoremove && \ apt-get clean && \ - rm -rf "/data/moloch-"$MOLOCH_VERSION "/data/bro-"$ZEEK_VERSION /var/lib/apt/lists/* /tmp/* /var/tmp/* + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # add configuration and scripts ADD moloch/scripts /data/ ADD shared/bin/elastic_search_status.sh /data/ ADD shared/bin/cron_env_deb.sh /data/ -ADD moloch/etc /data/moloch/etc/ -ADD https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv /data/moloch/etc/ipv4-address-space.csv -ADD https://raw.githubusercontent.com/wireshark/wireshark/master/manuf /data/moloch/etc/oui.txt +ADD moloch/etc $MOLOCHDIR/etc/ +ADD https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv $MOLOCHDIR/etc/ipv4-address-space.csv +ADD https://raw.githubusercontent.com/wireshark/wireshark/master/manuf $MOLOCHDIR/etc/oui.txt ADD https://updates.maxmind.com/app/update_secure?edition_id=GeoLite2-Country /tmp/GeoLite2-Country.mmdb.gz ADD https://updates.maxmind.com/app/update_secure?edition_id=GeoLite2-ASN /tmp/GeoLite2-ASN.mmdb.gz -ADD moloch/wise/source.*.js /data/moloch/wiseService/ +ADD moloch/wise/source.*.js $MOLOCHDIR/wiseService/ ADD moloch/supervisord.conf /etc/supervisord.conf -ADD moloch/zeek/*.bro /usr/share/bro/site/ -RUN chmod 755 /data/*.sh && \ - cp -f /data/moloch_update_geo.sh /data/moloch/bin/moloch_update_geo.sh && \ - bash -c 'zcat /tmp/GeoLite2-Country.mmdb.gz > /data/moloch/etc/GeoLite2-Country.mmdb' && \ +ADD moloch/zeek/*.bro $ZEEK_DIR/share/bro/site/ + +RUN groupadd --gid 1000 $MOLOCHUSER && \ + useradd -M --uid 1000 --gid 1000 --home $MOLOCHDIR $MOLOCHUSER && \ + chmod 755 /data/*.sh && \ + cp -f /data/moloch_update_geo.sh $MOLOCHDIR/bin/moloch_update_geo.sh && \ + bash -c "zcat /tmp/GeoLite2-Country.mmdb.gz > $MOLOCHDIR/etc/GeoLite2-Country.mmdb" && \ rm -f /tmp/GeoLite2-Country.mmdb.gz && \ - bash -c 'zcat /tmp/GeoLite2-ASN.mmdb.gz > /data/moloch/etc/GeoLite2-ASN.mmdb' && \ + bash -c "zcat /tmp/GeoLite2-ASN.mmdb.gz > $MOLOCHDIR/etc/GeoLite2-ASN.mmdb" && \ rm -f /tmp/GeoLite2-ASN.mmdb.gz && \ - sed -i "s/^\(MOLOCH_LOCALELASTICSEARCH=\).*/\1"$MOLOCH_LOCALELASTICSEARCH"/" /data/moloch/bin/Configure && \ - sed -i "s/^\(MOLOCH_INET=\).*/\1"$MOLOCH_INET"/" /data/moloch/bin/Configure && \ + sed -i "s/^\(MOLOCH_LOCALELASTICSEARCH=\).*/\1"$MOLOCH_LOCALELASTICSEARCH"/" $MOLOCHDIR/bin/Configure && \ + sed -i "s/^\(MOLOCH_INET=\).*/\1"$MOLOCH_INET"/" $MOLOCHDIR/bin/Configure && \ chown -R 1000:1000 $MOLOCHDIR/logs && \ chmod u+s $MOLOCHDIR/bin/moloch-capture && \ - bash -c 'echo -e "* * * * * su -c /data/moloch-parse-pcap-folder.sh $MOLOCHUSER >/dev/null 2>&1\n* * * * * su -c /data/moloch-parse-autozeek-folder.sh $MOLOCHUSER >/dev/null 2>&1" | crontab -' + bash -c 'echo -e "* * * * * su -c /data/moloch-parse-pcap-folder.sh $MOLOCHUSER >/dev/null 2>&1\n* * * * * su -c $MOLOCHDIR-parse-autozeek-folder.sh $MOLOCHUSER >/dev/null 2>&1" | crontab -' #Update Path -ENV PATH="/data:/data/moloch/bin:${PATH}" +ENV PATH="/data:$MOLOCHDIR/bin:$ZEEK_DIR/bin:${PATH}" VOLUME ["/data/configured"] EXPOSE 8000 8005 8081 -WORKDIR /data/moloch +WORKDIR $MOLOCHDIR # ENTRYPOINT ["/data/startmoloch.sh"] CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf", "-u", "root", "-n"] diff --git a/Dockerfiles/pcap-capture.Dockerfile b/Dockerfiles/pcap-capture.Dockerfile index 2d9fd09e5..77a4381d2 100644 --- a/Dockerfiles/pcap-capture.Dockerfile +++ b/Dockerfiles/pcap-capture.Dockerfile @@ -1,4 +1,4 @@ -FROM debian:buster-slim AS build +FROM debian:buster-slim # Copyright (c) 2019 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="Seth.Grover@inl.gov" diff --git a/moloch/scripts/zeek-process-pcap.py b/moloch/scripts/zeek-process-pcap.py index fcbda661d..49311a5f7 100755 --- a/moloch/scripts/zeek-process-pcap.py +++ b/moloch/scripts/zeek-process-pcap.py @@ -22,6 +22,7 @@ ZEEK_STATE_DIR = '.state' ZEEK_UPLOAD_DIR_DEFAULT = '/data/zeek/upload' ZEEK_UPLOAD_DIR_ENV_VAR = 'ZEEK_UPLOAD_DIR' +ZEEK_INSTALL_DIR_ENV_VAR = 'ZEEK_DIR' ZEEK_AUTOZEEK_TAG = 'AUTOZEEK' ZEEK_AUTOCARVE_TAG_PREFIX = 'AUTOCARVE' @@ -61,7 +62,7 @@ def main(): os.chdir(tmpLogDir) # use Zeek to process the pcap - broCmd = ["bro", "-r", pcapFile, ZEEK_LOCAL_SCRIPT] + broCmd = [os.path.join(os.getenv(ZEEK_INSTALL_DIR_ENV_VAR, "/opt/bro"), "bin/bro"), "-r", pcapFile, ZEEK_LOCAL_SCRIPT] # set file extraction parameters if required if (extractFileMode != ZEEK_EXTRACTOR_MODE_NONE):