Skip to content

Commit

Permalink
Merge remote-tracking branch 'mmguero-dev/development' into v5210_mer…
Browse files Browse the repository at this point in the history 
…ge_cisagov
  • Loading branch information
@mmguero
mmguero committed Apr 4, 2022
2 parents a2fa871 + f1c378f commit 07826c4
Show file tree
Hide file tree
Showing 25 changed files with 155 additions and 555 deletions.
2 changes: 1 addition & 1 deletion Dockerfiles/arkime.Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ FROM debian:11-slim AS build

ENV DEBIAN_FRONTEND noninteractive

ENV ARKIME_VERSION "3.4.1"
ENV ARKIME_VERSION "3.4.2"
ENV ARKIMEDIR "/opt/arkime"
ENV ARKIME_URL "https://github.com/arkime/arkime.git"
ENV ARKIME_LOCALELASTICSEARCH no
Expand Down
6 changes: 3 additions & 3 deletions Dockerfiles/dashboards.Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ ENV PGROUP "dashboarder"

ENV TERM xterm

ARG OPENSEARCH_VERSION="1.2.4"
ARG OPENSEARCH_VERSION="1.3.0"
ENV OPENSEARCH_VERSION $OPENSEARCH_VERSION

ARG OPENSEARCH_DASHBOARDS_VERSION="1.2.0"
ARG OPENSEARCH_DASHBOARDS_VERSION="1.3.0"
ENV OPENSEARCH_DASHBOARDS_VERSION $OPENSEARCH_DASHBOARDS_VERSION

# base system dependencies for checking out and building plugins
Expand Down Expand Up @@ -68,7 +68,7 @@ RUN eval "$(nodenv init -)" && \

# runtime ##################################################################

FROM opensearchproject/opensearch-dashboards:1.2.0
FROM opensearchproject/opensearch-dashboards:1.3.0

LABEL maintainer="[email protected]"
LABEL org.opencontainers.image.authors='[email protected]'
Expand Down
8 changes: 0 additions & 8 deletions Dockerfiles/file-monitor.Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,6 @@ ARG EXTRACTED_FILE_MIN_BYTES=64
ARG EXTRACTED_FILE_MAX_BYTES=134217728
ARG VTOT_API2_KEY=0
ARG VTOT_REQUESTS_PER_MINUTE=4
ARG MALASS_HOST=0
ARG MALASS_PORT=80
ARG MALASS_MAX_REQUESTS=20
ARG EXTRACTED_FILE_ENABLE_CLAMAV=false
ARG EXTRACTED_FILE_UPDATE_RULES=false
ARG EXTRACTED_FILE_PIPELINE_DEBUG=false
Expand Down Expand Up @@ -64,9 +61,6 @@ ENV EXTRACTED_FILE_MIN_BYTES $EXTRACTED_FILE_MIN_BYTES
ENV EXTRACTED_FILE_MAX_BYTES $EXTRACTED_FILE_MAX_BYTES
ENV VTOT_API2_KEY $VTOT_API2_KEY
ENV VTOT_REQUESTS_PER_MINUTE $VTOT_REQUESTS_PER_MINUTE
ENV MALASS_HOST $MALASS_HOST
ENV MALASS_PORT $MALASS_PORT
ENV MALASS_MAX_REQUESTS $MALASS_MAX_REQUESTS
ENV EXTRACTED_FILE_ENABLE_CLAMAV $EXTRACTED_FILE_ENABLE_CLAMAV
ENV EXTRACTED_FILE_UPDATE_RULES $EXTRACTED_FILE_UPDATE_RULES
ENV EXTRACTED_FILE_PIPELINE_DEBUG $EXTRACTED_FILE_PIPELINE_DEBUG
Expand Down Expand Up @@ -204,12 +198,10 @@ RUN sed -i "s/bullseye main/bullseye main contrib non-free/g" /etc/apt/sources.l
ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/clam_scan.py && \
ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/yara_scan.py && \
ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/capa_scan.py && \
ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/malass_scan.py && \
echo "0 */6 * * * /bin/bash /usr/local/bin/capa-update.sh\n0 */6 * * * /bin/bash /usr/local/bin/yara-rules-update.sh" > ${SUPERCRONIC_CRONTAB}

ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
ADD shared/bin/zeek_carve*.py /usr/local/bin/
ADD shared/bin/malass_client.py /usr/local/bin/
ADD file-monitor/supervisord.conf /etc/supervisord.conf
ADD file-monitor/docker-entrypoint.sh /docker-entrypoint.sh
ADD file-monitor/*update.sh /usr/local/bin/
Expand Down
2 changes: 1 addition & 1 deletion Dockerfiles/opensearch.Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM opensearchproject/opensearch:1.2.4
FROM opensearchproject/opensearch:1.3.0

# Copyright (c) 2022 Battelle Energy Alliance, LLC. All rights reserved.
LABEL maintainer="[email protected]"
Expand Down
2 changes: 1 addition & 1 deletion Dockerfiles/pcap-monitor.Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ RUN apt-get -q update && \
vim-tiny && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* && \
pip3 install --no-cache-dir opensearch-py opensearch-dsl pyzmq pyinotify python-magic && \
pip3 install --no-cache-dir opensearch-py opensearch-dsl pyzmq pyinotify python-magic requests && \
groupadd --gid ${DEFAULT_GID} ${PGROUP} && \
useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER}

Expand Down
66 changes: 33 additions & 33 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,22 +181,22 @@ You can then observe that the images have been retrieved by running `docker imag
```
$ docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
malcolmnetsec/api 5.2.9 xxxxxxxxxxxx 3 days ago 158MB
malcolmnetsec/arkime 5.2.9 xxxxxxxxxxxx 3 days ago 816MB
malcolmnetsec/dashboards 5.2.9 xxxxxxxxxxxx 3 days ago 1.02GB
malcolmnetsec/dashboards-helper 5.2.9 xxxxxxxxxxxx 3 days ago 184MB
malcolmnetsec/filebeat-oss 5.2.9 xxxxxxxxxxxx 3 days ago 624MB
malcolmnetsec/file-monitor 5.2.9 xxxxxxxxxxxx 3 days ago 588MB
malcolmnetsec/file-upload 5.2.9 xxxxxxxxxxxx 3 days ago 259MB
malcolmnetsec/freq 5.2.9 xxxxxxxxxxxx 3 days ago 132MB
malcolmnetsec/htadmin 5.2.9 xxxxxxxxxxxx 3 days ago 242MB
malcolmnetsec/logstash-oss 5.2.9 xxxxxxxxxxxx 3 days ago 1.35GB
malcolmnetsec/name-map-ui 5.2.9 xxxxxxxxxxxx 3 days ago 143MB
malcolmnetsec/nginx-proxy 5.2.9 xxxxxxxxxxxx 3 days ago 121MB
malcolmnetsec/opensearch 5.2.9 xxxxxxxxxxxx 3 days ago 1.17GB
malcolmnetsec/pcap-capture 5.2.9 xxxxxxxxxxxx 3 days ago 121MB
malcolmnetsec/pcap-monitor 5.2.9 xxxxxxxxxxxx 3 days ago 213MB
malcolmnetsec/zeek 5.2.9 xxxxxxxxxxxx 3 days ago 1GB
malcolmnetsec/api 5.2.10 xxxxxxxxxxxx 3 days ago 158MB
malcolmnetsec/arkime 5.2.10 xxxxxxxxxxxx 3 days ago 816MB
malcolmnetsec/dashboards 5.2.10 xxxxxxxxxxxx 3 days ago 1.02GB
malcolmnetsec/dashboards-helper 5.2.10 xxxxxxxxxxxx 3 days ago 184MB
malcolmnetsec/filebeat-oss 5.2.10 xxxxxxxxxxxx 3 days ago 624MB
malcolmnetsec/file-monitor 5.2.10 xxxxxxxxxxxx 3 days ago 588MB
malcolmnetsec/file-upload 5.2.10 xxxxxxxxxxxx 3 days ago 259MB
malcolmnetsec/freq 5.2.10 xxxxxxxxxxxx 3 days ago 132MB
malcolmnetsec/htadmin 5.2.10 xxxxxxxxxxxx 3 days ago 242MB
malcolmnetsec/logstash-oss 5.2.10 xxxxxxxxxxxx 3 days ago 1.35GB
malcolmnetsec/name-map-ui 5.2.10 xxxxxxxxxxxx 3 days ago 143MB
malcolmnetsec/nginx-proxy 5.2.10 xxxxxxxxxxxx 3 days ago 121MB
malcolmnetsec/opensearch 5.2.10 xxxxxxxxxxxx 3 days ago 1.17GB
malcolmnetsec/pcap-capture 5.2.10 xxxxxxxxxxxx 3 days ago 121MB
malcolmnetsec/pcap-monitor 5.2.10 xxxxxxxxxxxx 3 days ago 213MB
malcolmnetsec/zeek 5.2.10 xxxxxxxxxxxx 3 days ago 1GB
```

#### Import from pre-packaged tarballs
Expand Down Expand Up @@ -3446,7 +3446,7 @@ Building the ISO may take 30 minutes or more depending on your system. As the bu

```
…
Finished, created "/malcolm-build/malcolm-iso/malcolm-5.2.9.iso"
Finished, created "/malcolm-build/malcolm-iso/malcolm-5.2.10.iso"
…
```

Expand Down Expand Up @@ -3835,22 +3835,22 @@ Pulling zeek ... done

user@host:~/Malcolm$ docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
malcolmnetsec/api 5.2.9 xxxxxxxxxxxx 3 days ago 158MB
malcolmnetsec/arkime 5.2.9 xxxxxxxxxxxx 3 days ago 816MB
malcolmnetsec/dashboards 5.2.9 xxxxxxxxxxxx 3 days ago 1.02GB
malcolmnetsec/dashboards-helper 5.2.9 xxxxxxxxxxxx 3 days ago 184MB
malcolmnetsec/filebeat-oss 5.2.9 xxxxxxxxxxxx 3 days ago 624MB
malcolmnetsec/file-monitor 5.2.9 xxxxxxxxxxxx 3 days ago 588MB
malcolmnetsec/file-upload 5.2.9 xxxxxxxxxxxx 3 days ago 259MB
malcolmnetsec/freq 5.2.9 xxxxxxxxxxxx 3 days ago 132MB
malcolmnetsec/htadmin 5.2.9 xxxxxxxxxxxx 3 days ago 242MB
malcolmnetsec/logstash-oss 5.2.9 xxxxxxxxxxxx 3 days ago 1.35GB
malcolmnetsec/name-map-ui 5.2.9 xxxxxxxxxxxx 3 days ago 143MB
malcolmnetsec/nginx-proxy 5.2.9 xxxxxxxxxxxx 3 days ago 121MB
malcolmnetsec/opensearch 5.2.9 xxxxxxxxxxxx 3 days ago 1.17GB
malcolmnetsec/pcap-capture 5.2.9 xxxxxxxxxxxx 3 days ago 121MB
malcolmnetsec/pcap-monitor 5.2.9 xxxxxxxxxxxx 3 days ago 213MB
malcolmnetsec/zeek 5.2.9 xxxxxxxxxxxx 3 days ago 1GB
malcolmnetsec/api 5.2.10 xxxxxxxxxxxx 3 days ago 158MB
malcolmnetsec/arkime 5.2.10 xxxxxxxxxxxx 3 days ago 816MB
malcolmnetsec/dashboards 5.2.10 xxxxxxxxxxxx 3 days ago 1.02GB
malcolmnetsec/dashboards-helper 5.2.10 xxxxxxxxxxxx 3 days ago 184MB
malcolmnetsec/filebeat-oss 5.2.10 xxxxxxxxxxxx 3 days ago 624MB
malcolmnetsec/file-monitor 5.2.10 xxxxxxxxxxxx 3 days ago 588MB
malcolmnetsec/file-upload 5.2.10 xxxxxxxxxxxx 3 days ago 259MB
malcolmnetsec/freq 5.2.10 xxxxxxxxxxxx 3 days ago 132MB
malcolmnetsec/htadmin 5.2.10 xxxxxxxxxxxx 3 days ago 242MB
malcolmnetsec/logstash-oss 5.2.10 xxxxxxxxxxxx 3 days ago 1.35GB
malcolmnetsec/name-map-ui 5.2.10 xxxxxxxxxxxx 3 days ago 143MB
malcolmnetsec/nginx-proxy 5.2.10 xxxxxxxxxxxx 3 days ago 121MB
malcolmnetsec/opensearch 5.2.10 xxxxxxxxxxxx 3 days ago 1.17GB
malcolmnetsec/pcap-capture 5.2.10 xxxxxxxxxxxx 3 days ago 121MB
malcolmnetsec/pcap-monitor 5.2.10 xxxxxxxxxxxx 3 days ago 213MB
malcolmnetsec/zeek 5.2.10 xxxxxxxxxxxx 3 days ago 1GB
```

Finally, we can start Malcolm. When Malcolm starts it will stream informational and debug messages to the console. If you wish, you can safely close the console or use `Ctrl+C` to stop these messages; Malcolm will continue running in the background.
Expand Down
5 changes: 4 additions & 1 deletion dashboards/malcolm_template.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@
"session": {
"properties": {
"malcolmDocId": { "type": "keyword" },
"firstPacket": { "type": "date" },
"lastPacket": { "type": "date" },
"timestamp": { "type": "date" },
"client.domain": { "type": "keyword" },
"destination.domain": { "type": "keyword" },
"destination.geo.city_name": { "type": "keyword", "ignore_above": 128, "fields": { "text": { "type": "text" } } },
Expand Down Expand Up @@ -262,7 +265,7 @@
"zeek.cip_identity.vendor_name": { "type": "keyword" },
"zeek.cip_io.connection_id": { "type": "keyword" },
"zeek.cip_io.data_length": { "type": "integer" },
"zeek.cip_io.io_data": { "type": "keyword", "ignore_above": 128, "fields": { "text": { "type": "text", "norms": false } } },
"zeek.cip_io.io_data": { "type": "keyword", "ignore_above": 8, "fields": { "text": { "type": "text", "norms": false } } },
"zeek.cip_io.sequence_number": { "type": "integer" },
"zeek.conn.conn_state": { "type": "keyword" },
"zeek.conn.conn_state_description": { "type": "keyword" },
Expand Down
32 changes: 16 additions & 16 deletions docker-compose-standalone.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ x-pcap-capture-variables: &pcap-capture-variables

services:
opensearch:
image: malcolmnetsec/opensearch:5.2.9
image: malcolmnetsec/opensearch:5.2.10
restart: "no"
stdin_open: false
tty: true
Expand Down Expand Up @@ -178,7 +178,7 @@ services:
retries: 3
start_period: 180s
dashboards-helper:
image: malcolmnetsec/dashboards-helper:5.2.9
image: malcolmnetsec/dashboards-helper:5.2.10
restart: "no"
stdin_open: false
tty: true
Expand Down Expand Up @@ -206,7 +206,7 @@ services:
retries: 3
start_period: 30s
dashboards:
image: malcolmnetsec/dashboards:5.2.9
image: malcolmnetsec/dashboards:5.2.10
restart: "no"
stdin_open: false
tty: true
Expand All @@ -227,7 +227,7 @@ services:
retries: 3
start_period: 210s
logstash:
image: malcolmnetsec/logstash-oss:5.2.9
image: malcolmnetsec/logstash-oss:5.2.10
restart: "no"
stdin_open: false
tty: true
Expand Down Expand Up @@ -268,7 +268,7 @@ services:
retries: 3
start_period: 600s
filebeat:
image: malcolmnetsec/filebeat-oss:5.2.9
image: malcolmnetsec/filebeat-oss:5.2.10
restart: "no"
stdin_open: false
tty: true
Expand Down Expand Up @@ -305,7 +305,7 @@ services:
retries: 3
start_period: 60s
arkime:
image: malcolmnetsec/arkime:5.2.9
image: malcolmnetsec/arkime:5.2.10
restart: "no"
stdin_open: false
tty: true
Expand Down Expand Up @@ -343,7 +343,7 @@ services:
retries: 3
start_period: 210s
zeek:
image: malcolmnetsec/zeek:5.2.9
image: malcolmnetsec/zeek:5.2.10
restart: "no"
stdin_open: false
tty: true
Expand Down Expand Up @@ -372,7 +372,7 @@ services:
retries: 3
start_period: 60s
file-monitor:
image: malcolmnetsec/file-monitor:5.2.9
image: malcolmnetsec/file-monitor:5.2.10
restart: "no"
stdin_open: false
tty: true
Expand All @@ -395,7 +395,7 @@ services:
retries: 3
start_period: 60s
pcap-capture:
image: malcolmnetsec/pcap-capture:5.2.9
image: malcolmnetsec/pcap-capture:5.2.10
restart: "no"
stdin_open: false
tty: true
Expand All @@ -415,7 +415,7 @@ services:
volumes:
- ./pcap/upload:/pcap
pcap-monitor:
image: malcolmnetsec/pcap-monitor:5.2.9
image: malcolmnetsec/pcap-monitor:5.2.10
restart: "no"
stdin_open: false
tty: true
Expand All @@ -438,7 +438,7 @@ services:
retries: 3
start_period: 90s
upload:
image: malcolmnetsec/file-upload:5.2.9
image: malcolmnetsec/file-upload:5.2.10
restart: "no"
stdin_open: false
tty: true
Expand All @@ -464,7 +464,7 @@ services:
retries: 3
start_period: 60s
htadmin:
image: malcolmnetsec/htadmin:5.2.9
image: malcolmnetsec/htadmin:5.2.10
restart: "no"
stdin_open: false
tty: true
Expand All @@ -486,7 +486,7 @@ services:
retries: 3
start_period: 60s
freq:
image: malcolmnetsec/freq:5.2.9
image: malcolmnetsec/freq:5.2.10
restart: "no"
stdin_open: false
tty: true
Expand All @@ -504,7 +504,7 @@ services:
retries: 3
start_period: 60s
name-map-ui:
image: malcolmnetsec/name-map-ui:5.2.9
image: malcolmnetsec/name-map-ui:5.2.10
restart: "no"
stdin_open: false
tty: true
Expand All @@ -525,7 +525,7 @@ services:
retries: 3
start_period: 60s
api:
image: malcolmnetsec/api:5.2.9
image: malcolmnetsec/api:5.2.10
command: gunicorn --bind 0:5000 manage:app
restart: "no"
stdin_open: false
Expand All @@ -543,7 +543,7 @@ services:
retries: 3
start_period: 60s
nginx-proxy:
image: malcolmnetsec/nginx-proxy:5.2.9
image: malcolmnetsec/nginx-proxy:5.2.10
restart: "no"
stdin_open: false
tty: true
Expand Down
Loading

0 comments on commit 07826c4

Please sign in to comment.