From b3ae65d132bbe15248a3627005a39886ff599b72 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 27 Aug 2024 08:19:24 -0600 Subject: [PATCH 1/3] fix python watchdog package at 4.0.2 for now --- Dockerfiles/arkime.Dockerfile | 2 +- Dockerfiles/file-monitor.Dockerfile | 2 +- Dockerfiles/filebeat.Dockerfile | 2 +- Dockerfiles/pcap-monitor.Dockerfile | 2 +- Dockerfiles/suricata.Dockerfile | 2 +- hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index d2a6413d2..a6e573a78 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -148,7 +148,7 @@ RUN export DEBARCH=$(dpkg --print-architecture) && \ mkdir -p "${ARKIME_DIR}"/plugins && \ curl -fsSL -o "${ARKIME_DIR}/plugins/ja4plus.${DEBARCH}.so" "$(echo "${ARKIME_JA4_SO_URL}" | sed "s/XXX/${DEBARCH}/g")" && \ chmod 755 "${ARKIME_DIR}/plugins/ja4plus.${DEBARCH}.so" && \ - python3 -m pip install --break-system-packages --no-compile --no-cache-dir beautifulsoup4 pyzmq watchdog && \ + python3 -m pip install --break-system-packages --no-compile --no-cache-dir beautifulsoup4 pyzmq watchdog==4.0.2 && \ ln -sfr $ARKIME_DIR/bin/npm /usr/local/bin/npm && \ ln -sfr $ARKIME_DIR/bin/node /usr/local/bin/node && \ ln -sfr $ARKIME_DIR/bin/npx /usr/local/bin/npx && \ diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index fd56e1291..caa686704 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -159,7 +159,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') python-magic \ stream-zip \ supervisor \ - watchdog \ + watchdog==4.0.2 \ yara-python && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index 2df341307..25272e5a3 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -101,7 +101,7 @@ RUN export EVTXARCH=$(uname -m | sed 's/arm64/aarch64/') && \ unar \ unzip \ xz-utils && \ - python3 -m pip install --no-compile --no-cache-dir patool entrypoint2 pyunpack python-magic ordered-set supervisor watchdog && \ + python3 -m pip install --no-compile --no-cache-dir patool entrypoint2 pyunpack python-magic ordered-set supervisor watchdog==4.0.2 && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ curl -fsSL -o /usr/local/bin/yq "${YQ_URL}${BINARCH}" && \ diff --git a/Dockerfiles/pcap-monitor.Dockerfile b/Dockerfiles/pcap-monitor.Dockerfile index 9efa4aabb..b710fcb23 100644 --- a/Dockerfiles/pcap-monitor.Dockerfile +++ b/Dockerfiles/pcap-monitor.Dockerfile @@ -68,7 +68,7 @@ RUN apt-get -q update && \ python-magic \ pyzmq \ requests \ - watchdog && \ + watchdog==4.0.2 && \ groupadd --gid ${DEFAULT_GID} ${PGROUP} && \ useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile index 50ae17df7..d2872cac8 100644 --- a/Dockerfiles/suricata.Dockerfile +++ b/Dockerfiles/suricata.Dockerfile @@ -108,7 +108,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') apt-get install -q -y --no-install-recommends -t bookworm-backports \ suricata=${SURICATA_VERSION_PATTERN} \ suricata-update && \ - python3 -m pip install --break-system-packages --no-compile --no-cache-dir watchdog && \ + python3 -m pip install --break-system-packages --no-compile --no-cache-dir watchdog==4.0.2 && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ curl -fsSL -o /usr/bin/yq "${YQ_URL}${BINARCH}" && \ diff --git a/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot index 981f3ffcb..0667b4b72 100755 --- a/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot +++ b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot @@ -20,4 +20,4 @@ pip3 install --break-system-packages --no-compile --no-cache-dir --force-reinsta stix2 \ stream-zip \ taxii2-client \ - watchdog + watchdog==4.0.2 From f72d1bd77b9909d7df30bed343f3b26bc5b6aa62 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 27 Aug 2024 08:38:03 -0600 Subject: [PATCH 2/3] Reduce verbosity of pruning script --- shared/bin/prune_files.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/bin/prune_files.sh b/shared/bin/prune_files.sh index 29ad8d474..010327ae3 100755 --- a/shared/bin/prune_files.sh +++ b/shared/bin/prune_files.sh @@ -101,7 +101,7 @@ while true ; do if (( $DELETED_FILES > 0 )) ; then echo "Pruned $DELETED_FILES files ($(numfmt --to=iec-i --suffix=B $DELETED_BYTES)) in \"$PRUNE_PATH\"" else - echo "Nothing was pruned in \"$PRUNE_PATH\"!" >&2 + [[ "$VERBOSE" == "1" ]] && echo "Nothing was pruned in \"$PRUNE_PATH\"!" >&2 fi else From 46adb9cc64b95b3e7af176ad44491ccaa707b96d Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 27 Aug 2024 08:49:46 -0600 Subject: [PATCH 3/3] Update Watchdog package to v5.0.0 (idaholab/Malcolm#550) --- Dockerfiles/arkime.Dockerfile | 2 +- Dockerfiles/file-monitor.Dockerfile | 2 +- Dockerfiles/filebeat.Dockerfile | 2 +- Dockerfiles/pcap-monitor.Dockerfile | 2 +- Dockerfiles/suricata.Dockerfile | 2 +- .../config/hooks/normal/0169-pip-installs.hook.chroot | 2 +- shared/bin/pcap_watcher.py | 6 +++--- shared/bin/watch_common.py | 6 +++--- shared/bin/zeek_carve_watcher.py | 6 +++--- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index a6e573a78..4b246041f 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -148,7 +148,7 @@ RUN export DEBARCH=$(dpkg --print-architecture) && \ mkdir -p "${ARKIME_DIR}"/plugins && \ curl -fsSL -o "${ARKIME_DIR}/plugins/ja4plus.${DEBARCH}.so" "$(echo "${ARKIME_JA4_SO_URL}" | sed "s/XXX/${DEBARCH}/g")" && \ chmod 755 "${ARKIME_DIR}/plugins/ja4plus.${DEBARCH}.so" && \ - python3 -m pip install --break-system-packages --no-compile --no-cache-dir beautifulsoup4 pyzmq watchdog==4.0.2 && \ + python3 -m pip install --break-system-packages --no-compile --no-cache-dir beautifulsoup4 pyzmq watchdog==5.0.0 && \ ln -sfr $ARKIME_DIR/bin/npm /usr/local/bin/npm && \ ln -sfr $ARKIME_DIR/bin/node /usr/local/bin/node && \ ln -sfr $ARKIME_DIR/bin/npx /usr/local/bin/npx && \ diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index caa686704..bfbee2de5 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -159,7 +159,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') python-magic \ stream-zip \ supervisor \ - watchdog==4.0.2 \ + watchdog==5.0.0 \ yara-python && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index 25272e5a3..021077425 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -101,7 +101,7 @@ RUN export EVTXARCH=$(uname -m | sed 's/arm64/aarch64/') && \ unar \ unzip \ xz-utils && \ - python3 -m pip install --no-compile --no-cache-dir patool entrypoint2 pyunpack python-magic ordered-set supervisor watchdog==4.0.2 && \ + python3 -m pip install --no-compile --no-cache-dir patool entrypoint2 pyunpack python-magic ordered-set supervisor watchdog==5.0.0 && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ curl -fsSL -o /usr/local/bin/yq "${YQ_URL}${BINARCH}" && \ diff --git a/Dockerfiles/pcap-monitor.Dockerfile b/Dockerfiles/pcap-monitor.Dockerfile index b710fcb23..c5187fcab 100644 --- a/Dockerfiles/pcap-monitor.Dockerfile +++ b/Dockerfiles/pcap-monitor.Dockerfile @@ -68,7 +68,7 @@ RUN apt-get -q update && \ python-magic \ pyzmq \ requests \ - watchdog==4.0.2 && \ + watchdog==5.0.0 && \ groupadd --gid ${DEFAULT_GID} ${PGROUP} && \ useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile index d2872cac8..9ccc973a2 100644 --- a/Dockerfiles/suricata.Dockerfile +++ b/Dockerfiles/suricata.Dockerfile @@ -108,7 +108,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') apt-get install -q -y --no-install-recommends -t bookworm-backports \ suricata=${SURICATA_VERSION_PATTERN} \ suricata-update && \ - python3 -m pip install --break-system-packages --no-compile --no-cache-dir watchdog==4.0.2 && \ + python3 -m pip install --break-system-packages --no-compile --no-cache-dir watchdog==5.0.0 && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ curl -fsSL -o /usr/bin/yq "${YQ_URL}${BINARCH}" && \ diff --git a/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot index 0667b4b72..16aba39e8 100755 --- a/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot +++ b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot @@ -20,4 +20,4 @@ pip3 install --break-system-packages --no-compile --no-cache-dir --force-reinsta stix2 \ stream-zip \ taxii2-client \ - watchdog==4.0.2 + watchdog==5.0.0 diff --git a/shared/bin/pcap_watcher.py b/shared/bin/pcap_watcher.py index 25816e13d..428723f95 100755 --- a/shared/bin/pcap_watcher.py +++ b/shared/bin/pcap_watcher.py @@ -46,7 +46,7 @@ from watchdog.observers import Observer from watchdog.observers.polling import PollingObserver -from watchdog.utils import WatchdogShutdown +from watchdog.utils import WatchdogShutdownError ################################################################################################### MINIMUM_CHECKED_FILE_SIZE_DEFAULT = 24 @@ -564,9 +564,9 @@ def main(): logging.info(f"{scriptName}:\tshutting down...") if shuttingDown[0]: - raise WatchdogShutdown() + raise WatchdogShutdownError() - except WatchdogShutdown: + except WatchdogShutdownError: observer.unschedule_all() finally: diff --git a/shared/bin/watch_common.py b/shared/bin/watch_common.py index 9457a1604..e448ee8f2 100644 --- a/shared/bin/watch_common.py +++ b/shared/bin/watch_common.py @@ -20,7 +20,7 @@ from multiprocessing.pool import ThreadPool from threading import get_native_id -from watchdog.utils import WatchdogShutdown +from watchdog.utils import WatchdogShutdownError from watchdog.observers import Observer from watchdog.observers.polling import PollingObserver from collections import namedtuple, defaultdict, OrderedDict @@ -327,9 +327,9 @@ def WatchAndProcessDirectory( observer.join(1) if shuttingDown[0]: - raise WatchdogShutdown() + raise WatchdogShutdownError() - except WatchdogShutdown: + except WatchdogShutdownError: observer.unschedule_all() finally: diff --git a/shared/bin/zeek_carve_watcher.py b/shared/bin/zeek_carve_watcher.py index 960464905..e457edd19 100755 --- a/shared/bin/zeek_carve_watcher.py +++ b/shared/bin/zeek_carve_watcher.py @@ -24,7 +24,7 @@ from multiprocessing.pool import ThreadPool from watchdog.observers import Observer from watchdog.observers.polling import PollingObserver -from watchdog.utils import WatchdogShutdown +from watchdog.utils import WatchdogShutdownError from zeek_carve_utils import ( CAPA_VIV_MIME, @@ -320,9 +320,9 @@ def main(): logging.info(f"{scriptName}:\tshutting down...") if shuttingDown[0]: - raise WatchdogShutdown() + raise WatchdogShutdownError() - except WatchdogShutdown: + except WatchdogShutdownError: observer.unschedule_all() finally: