next/595/20241007/v1

.github/workflows/builds.yml

@@ -3144,6 +3144,7 @@ jobs:
          brew install \
           autoconf \
           automake \
+          cbindgen \
           curl \
           hiredis \
           jansson \
@@ -3152,14 +3153,11 @@ jobs:
           libnet \
           libtool \
           libyaml \
-          pyyaml \
           pcre2 \
           pkg-config \
           python \
           rust \
           xz
-      - name: Install cbindgen
-        run: cargo install --debug --version 0.24.3 cbindgen
       - run: echo "$HOME/.cargo/bin" >> $GITHUB_PATH
       - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938
       - run: git config --global --add safe.directory /__w/suricata/suricata
@@ -3170,16 +3168,28 @@ jobs:
           path: prep
       - run: tar xvf prep/libhtp.tar.gz
       - run: tar xvf prep/suricata-update.tar.gz
+      - name: Create Python virtual environment
+        run: python3 -m venv ./testenv
+      - name: Install PyYAML
+        run: |
+          . ./testenv/bin/activate
+          pip install pyyaml
       - run: ./autogen.sh
       - run: CPATH="$HOMEBREW_PREFIX/include:$CPATH" LIBRARY_PATH="$HOMEBREW_PREFIX/lib:$LIBRARY_PATH" PATH="/opt/homebrew/opt/libtool/libexec/gnubin:$PATH" CFLAGS="${DEFAULT_CFLAGS}" ./configure  --enable-warnings --enable-unittests --prefix="$HOME/.local/"
       - run: CPATH="$HOMEBREW_PREFIX/include:$CPATH" LIBRARY_PATH="$HOMEBREW_PREFIX/lib:$LIBRARY_PATH" PATH="/opt/homebrew/opt/libtool/libexec/gnubin:$PATH" CFLAGS="${DEFAULT_CFLAGS}" make -j2
       # somehow it gets included by some C++ stdlib header (case unsensitive)
       - run: rm libhtp/VERSION && make check
       - run: tar xf prep/suricata-verify.tar.gz
       - name: Running suricata-verify
-        run: python3 ./suricata-verify/run.py -q --debug-failed
+        run: |
+          . ./testenv/bin/activate
+          python3 ./suricata-verify/run.py -q --debug-failed
       - run: make install
-      - run: suricata-update -V
+      - name: Check Suricata-Update
+        run: |
+          . ./testenv/bin/activate
+          which suricata-update
+          python3 $(which suricata-update) -V
       - run: suricatasc -h
 
   windows-msys2-mingw64-npcap:

Makefile.am

@@ -10,6 +10,7 @@ EXTRA_DIST = ChangeLog COPYING LICENSE suricata.yaml.in \
 	     scripts/generate-images.sh \
 	     scripts/docs-almalinux9-minimal-build.sh \
 	     scripts/docs-ubuntu-debian-minimal-build.sh \
+       scripts/evedoc.py \
 	     examples/plugins
 SUBDIRS = $(HTP_DIR) rust src plugins qa rules doc contrib etc python ebpf \
           $(SURICATA_UPDATE_DIR)

doc/userguide/.gitignore

@@ -1 +1,2 @@
 _build
+_generated

doc/userguide/Makefile.am

@@ -1,7 +1,9 @@
 EXTRA_DIST = \
+	_generated \
 	_static \
 	3rd-party-integration \
 	acknowledgements.rst \
+	appendix \
 	capture-hardware \
 	command-line-options.rst \
 	conf.py \
@@ -47,14 +49,14 @@ endif
 
 SPHINX_BUILD = sphinx-build -q
 
-html:
+html: _generated
 	sysconfdir=$(sysconfdir) \
 	localstatedir=$(localstatedir) \
 	version=$(PACKAGE_VERSION) \
 		$(SPHINX_BUILD) -W -b html -d _build/doctrees \
 		$(top_srcdir)/doc/userguide _build/html
 
-_build/latex/Suricata.pdf:
+_build/latex/Suricata.pdf: _generated
 	sysconfdir=$(sysconfdir) \
 	localstatedir=$(localstatedir) \
 	version=$(PACKAGE_VERSION) \
@@ -74,7 +76,7 @@ userguide.pdf: _build/latex/Suricata.pdf
 
 pdf: userguide.pdf
 
-_build/man: manpages/suricata.rst manpages/suricatasc.rst manpages/suricatactl.rst manpages/suricatactl-filestore.rst
+_build/man: _generated manpages/suricata.rst manpages/suricatasc.rst manpages/suricatactl.rst manpages/suricatactl-filestore.rst
 	RELEASE_DATE=$(RELEASE_DATE) \
 	sysconfdir=$(sysconfdir) \
 	localstatedir=$(localstatedir) \
@@ -95,3 +97,6 @@ clean-local:
 	rm -f $(top_builddir)/doc/userguide/userguide.pdf
 
 endif # SPHINX_BUILD
+
+_generated:
+	./generate-evedoc.sh

doc/userguide/appendix/eve-index.rst

@@ -0,0 +1,7 @@
+EVE Index
+=========
+
+.. toctree::
+   :maxdepth: 1
+
+.. include:: ../_generated/eve-index.rst

doc/userguide/appendix/index.rst

@@ -0,0 +1,7 @@
+Appendix
+========
+
+.. toctree::
+   :maxdepth: 1
+
+   eve-index

doc/userguide/conf.py

@@ -151,6 +151,11 @@ def setup(app):
     else:
         app.add_stylesheet('css/suricata.css')
 
+    # Build generated pages if they don't exist. For example, when on
+    # RTD and we're build from git instead of a distribution package.
+    if not os.path.exists("./_generated"):
+        os.system("./generate-evedoc.sh")
+
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
@@ -338,3 +343,11 @@ def setup(app):
     "sysconfdir": os.getenv("sysconfdir", "/etc"),
     "localstatedir": os.getenv("localstatedir", "/var"),
 }
+
+# Custom code generate some documentation.
+# evedoc = "./evedoc.py"
+# eve_schema = "../../etc/schema.json"
+# os.makedirs("_generated", exist_ok=True)
+# subprocess.call([evedoc, "--output", "_generated/eve-index.rst", eve_schema])
+# for proto in ["quic", "pgsql"]:
+#     subprocess.call([evedoc, "--output", "_generated/{}.rst".format(proto), "--object", proto, eve_schema])

doc/userguide/generate-evedoc.sh

@@ -0,0 +1,13 @@
+#! /bin/sh
+#
+# Generate RST EVE documentation.
+#
+# This has been broken out of the Makefile so it can be called by
+# make, and Sphinx via conf.py.
+
+set -e
+
+mkdir -p _generated
+../../scripts/evedoc.py --output _generated/eve-index.rst ../../etc/schema.json
+../../scripts/evedoc.py --output _generated/quic.rst --object quic ../../etc/schema.json
+../../scripts/evedoc.py --output _generated/pgsql.rst --object pgsql ../../etc/schema.json

doc/userguide/index.rst

@@ -34,4 +34,5 @@ This is the documentation for Suricata |version|.
    acknowledgements
    licenses/index.rst
    devguide/index.rst
-   verifying-source-files
+   verifying-source-files
+   appendix/index.rst

doc/userguide/output/eve/eve-json-format.rst

@@ -2682,6 +2682,10 @@ References:
 .. _PostgreSQL message format - BackendKeyData: https://www.postgresql.org/docs
    /current/protocol-message-formats.html#PROTOCOL-MESSAGE-FORMATS-BACKENDKEYDATA
 
+Field Reference
+~~~~~~~~~~~~~~~
+
+.. include:: ../../_generated/pgsql.rst
 
 Event type: IKE
 ---------------
@@ -2937,6 +2941,11 @@ Example of QUIC logging with CYU, JA3 and JA4 hashes (note that the JA4 hash is
     "ja4": "q13d0310h3_55b375c5d22e_cd85d2d88918"
   }
 
+Output Reference
+~~~~~~~~~~~~~~~~
+
+.. include:: ../../_generated/quic.rst
+
 Event type: DHCP
 -----------------
 
@@ -3048,4 +3057,3 @@ Example of ARP logging: request and response
     "dest_mac": "00:1d:09:f0:92:ab",
     "dest_ip": "10.10.10.1"
   }
-

etc/Makefile.am

@@ -1,6 +1,7 @@
 etcdatadir =	$(datadir)/suricata
 
-EXTRA_DIST =	suricata.logrotate.in \
+EXTRA_DIST =	schema.json \
+		suricata.logrotate.in \
 		suricata.service.in
 
 dist_etcdata_DATA =	classification.config \