diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 46ed7b0e5243..a601be609fc4 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -3144,6 +3144,7 @@ jobs: brew install \ autoconf \ automake \ + cbindgen \ curl \ hiredis \ jansson \ @@ -3152,14 +3153,11 @@ jobs: libnet \ libtool \ libyaml \ - pyyaml \ pcre2 \ pkg-config \ python \ rust \ xz - - name: Install cbindgen - run: cargo install --debug --version 0.24.3 cbindgen - run: echo "$HOME/.cargo/bin" >> $GITHUB_PATH - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 - run: git config --global --add safe.directory /__w/suricata/suricata @@ -3170,6 +3168,12 @@ jobs: path: prep - run: tar xvf prep/libhtp.tar.gz - run: tar xvf prep/suricata-update.tar.gz + - name: Create Python virtual environment + run: python3 -m venv ./testenv + - name: Install PyYAML + run: | + . ./testenv/bin/activate + pip install pyyaml - run: ./autogen.sh - run: CPATH="$HOMEBREW_PREFIX/include:$CPATH" LIBRARY_PATH="$HOMEBREW_PREFIX/lib:$LIBRARY_PATH" PATH="/opt/homebrew/opt/libtool/libexec/gnubin:$PATH" CFLAGS="${DEFAULT_CFLAGS}" ./configure --enable-warnings --enable-unittests --prefix="$HOME/.local/" - run: CPATH="$HOMEBREW_PREFIX/include:$CPATH" LIBRARY_PATH="$HOMEBREW_PREFIX/lib:$LIBRARY_PATH" PATH="/opt/homebrew/opt/libtool/libexec/gnubin:$PATH" CFLAGS="${DEFAULT_CFLAGS}" make -j2 @@ -3177,9 +3181,15 @@ jobs: - run: rm libhtp/VERSION && make check - run: tar xf prep/suricata-verify.tar.gz - name: Running suricata-verify - run: python3 ./suricata-verify/run.py -q --debug-failed + run: | + . ./testenv/bin/activate + python3 ./suricata-verify/run.py -q --debug-failed - run: make install - - run: suricata-update -V + - name: Check Suricata-Update + run: | + . ./testenv/bin/activate + which suricata-update + python3 $(which suricata-update) -V - run: suricatasc -h windows-msys2-mingw64-npcap: diff --git a/Makefile.am b/Makefile.am index 6e2f77cadc24..20e50bdc4a03 100644 --- a/Makefile.am +++ b/Makefile.am @@ -10,6 +10,7 @@ EXTRA_DIST = ChangeLog COPYING LICENSE suricata.yaml.in \ scripts/generate-images.sh \ scripts/docs-almalinux9-minimal-build.sh \ scripts/docs-ubuntu-debian-minimal-build.sh \ + scripts/evedoc.py \ examples/plugins SUBDIRS = $(HTP_DIR) rust src plugins qa rules doc contrib etc python ebpf \ $(SURICATA_UPDATE_DIR) diff --git a/doc/userguide/.gitignore b/doc/userguide/.gitignore index e35d8850c968..1f45e1e2c5a8 100644 --- a/doc/userguide/.gitignore +++ b/doc/userguide/.gitignore @@ -1 +1,2 @@ _build +_generated diff --git a/doc/userguide/Makefile.am b/doc/userguide/Makefile.am index 53f4907eb743..aeab4768b086 100644 --- a/doc/userguide/Makefile.am +++ b/doc/userguide/Makefile.am @@ -1,7 +1,9 @@ EXTRA_DIST = \ + _generated \ _static \ 3rd-party-integration \ acknowledgements.rst \ + appendix \ capture-hardware \ command-line-options.rst \ conf.py \ @@ -47,14 +49,14 @@ endif SPHINX_BUILD = sphinx-build -q -html: +html: _generated sysconfdir=$(sysconfdir) \ localstatedir=$(localstatedir) \ version=$(PACKAGE_VERSION) \ $(SPHINX_BUILD) -W -b html -d _build/doctrees \ $(top_srcdir)/doc/userguide _build/html -_build/latex/Suricata.pdf: +_build/latex/Suricata.pdf: _generated sysconfdir=$(sysconfdir) \ localstatedir=$(localstatedir) \ version=$(PACKAGE_VERSION) \ @@ -74,7 +76,7 @@ userguide.pdf: _build/latex/Suricata.pdf pdf: userguide.pdf -_build/man: manpages/suricata.rst manpages/suricatasc.rst manpages/suricatactl.rst manpages/suricatactl-filestore.rst +_build/man: _generated manpages/suricata.rst manpages/suricatasc.rst manpages/suricatactl.rst manpages/suricatactl-filestore.rst RELEASE_DATE=$(RELEASE_DATE) \ sysconfdir=$(sysconfdir) \ localstatedir=$(localstatedir) \ @@ -95,3 +97,6 @@ clean-local: rm -f $(top_builddir)/doc/userguide/userguide.pdf endif # SPHINX_BUILD + +_generated: + ./generate-evedoc.sh diff --git a/doc/userguide/appendix/eve-index.rst b/doc/userguide/appendix/eve-index.rst new file mode 100644 index 000000000000..0789c04e5ddc --- /dev/null +++ b/doc/userguide/appendix/eve-index.rst @@ -0,0 +1,7 @@ +EVE Index +========= + +.. toctree:: + :maxdepth: 1 + +.. include:: ../_generated/eve-index.rst diff --git a/doc/userguide/appendix/index.rst b/doc/userguide/appendix/index.rst new file mode 100644 index 000000000000..56cf7923b4e8 --- /dev/null +++ b/doc/userguide/appendix/index.rst @@ -0,0 +1,7 @@ +Appendix +======== + +.. toctree:: + :maxdepth: 1 + + eve-index diff --git a/doc/userguide/conf.py b/doc/userguide/conf.py index 959744e88b7b..48d5e24dad26 100644 --- a/doc/userguide/conf.py +++ b/doc/userguide/conf.py @@ -151,6 +151,11 @@ def setup(app): else: app.add_stylesheet('css/suricata.css') + # Build generated pages if they don't exist. For example, when on + # RTD and we're build from git instead of a distribution package. + if not os.path.exists("./_generated"): + os.system("./generate-evedoc.sh") + # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. @@ -338,3 +343,11 @@ def setup(app): "sysconfdir": os.getenv("sysconfdir", "/etc"), "localstatedir": os.getenv("localstatedir", "/var"), } + +# Custom code generate some documentation. +# evedoc = "./evedoc.py" +# eve_schema = "../../etc/schema.json" +# os.makedirs("_generated", exist_ok=True) +# subprocess.call([evedoc, "--output", "_generated/eve-index.rst", eve_schema]) +# for proto in ["quic", "pgsql"]: +# subprocess.call([evedoc, "--output", "_generated/{}.rst".format(proto), "--object", proto, eve_schema]) diff --git a/doc/userguide/generate-evedoc.sh b/doc/userguide/generate-evedoc.sh new file mode 100755 index 000000000000..b9dca7867e86 --- /dev/null +++ b/doc/userguide/generate-evedoc.sh @@ -0,0 +1,13 @@ +#! /bin/sh +# +# Generate RST EVE documentation. +# +# This has been broken out of the Makefile so it can be called by +# make, and Sphinx via conf.py. + +set -e + +mkdir -p _generated +../../scripts/evedoc.py --output _generated/eve-index.rst ../../etc/schema.json +../../scripts/evedoc.py --output _generated/quic.rst --object quic ../../etc/schema.json +../../scripts/evedoc.py --output _generated/pgsql.rst --object pgsql ../../etc/schema.json diff --git a/doc/userguide/index.rst b/doc/userguide/index.rst index 1440fa82801a..7bfead762759 100644 --- a/doc/userguide/index.rst +++ b/doc/userguide/index.rst @@ -34,4 +34,5 @@ This is the documentation for Suricata |version|. acknowledgements licenses/index.rst devguide/index.rst - verifying-source-files \ No newline at end of file + verifying-source-files + appendix/index.rst diff --git a/doc/userguide/output/eve/eve-json-format.rst b/doc/userguide/output/eve/eve-json-format.rst index e8a5ffc03119..952945dffc98 100644 --- a/doc/userguide/output/eve/eve-json-format.rst +++ b/doc/userguide/output/eve/eve-json-format.rst @@ -2682,6 +2682,10 @@ References: .. _PostgreSQL message format - BackendKeyData: https://www.postgresql.org/docs /current/protocol-message-formats.html#PROTOCOL-MESSAGE-FORMATS-BACKENDKEYDATA +Field Reference +~~~~~~~~~~~~~~~ + +.. include:: ../../_generated/pgsql.rst Event type: IKE --------------- @@ -2937,6 +2941,11 @@ Example of QUIC logging with CYU, JA3 and JA4 hashes (note that the JA4 hash is "ja4": "q13d0310h3_55b375c5d22e_cd85d2d88918" } +Output Reference +~~~~~~~~~~~~~~~~ + +.. include:: ../../_generated/quic.rst + Event type: DHCP ----------------- @@ -3048,4 +3057,3 @@ Example of ARP logging: request and response "dest_mac": "00:1d:09:f0:92:ab", "dest_ip": "10.10.10.1" } - diff --git a/etc/Makefile.am b/etc/Makefile.am index 0466d5b7dd42..586cb51ce1cb 100644 --- a/etc/Makefile.am +++ b/etc/Makefile.am @@ -1,6 +1,7 @@ etcdatadir = $(datadir)/suricata -EXTRA_DIST = suricata.logrotate.in \ +EXTRA_DIST = schema.json \ + suricata.logrotate.in \ suricata.service.in dist_etcdata_DATA = classification.config \ diff --git a/scripts/evedoc.py b/scripts/evedoc.py new file mode 100755 index 000000000000..ad233563b6bb --- /dev/null +++ b/scripts/evedoc.py @@ -0,0 +1,211 @@ +#! /usr/bin/env python3 +# +# Generate Sphinx documentation from JSON schema + +import argparse +import sys +import json + + +def errprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def find_ref(schema: dict, ref: str) -> dict: + parts = ref.split("/") + + root = parts.pop(0) + if root != "#": + raise Exception("Unsupported reference: {}".format(ref)) + + while parts: + schema = schema[parts.pop(0)] + + return schema + + +def get_type(props: dict, name: str) -> str: + prop_type = props["type"] + if prop_type == "array": + try: + array_type = props["items"]["type"] + except KeyError: + errprint("warning: array property without items: {}".format(name)) + array_type = "unknown" + prop_type = "array of {}s".format(array_type) + return prop_type + + +def render_flat(schema: dict): + stack = [(schema, [])] + + while stack: + (current, path) = stack.pop(0) + + for name, props in current["properties"].items(): + if "$ref" in props: + ref = find_ref(schema, props["$ref"]) + if not ref: + raise Exception("$ref not found: {}".format(props["$ref"])) + props = ref + if props["type"] in ["string", "integer", "boolean", "number"]: + # End of the line... + print("{}: {}".format(".".join(path + [name]), props["type"])) + elif props["type"] == "object": + print("{}: object".format(".".join(path + [name]))) + if "properties" in props: + stack.insert(0, (props, path + [name])) + else: + errprint( + "warning: object without properties: {}".format( + ".".join(path + [name]) + ) + ) + elif props["type"] == "array": + if "items" in props and "type" in props["items"]: + print( + "{}: {}[]".format( + ".".join(path + [name]), props["items"]["type"] + ) + ) + if "properties" in props["items"]: + stack.insert( + 0, + ( + props["items"], + path + ["{}[]".format(name)], + ), + ) + else: + errprint( + "warning: undocumented array: {}".format( + ".".join(path + [name]) + ) + ) + print("{}: array".format(".".join(path + [name]))) + else: + raise Exception("Unsupported type: {}".format(props["type"])) + + +def render_rst(schema: dict): + stack = [(schema, [], "object")] + + while stack: + (current, path, type) = stack.pop(0) + + items = [] + + for name, props in current["properties"].items(): + if "$ref" in props: + ref = find_ref(schema, props["$ref"]) + if not ref: + raise Exception( + "Reference not found: {}".format(props["$ref"]) + ) + props = ref + prop_type = get_type(props, name) + description = props.get("description", "") + + items.append( + {"name": name, "type": prop_type, "description": description} + ) + + if props["type"] == "object" and "properties" in props: + stack.insert(0, (props, path + [name], "object")) + elif ( + props["type"] == "array" + and "items" in props + and "properties" in props["items"] + ): + array_type = props["items"]["type"] + stack.insert( + 0, + ( + props["items"], + path + ["{}".format(name)], + "array of {}s".format(array_type), + ), + ) + + render_rst_table(items, path, type) + + +def render_rst_table(items: list, path: list, type: str): + if not path: + title = "Top Level" + else: + title = ".".join(path) + title = "{} ({})".format(title, type) + print(title) + print("^" * len(title)) + + name_len = max([len(item["name"]) for item in items] + [len("Name")]) + desc_len = max( + [len(item["description"]) for item in items] + [len("Description")] + ) + type_len = max([len(item["type"]) for item in items]) + + print(".. table::") + print(" :width: 100%") + print(" :widths: 30 25 45") + print("") + + print(" {} {} {}".format("=" * name_len, "=" * type_len, "=" * desc_len)) + print( + " {} {} {}".format( + "Name".ljust(name_len), + "Type".ljust(type_len), + "Description".ljust(desc_len), + ) + ) + print(" {} {} {}".format("=" * name_len, "=" * type_len, "=" * desc_len)) + for item in items: + print( + " {} {} {}".format( + item["name"].ljust(name_len), + item["type"].ljust(type_len), + item["description"].ljust(desc_len), + ) + ) + print(" {} {} {}".format("=" * name_len, "=" * type_len, "=" * desc_len)) + print("") + + +epilog = """ + +By default, the EVE schema is rendered as Sphinx documentation. To +create "flat" or "dot" separated output, use the --flat option. + +""" + + +def main(): + parser = argparse.ArgumentParser( + description="Generate documentation from JSON schema", + epilog=epilog, + formatter_class=argparse.RawTextHelpFormatter, + ) + parser.add_argument("--object", help="Object name") + parser.add_argument("--output", help="Output file") + parser.add_argument("--flat", help="Flatten output", action="store_true") + parser.add_argument("filename", help="JSON schema file") + + args = parser.parse_args() + + root = json.load(open(args.filename)) + schema = root + + if args.object: + schema = schema["properties"][args.object] + + if args.output: + sys.stdout = open(args.output, "w") + + if args.flat: + render_flat(schema) + else: + render_rst(schema) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/tm-modules.c b/src/tm-modules.c index dbab61fa8895..0b3212eb7425 100644 --- a/src/tm-modules.c +++ b/src/tm-modules.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2010 Open Information Security Foundation +/* Copyright (C) 2007-2024 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -23,22 +23,15 @@ * Thread Module functions */ -#include "suricata-common.h" -#include "packet-queue.h" -#include "tm-threads.h" +#include "tm-modules.h" #include "util-debug.h" -#include "threads.h" -#include "util-logopenfile.h" TmModule tmm_modules[TMM_SIZE]; void TmModuleDebugList(void) { - TmModule *t; - uint16_t i; - - for (i = 0; i < TMM_SIZE; i++) { - t = &tmm_modules[i]; + for (uint16_t i = 0; i < TMM_SIZE; i++) { + TmModule *t = &tmm_modules[i]; if (t->name == NULL) continue; @@ -52,11 +45,8 @@ void TmModuleDebugList(void) * \retval ptr to the module or NULL */ TmModule *TmModuleGetByName(const char *name) { - TmModule *t; - uint16_t i; - - for (i = 0; i < TMM_SIZE; i++) { - t = &tmm_modules[i]; + for (uint16_t i = 0; i < TMM_SIZE; i++) { + TmModule *t = &tmm_modules[i]; if (t->name == NULL) continue; @@ -78,7 +68,6 @@ TmModule *TmModuleGetByName(const char *name) */ TmModule *TmModuleGetById(int id) { - if (id < 0 || id >= TMM_SIZE) { SCLogError("Threading module with the id " "\"%d\" doesn't exist", @@ -98,11 +87,8 @@ TmModule *TmModuleGetById(int id) */ int TmModuleGetIDForTM(TmModule *tm) { - TmModule *t; - int i; - - for (i = 0; i < TMM_SIZE; i++) { - t = &tmm_modules[i]; + for (uint16_t i = 0; i < TMM_SIZE; i++) { + TmModule *t = &tmm_modules[i]; if (t->name == NULL) continue; @@ -117,11 +103,8 @@ int TmModuleGetIDForTM(TmModule *tm) void TmModuleRunInit(void) { - TmModule *t; - uint16_t i; - - for (i = 0; i < TMM_SIZE; i++) { - t = &tmm_modules[i]; + for (uint16_t i = 0; i < TMM_SIZE; i++) { + TmModule *t = &tmm_modules[i]; if (t->name == NULL) continue; @@ -135,11 +118,8 @@ void TmModuleRunInit(void) void TmModuleRunDeInit(void) { - TmModule *t; - uint16_t i; - - for (i = 0; i < TMM_SIZE; i++) { - t = &tmm_modules[i]; + for (uint16_t i = 0; i < TMM_SIZE; i++) { + TmModule *t = &tmm_modules[i]; if (t->name == NULL) continue; @@ -155,11 +135,8 @@ void TmModuleRunDeInit(void) void TmModuleRegisterTests(void) { #ifdef UNITTESTS - TmModule *t; - uint16_t i; - - for (i = 0; i < TMM_SIZE; i++) { - t = &tmm_modules[i]; + for (uint16_t i = 0; i < TMM_SIZE; i++) { + TmModule *t = &tmm_modules[i]; if (t->name == NULL) continue; diff --git a/src/tm-modules.h b/src/tm-modules.h index 70817fb0792d..f155089e2cc9 100644 --- a/src/tm-modules.h +++ b/src/tm-modules.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2014 Open Information Security Foundation +/* Copyright (C) 2007-2024 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -33,9 +33,8 @@ #define TM_FLAG_DECODE_TM 0x02 #define TM_FLAG_STREAM_TM 0x04 #define TM_FLAG_DETECT_TM 0x08 -#define TM_FLAG_LOGAPI_TM 0x10 /**< TM is run by Log API */ -#define TM_FLAG_MANAGEMENT_TM 0x20 -#define TM_FLAG_COMMAND_TM 0x40 +#define TM_FLAG_MANAGEMENT_TM 0x10 +#define TM_FLAG_COMMAND_TM 0x20 typedef TmEcode (*ThreadInitFunc)(ThreadVars *, const void *, void **); typedef TmEcode (*ThreadDeinitFunc)(ThreadVars *, void *); diff --git a/src/util-profiling.c b/src/util-profiling.c index 56c31557285b..101b1e5ac80f 100644 --- a/src/util-profiling.c +++ b/src/util-profiling.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2023 Open Information Security Foundation +/* Copyright (C) 2007-2024 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -434,9 +434,6 @@ void SCProfilingDumpPacketStats(void) #endif total = 0; for (int m = 0; m < TMM_SIZE; m++) { - if (tmm_modules[m].flags & TM_FLAG_LOGAPI_TM) - continue; - for (int p = 0; p < 257; p++) { SCProfilePacketData *pd = &packet_profile_tmm_data4[m][p]; total += pd->tot; @@ -447,9 +444,6 @@ void SCProfilingDumpPacketStats(void) } for (int m = 0; m < TMM_SIZE; m++) { - if (tmm_modules[m].flags & TM_FLAG_LOGAPI_TM) - continue; - for (int p = 0; p < 257; p++) { SCProfilePacketData *pd = &packet_profile_tmm_data4[m][p]; if (pd->cnt == 0) { @@ -472,9 +466,6 @@ void SCProfilingDumpPacketStats(void) } for (int m = 0; m < TMM_SIZE; m++) { - if (tmm_modules[m].flags & TM_FLAG_LOGAPI_TM) - continue; - for (int p = 0; p < 257; p++) { SCProfilePacketData *pd = &packet_profile_tmm_data6[m][p]; if (pd->cnt == 0) { @@ -601,9 +592,6 @@ void SCProfilingDumpPacketStats(void) #endif total = 0; for (int m = 0; m < TMM_SIZE; m++) { - if (!(tmm_modules[m].flags & TM_FLAG_LOGAPI_TM)) - continue; - for (int p = 0; p < 257; p++) { SCProfilePacketData *pd = &packet_profile_tmm_data4[m][p]; total += pd->tot; @@ -614,9 +602,6 @@ void SCProfilingDumpPacketStats(void) } for (int m = 0; m < TMM_SIZE; m++) { - if (!(tmm_modules[m].flags & TM_FLAG_LOGAPI_TM)) - continue; - for (int p = 0; p < 257; p++) { SCProfilePacketData *pd = &packet_profile_tmm_data4[m][p]; if (pd->cnt == 0) { @@ -639,9 +624,6 @@ void SCProfilingDumpPacketStats(void) } for (int m = 0; m < TMM_SIZE; m++) { - if (!(tmm_modules[m].flags & TM_FLAG_LOGAPI_TM)) - continue; - for (int p = 0; p < 257; p++) { SCProfilePacketData *pd = &packet_profile_tmm_data6[m][p]; if (pd->cnt == 0) {