ndpiReader: update JA statistics #4940
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| on: | |
| push: | |
| branches: | |
| - dev | |
| pull_request: | |
| branches: | |
| - dev | |
| types: [opened, synchronize, reopened] | |
| release: | |
| types: [created] | |
| jobs: | |
| python-bindings: | |
| name: Python Bindings (ubuntu-latest) | |
| runs-on: ubuntu-latest | |
| env: | |
| CFLAGS: -Wextra -Werror | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python 3.9 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.9" | |
| - name: Install Ubuntu Prerequisites | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev | |
| - name: Build nDPI library | |
| run: | | |
| ./autogen.sh | |
| make -j | |
| sudo make install | |
| - name: Generate Python bindings | |
| run: | | |
| pip install --upgrade pip | |
| pip install -r python/requirements.txt | |
| cd python | |
| python setup.py install | |
| cd .. | |
| - name: Test Python Bindings | |
| run: | | |
| cd python | |
| python tests.py | |
| test-scripts: | |
| name: Test Utils (ubuntu-latest) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install Ubuntu Prerequisites | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install python3-netaddr git whois libxml2-utils | |
| - name: Run Scripts | |
| run: | | |
| echo 'Running ./utils/bitcoinnodes.sh' | |
| ./utils/bitcoinnodes.sh >/dev/null | |
| echo 'Running ./utils/get_routes_by_asn.sh AS714' | |
| ./utils/get_routes_by_asn.sh AS714 >/dev/null | |
| echo 'Running ./utils/update_every_lists.sh' | |
| ./utils/update_every_lists.sh | |
| echo 'Checking for changes in the git tree..' | |
| git update-index --refresh || echo "::warning file=utils/update_every_lists.sh::Please re-run utils/update_every_lists.sh and commit any changes." | |
| git diff-index --quiet HEAD -- || true | |
| test: | |
| name: ${{ matrix.os }} ${{ matrix.arch }} ${{ matrix.gcrypt }} ${{ matrix.compiler }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} ${{ matrix.msan }} ${{ matrix.nBPF }} ${{matrix.lto_gold_linker}} ${{matrix.global_context}} | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| CC: ${{ matrix.compiler }} | |
| CFLAGS: -Wextra -Werror -DNDPI_EXTENDED_SANITY_CHECKS | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| # macOS-latest == macos-14 on **ARM64**. Also macos-15 is on arm64 | |
| # There are some issues with external dependencies on macOS-14/15. Disable it for the time being | |
| os: ["ubuntu-20.04", "ubuntu-22.04", "ubuntu-24.04", "macOS-13", "windows-latest"] | |
| arch: ["x86_64"] | |
| gcrypt: ["--with-local-libgcrypt", ""] | |
| compiler: ["cc"] | |
| ar: ["ar"] | |
| ranlib: ["ranlib"] | |
| pcre: [""] | |
| maxminddb: [""] | |
| msan: [""] | |
| nBPF: [""] | |
| lto_gold_linker: [""] | |
| global_context: [""] #Enable by default | |
| include: | |
| - compiler: "gcc-4.9" # "Oldest" gcc easily available. To simulate RHEL7 | |
| os: ubuntu-20.04 | |
| arch: "x86_64" | |
| gcrypt: "" | |
| pcre: "--with-pcre2" | |
| maxminddb: "--with-maxminddb" | |
| msan: "--with-sanitizer" | |
| nBPF: "" | |
| - compiler: "gcc-14" # "Newest" gcc easily available | |
| os: ubuntu-24.04 | |
| arch: "x86_64" | |
| gcrypt: "" | |
| pcre: "--with-pcre2" | |
| maxminddb: "--with-maxminddb" | |
| msan: "--with-sanitizer" | |
| nBPF: "" | |
| lto_gold_linker: "--with-lto-and-gold-linker" | |
| - compiler: "clang-9" # "Oldest" clang easily available | |
| os: ubuntu-20.04 | |
| arch: "x86_64" | |
| gcrypt: "" | |
| pcre: "--with-pcre2" | |
| maxminddb: "--with-maxminddb" | |
| msan: "--with-sanitizer" | |
| nBPF: "" | |
| - compiler: "clang-18" # "Newest" clang easily available. See also below... | |
| ar: "llvm-ar-18" | |
| ranlib: "llvm-ranlib-18" | |
| os: ubuntu-24.04 | |
| arch: "x86_64" | |
| gcrypt: "" | |
| pcre: "--with-pcre2" | |
| maxminddb: "--with-maxminddb" | |
| msan: "--with-sanitizer" | |
| nBPF: "" | |
| lto_gold_linker: "--with-lto-and-gold-linker" | |
| - compiler: "cc" | |
| os: ubuntu-latest | |
| arch: "x86_64" | |
| gcrypt: "" | |
| pcre: "--with-pcre2" | |
| maxminddb: "--with-maxminddb" | |
| msan: "--with-thread-sanitizer" | |
| nBPF: "" | |
| - compiler: "cc" | |
| os: ubuntu-latest | |
| arch: "x86_64" | |
| gcrypt: "" | |
| pcre: "--with-pcre2" | |
| maxminddb: "--with-maxminddb" | |
| msan: "--with-sanitizer" | |
| nBPF: "nBPF" | |
| - compiler: "cc" | |
| os: ubuntu-latest | |
| arch: "x86_64" | |
| gcrypt: "" | |
| pcre: "--with-pcre2" | |
| maxminddb: "--with-maxminddb" | |
| msan: "--with-sanitizer" | |
| nBPF: "" | |
| global_context: "--disable-global-context-support" | |
| - compiler: "clang" #TODO: some issues with masan/clang/ubuntu-24.04 | |
| os: ubuntu-22.04 | |
| arch: "x86_64" | |
| gcrypt: "" | |
| pcre: "--with-pcre2" | |
| maxminddb: "--with-maxminddb" | |
| msan: "--with-memory-sanitizer --disable-memory-track-origins" | |
| nBPF: "" | |
| - compiler: "cc" | |
| os: macOS-13 | |
| arch: "x86_64" | |
| gcrypt: "" | |
| pcre: "--with-pcre2" | |
| maxminddb: "--with-maxminddb" | |
| msan: "" # Disable sanitizer on macos | |
| nBPF: "" | |
| - compiler: "cc" | |
| os: macos-14 | |
| arch: "x86_64" | |
| gcrypt: "" | |
| pcre: "" | |
| maxminddb: "" | |
| msan: "" # Disable sanitizer on macos | |
| nBPF: "" | |
| - compiler: "cc" | |
| os: macos-15 | |
| arch: "x86_64" | |
| gcrypt: "" | |
| pcre: "" | |
| maxminddb: "" | |
| msan: "" # Disable sanitizer on macos | |
| nBPF: "" | |
| - compiler: "cc" | |
| os: ubuntu-latest | |
| arch: "arm64" | |
| gcrypt: "" | |
| pcre: "--with-pcre2" | |
| maxminddb: "--with-maxminddb" | |
| msan: "" # Disable sanitizer on arm64 | |
| nBPF: "" | |
| - compiler: "cc" | |
| os: ubuntu-latest | |
| arch: "armhf" | |
| gcrypt: "" | |
| pcre: "--with-pcre2" | |
| maxminddb: "--with-maxminddb" | |
| msan: "" | |
| nBPF: "" | |
| - compiler: "cc" | |
| os: ubuntu-latest | |
| arch: "s390x" | |
| gcrypt: "" | |
| pcre: "--with-pcre2" | |
| maxminddb: "--with-maxminddb" | |
| msan: "" | |
| nBPF: "" | |
| steps: | |
| - name: Setup multiarch/qemu-user-static | |
| if: startsWith(matrix.os, 'ubuntu') && !startsWith(matrix.arch, 'x86_64') | |
| run: | | |
| docker run --rm --privileged multiarch/qemu-user-static:register --reset | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis | |
| - name: Install Ubuntu Prerequisites | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install autoconf automake debhelper libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev | |
| sudo apt-get install rrdtool librrd-dev parallel | |
| - name: Install Ubuntu Prerequisites [Mingw-w64] (runs only on ubuntu jobs) | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') #Only on a few "standard" builds, without any sanitizers | |
| run: | | |
| sudo apt-get install gcc-mingw-w64 libc6-dev | |
| - name: Install Ubuntu Prerequisites (libgcrypt) | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.gcrypt, '--with-local-libgcrypt') | |
| run: | | |
| sudo apt-get install libgcrypt20-dev | |
| - name: Install Ubuntu Prerequisites (libpcre2) | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.pcre, '--with-pcre2') | |
| run: | | |
| sudo apt-get install libpcre3-dev | |
| - name: Install Ubuntu Prerequisites (maxminddb) | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.maxminddb, '--with-maxminddb') | |
| run: | | |
| sudo apt-get install libmaxminddb-dev | |
| - name: Install Ubuntu Prerequisites (nBPF) | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.nBPF, 'nBPF') | |
| run: | | |
| git clone https://github.com/ntop/PF_RING.git ../PF_RING | |
| cd ../PF_RING/userland/nbpf | |
| ./configure | |
| make | |
| cd - | |
| - name: Setup Ubuntu specified compiler | |
| if: startsWith(matrix.os, 'ubuntu-20.04') && startsWith(matrix.arch, 'x86_64') && ! startsWith(matrix.compiler, 'cc') | |
| run: | | |
| #For gcc-4.9 (on ubuntu-20.04) | |
| echo "deb http://dk.archive.ubuntu.com/ubuntu/ xenial main" | sudo tee -a /etc/apt/sources.list | |
| echo "deb http://dk.archive.ubuntu.com/ubuntu/ xenial universe" | sudo tee -a /etc/apt/sources.list | |
| sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5 | |
| sudo apt-get update | |
| sudo apt-get install ${{ matrix.compiler }} | |
| - name: Install Windows msys2 prerequisites | |
| if: startsWith(matrix.os, 'windows') | |
| uses: msys2/setup-msys2@v2 | |
| with: | |
| msystem: MINGW64 | |
| update: true | |
| install: git mingw-w64-x86_64-toolchain automake1.16 automake-wrapper autoconf libtool make mingw-w64-x86_64-json-c mingw-w64-x86_64-crt-git mingw-w64-x86_64-pcre mingw-w64-x86_64-libpcap mingw-w64-x86_64-libgcrypt parallel | |
| - name: Installing MacOS prerequisites | |
| if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') | |
| run: | | |
| # Avoid (re)installing pkg-config. See: https://github.com/actions/runner-images/issues/10984 | |
| brew install coreutils wdiff colordiff autoconf automake libtool gettext json-c rrdtool parallel | |
| - name: Install MacOS Prerequisites (libgcrypt) | |
| if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.gcrypt, '--with-local-libgcrypt') | |
| run: | | |
| brew install libgcrypt | |
| - name: Install MacOS Prerequisites (libpcre2) | |
| if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.pcre, '--with-pcre2') | |
| run: | | |
| brew install pcre | |
| - name: Install MacOS Prerequisites (maxminddb) | |
| if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.maxminddb, '--with-maxminddb') | |
| run: | | |
| brew install libmaxminddb | |
| - name: Configure nDPI on Ubuntu | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') | |
| run: | | |
| AR=${{ matrix.ar }} RANLIB=${{ matrix.ranlib }} ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs ${{matrix.lto_gold_linker}} ${{matrix.global_context}} | |
| - name: Configure nDPI on MacOS | |
| if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'cc') | |
| run: | | |
| ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs | |
| - name: Configure nDPI on Windows msys2 | |
| if: startsWith(matrix.os, 'windows') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'cc') | |
| run: | | |
| msys2 -c './autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --disable-npcap ${{ matrix.gcrypt }}' | |
| - name: Build nDPI on Windows msys2 | |
| if: startsWith(matrix.os, 'windows') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'cc') | |
| run: | | |
| msys2 -c 'make -j all' | |
| msys2 -c 'ldd ./example/ndpiReader.exe' | |
| - name: Build nDPI | |
| if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') && !startsWith(matrix.os, 'macos-14') && !startsWith(matrix.os, 'macos-15') | |
| run: | | |
| make -j all | |
| make -C example ndpiSimpleIntegration | |
| make -C rrdtool | |
| - name: Build nDPI (MacOS M1) | |
| if: startsWith(matrix.os, 'macos-14') || startsWith(matrix.os, 'macos-15') | |
| run: | | |
| make -j all | |
| make -C example ndpiSimpleIntegration | |
| #There are somes issues with librrd | |
| #make -C rrdtool | |
| - name: Print nDPI long help | |
| if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') | |
| run: | | |
| cd ./example && ./ndpiReader -H | |
| - name: Install nDPI | |
| if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') | |
| run: | | |
| DESTDIR=/tmp/ndpi make install | |
| ls -alhHR /tmp/ndpi | |
| - name: Test nDPI [SYMBOLS] | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') #Only on a few "standard" builds, without any sanitizers | |
| run: | | |
| ./utils/check_symbols.sh || { FAILED=$?; echo "::error file=${NDPI_LIB}::Unwanted libc symbols found: ${FAILED}. Please make sure to use only ndpi_malloc/ndpi_calloc/ndpi_realloc/ndpi_free wrapper instead of malloc/calloc/realloc/free."; false; } | |
| env: | |
| NDPI_LIB: src/lib/libndpi.a | |
| - name: Test nDPI [DIFF] | |
| if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') | |
| run: | | |
| NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh | |
| - name: Test nDPI [UNIT] | |
| #Some issues with masan + json-c. Disable the test as workaround | |
| if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') && !startsWith(matrix.msan, '--with-memory-sanitizer') && !startsWith(matrix.os, 'macos-14') && !startsWith(matrix.os, 'macos-15') | |
| run: | | |
| ./tests/do-unit.sh | |
| - name: Test nDPI [DGA] | |
| if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') | |
| run: | | |
| ./tests/do-dga.sh | |
| - name: Test nDPI [DIFF] (runs only on windows jobs) | |
| if: startsWith(matrix.arch, 'x86_64') && startsWith(matrix.os, 'windows') | |
| run: | | |
| msys2 -c 'NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh' | |
| - name: Test nDPI [UNIT] (runs only on windows jobs) | |
| if: startsWith(matrix.arch, 'x86_64') && startsWith(matrix.os, 'windows') | |
| run: | | |
| msys2 -c './tests/do-unit.sh' | |
| - name: Test nDPI [DGA] (runs only on windows jobs) | |
| if: startsWith(matrix.arch, 'x86_64') && startsWith(matrix.os, 'windows') | |
| run: | | |
| msys2 -c './tests/do-dga.sh' | |
| - name: Generate/Verify tarball | |
| if: startsWith(matrix.os, 'ubuntu-latest') && startsWith(matrix.arch, 'x86_64') | |
| run: | | |
| make dist | |
| ./utils/verify_dist_tarball.sh | |
| - name: Build Debian/Ubuntu package | |
| if: startsWith(matrix.os, 'ubuntu-24.04') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'cc') | |
| run: | | |
| cd packages/ubuntu | |
| ./configure --enable-no-sign | |
| make | |
| cd ../.. | |
| - name: Build nDPI [Mingw-w64] (runs only on ubuntu jobs) | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') #Only on a few "standard" builds, without any sanitizers | |
| run: | | |
| make distclean | |
| ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --host=x86_64-w64-mingw32 | |
| make -j $(nproc) all | |
| env: | |
| CC: | |
| - name: Display qemu specified architecture (arm64 - little endian) | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'arm64') | |
| uses: docker://multiarch/ubuntu-core:arm64-bionic | |
| with: | |
| args: > | |
| bash -c | |
| "uname -a && | |
| lscpu | grep Endian | |
| " | |
| - name: Configure, compile and test using qemu for the specified architecture (arm64 - little endian) | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'arm64') | |
| uses: docker://multiarch/ubuntu-core:arm64-bionic | |
| with: | |
| args: > | |
| bash -c | |
| "apt-get -y update && | |
| apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev && | |
| git config --global --add safe.directory $(realpath .) && | |
| env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs && | |
| make -j all && | |
| make -C example ndpiSimpleIntegration && | |
| make -C rrdtool && | |
| make check VERBOSE=1 | |
| " | |
| - name: Display qemu specified architecture (armhf - little endian) | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'armhf') | |
| uses: docker://multiarch/ubuntu-core:armhf-bionic | |
| with: | |
| args: > | |
| bash -c | |
| "uname -a && | |
| lscpu | grep Endian | |
| " | |
| - name: Configure, compile and test using qemu for the specified architecture (armhf - little endian) | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'armhf') | |
| uses: docker://multiarch/ubuntu-core:armhf-bionic | |
| with: | |
| args: > | |
| bash -c | |
| "apt-get -y update && | |
| apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev && | |
| git config --global --add safe.directory $(realpath .) && | |
| env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs && | |
| make -j all && | |
| make -C example ndpiSimpleIntegration && | |
| make -C rrdtool && | |
| make check VERBOSE=1 | |
| " | |
| - name: Display qemu specified architecture (s390x - big endian) | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 's390x') | |
| uses: docker://multiarch/ubuntu-core:s390x-bionic | |
| with: | |
| args: > | |
| bash -c | |
| "uname -a && | |
| lscpu | grep Endian | |
| " | |
| - name: Configure and compile using qemu for the specified architecture (s390x - big endian) | |
| if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 's390x') | |
| uses: docker://multiarch/ubuntu-core:s390x-bionic | |
| with: | |
| args: > | |
| bash -c | |
| "apt-get -y update && | |
| apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev && | |
| git config --global --add safe.directory $(realpath .) && | |
| env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs && | |
| make -j all && | |
| make -C example ndpiSimpleIntegration && | |
| make -C rrdtool && | |
| make check VERBOSE=1 | |
| " |