Add Python bindings for libnetplan

.github/workflows/autopkgtest.yml

@@ -42,6 +42,7 @@ jobs:
           pull-lp-source netplan.io
           cp -r netplan.io-*/debian .
           rm -r debian/patches/  # clear any distro patches
+          echo "usr/lib/python3/dist-packages/netplan/*" >> debian/netplan.io.install  # bindings
           TAG=$(git describe --tags $(git rev-list --tags --max-count=1))  # find latest (stable) tag
           REV=$(git rev-parse --short HEAD)  # get current git revision
           VER="$TAG+git~$REV"
@@ -50,4 +51,4 @@ jobs:
         run: |
           # using --setup-commands temporarily to install:
           # cmocka/pytest/rich/ethtool until they become proper test-deps
-          autopkgtest . --setup-commands='apt -y install ethtool python3-rich python3-pytest python3-pytest-cov libcmocka-dev' -U --env=DPKG_GENSYMBOLS_CHECK_LEVEL=0 --env=DEB_BUILD_OPTIONS=nocheck -- lxd autopkgtest/ubuntu/jammy/amd64
+          autopkgtest . --setup-commands='apt -y install ethtool python3-rich python3-pytest python3-pytest-cov python3-cffi libpython3-dev libcmocka-dev' -U --env=DPKG_GENSYMBOLS_CHECK_LEVEL=0 --env=DEB_BUILD_OPTIONS=nocheck -- lxd autopkgtest/ubuntu/jammy/amd64

.github/workflows/build-abi.yml

@@ -28,7 +28,7 @@ jobs:
           sudo sed -i '/deb-src/s/^# //' /etc/apt/sources.list
           sudo apt update
           #sudo apt install lcov python3-coverage curl
-          sudo apt install abigail-tools meson python3-coverage python3-pytest python3-pytest-cov
+          sudo apt install abigail-tools meson python3-coverage python3-pytest python3-pytest-cov python3-cffi libpython3-dev
           sudo apt build-dep netplan.io
 
       # Runs the build

.github/workflows/check-address-sanitizer.yml

@@ -23,7 +23,7 @@ jobs:
           echo "APT::Get::Always-Include-Phased-Updates \"true\";" | sudo tee /etc/apt/apt.conf.d/90phased-updates
           sudo sed -i '/deb-src/s/^# //' /etc/apt/sources.list
           sudo apt update
-          sudo apt -y install python3-rich python3-coverage python3-pytest python3-pytest-cov curl meson gcovr expect libcmocka-dev
+          sudo apt -y install python3-rich python3-coverage python3-pytest python3-pytest-cov curl meson gcovr expect libcmocka-dev python3-cffi libpython3-dev
           sudo apt -y build-dep netplan.io
 
       - name: Run unit tests

.github/workflows/check-coverage.yml

@@ -28,15 +28,17 @@ jobs:
           echo "APT::Get::Always-Include-Phased-Updates \"true\";" | sudo tee /etc/apt/apt.conf.d/90phased-updates
           sudo sed -i '/deb-src/s/^# //' /etc/apt/sources.list
           sudo apt update
-          sudo apt install python3-rich python3-coverage python3-pytest python3-pytest-cov curl meson gcovr expect libcmocka-dev
+          sudo apt install python3-rich python3-coverage python3-pytest python3-pytest-cov curl meson gcovr expect libcmocka-dev python3-cffi libpython3-dev
           sudo apt build-dep netplan.io
+          wget http://archive.ubuntu.com/ubuntu/pool/universe/g/gcovr/gcovr_5.2-1_all.deb
+          sudo dpkg -i gcovr*.deb # we need newer gcovr to make the gcovr.cfg:exclude setting work
 
       # Runs the unit tests with coverage
       - name: Run unit tests
         run: |
-          meson setup _build --prefix=/usr -Db_coverage=true -Dunit_testing=true
-          meson compile -C _build
-          unbuffer meson test -C _build --verbose
+          meson setup _build-cov --prefix=/usr -Db_coverage=true -Dunit_testing=true
+          meson compile -C _build-cov
+          unbuffer meson test -C _build-cov --verbose
 
       # Checks the coverage diff to the main branch
       #- name: Upload coverage to Codecov

.github/workflows/codeql-analysis.yml

@@ -52,7 +52,7 @@ jobs:
       run: |
         sudo sed -i '/deb-src/s/^# //' /etc/apt/sources.list
         sudo apt update
-        sudo apt install meson python3-coverage python3-pytest python3-pytest-cov libcmocka-dev
+        sudo apt install meson python3-coverage python3-pytest python3-pytest-cov libcmocka-dev python3-cffi libpython3-dev
         sudo apt build-dep netplan.io
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).

.github/workflows/coverity.yml

@@ -16,7 +16,7 @@ jobs:
           sudo sed -i '/deb-src/s/^# //' /etc/apt/sources.list
           sudo apt update
           sudo apt -y build-dep netplan.io
-          sudo apt -y install libcmocka-dev meson python3-pytest curl
+          sudo apt -y install libcmocka-dev meson python3-pytest curl python3-cffi libpython3-dev
       - name: Download Coverity
         run: |
           curl https://scan.coverity.com/download/cxx/linux64 --no-progress-meter --output ${HOME}/coverity.tar.gz --data "token=${{ secrets.COVERITY_TOKEN }}&project=Netplan"

.github/workflows/debci.yml

@@ -46,6 +46,9 @@ jobs:
           dget -u "https://deb.debian.org/debian/pool/main/n/netplan.io/netplan.io_$V.dsc"
           cp -r netplan.io-*/debian .
           rm -r debian/patches/  # clear any distro patches
+          echo "usr/lib/python3/dist-packages/netplan/*" >> debian/netplan.io.install  # bindings
+          echo "override_dh_auto_configure:" >> debian/rules
+          echo "	dh_auto_configure -- -Dpython.purelibdir=/usr/lib/python3/dist-packages -Dpython.platlibdir=/usr/lib/python3/dist-packages" >> debian/rules
           TAG=$(git describe --tags $(git rev-list --tags --max-count=1))  # find latest (stable) tag
           REV=$(git rev-parse --short HEAD)  # get current git revision
           VER="$TAG+git~$REV"
@@ -54,4 +57,4 @@ jobs:
         run: |
           # using --setup-commands='apt -y install ...' temporarily to install
           # (test-/build-) deps until they become part of the packaging
-          sudo autopkgtest . -U --env=DPKG_GENSYMBOLS_CHECK_LEVEL=0 --env=DEB_BUILD_OPTIONS=nocheck -- lxc autopkgtest-testing-amd64 || test $? -eq 2  # allow OVS test to be skipped (exit code = 2)
+          sudo autopkgtest . -U --env=DPKG_GENSYMBOLS_CHECK_LEVEL=0 --env=DEB_BUILD_OPTIONS=nocheck --setup-commands='apt -y install python3-cffi libpython3-dev' -- lxc autopkgtest-testing-amd64 || test $? -eq 2  # allow OVS test to be skipped (exit code = 2)

.github/workflows/network-manager.yml

@@ -17,7 +17,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       # Setup LXD + Docker fixes
       - uses: canonical/[email protected]
         with:
@@ -32,24 +32,35 @@ jobs:
           sudo sed -i '/deb-src/s/^# //' /etc/apt/sources.list
           sudo apt update
           sudo apt install autopkgtest ubuntu-dev-tools devscripts openvswitch-switch linux-modules-extra-$(uname -r)
-      # work around LP: #1878225 as fallback
-      - name: Preparing autopkgtest-build-lxd
-        run: |
-          sudo patch /usr/bin/autopkgtest-build-lxd .github/workflows/snapd.patch
-          autopkgtest-build-lxd ubuntu-daily:mantic
       - name: Prepare test
         run: |
           pull-lp-source netplan.io
           cp -r netplan.io-*/debian .
           rm -r debian/patches/  # clear any distro patches
+          echo "3.0 (native)" > debian/source/format  # force native build
+          echo "usr/lib/python3/dist-packages/netplan/*" >> debian/netplan.io.install  # bindings
+          echo "override_dh_auto_configure:" >> debian/rules
+          echo "	dh_auto_configure -- -Dpython.purelibdir=/usr/lib/python3/dist-packages -Dpython.platlibdir=/usr/lib/python3/dist-packages" >> debian/rules
           TAG=$(git describe --tags $(git rev-list --tags --max-count=1))  # find latest (stable) tag
           REV=$(git rev-parse --short HEAD)  # get current git revision
           VER="$TAG+git~$REV"
           dch -v "$VER" "Autopkgtest CI"
-          sudo apt -y build-dep ./
-          DEB_BUILD_OPTIONS=nocheck DPKG_GENSYMBOLS_CHECK_LEVEL=0 dpkg-buildpackage -b
+      # Build deb
+      - uses: jtdor/build-deb-action@v1
+        env:
+          DEB_BUILD_OPTIONS: nocheck
+          DPKG_GENSYMBOLS_CHECK_LEVEL: 0
+        with:
+          docker-image: ubuntu:mantic
+          buildpackage-opts: --build=binary --no-sign
+          extra-build-deps: python3-cffi libpython3-dev
+      # work around LP: #1878225 as fallback
+      - name: Preparing autopkgtest-build-lxd
+        run: |
+          sudo patch /usr/bin/autopkgtest-build-lxd .github/workflows/snapd.patch
+          autopkgtest-build-lxd ubuntu-daily:mantic
       - name: Run autopkgtest
         run: |
           # using --setup-commands temporarily to install:
           # cmocka/pytest/rich/ethtool until they become proper test-deps
-          autopkgtest -U ../*.deb network-manager --apt-pocket=proposed=src:network-manager -- lxd autopkgtest/ubuntu/mantic/amd64 || test $? -eq 2  # allow for skipped tests (exit code = 2)
+          sudo autopkgtest -U debian/artifacts/*.deb network-manager --apt-pocket=proposed=src:network-manager -- lxd autopkgtest/ubuntu/mantic/amd64 || test $? -eq 2  # allow for skipped tests (exit code = 2)

Makefile

@@ -26,6 +26,7 @@ clean:
 	rm -rf _build-cov
 	rm -rf _leakcheckbuild
 	rm -rf tmproot
+	rm -f python-cffi/netplan/_netplan_cffi.*
 
 check: default
 	meson test -C _build --verbose
@@ -38,7 +39,7 @@ pre-coverage: _build-cov
 	meson compile -C _build-cov --verbose
 
 check-coverage: pre-coverage
-	meson test -C _build-cov
+	meson test -C _build-cov --verbose
 
 install: default
 	meson install -C _build --destdir $(DESTDIR)

examples/cffi-bindings.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2023 Canonical, Ltd.
+# Author: Lukas Märdian <[email protected]>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import io
+import tempfile
+from netplan import Parser, State, _create_yaml_patch
+from netplan import NetplanException, NetplanParserException
+FALLBACK_FILENAME = '70-netplan-set.yaml'
+
+# This script is a demo/example, making use of Netplan's CFFI Python bindings.
+# It does process your local /etc/netplan/ hierarchy, so be careful using it.
+# At first, it creates a Parser() object and a YAML patch, setting a
+# "network.ethernets.eth99.dhcp4=true" value. It loads any existing Netplan
+# YAML hierarcy from /etc/netplan/ and loads/applies the above mentioned patch
+# on top of it. Afterwards, it creates a State() object, importing parsed data
+# for validation and checks for any errors.
+# On succesful validation, it walks through all NetDefs in the validated
+# Netplan state and prints the Netplan ID and backend renderer of given NetDef.
+# Finally, it writes the validated state (including the eth99.dhcp4 setting)
+# back to disk in /etc/netplan/.
+if __name__ == '__main__':
+    yaml_path = ['network', 'ethernets', 'eth99', 'dhcp4']
+    value = 'true'
+
+    parser = Parser()
+    with tempfile.TemporaryFile() as tmp:
+        _create_yaml_patch(yaml_path, value, tmp)
+        tmp.flush()
+
+        # Parse the full, existing YAML config hierarchy
+        parser.load_yaml_hierarchy(rootdir='/')
+
+        # Load YAML patch, containing our new settings
+        tmp.seek(0, io.SEEK_SET)
+        parser.load_yaml(tmp)
+
+        # Validate the final parser state
+        state = State()
+        try:
+            # validation of current state + new settings
+            state.import_parser_results(parser)
+        except NetplanParserException as e:
+            print('Error in', e.filename, 'Row/Col', e.line, e.column, '->', e.message)
+        except NetplanException as e:
+            print('Error:', e.message)
+
+        # Walk through all NetdefIDs in the state and print their backend
+        # renderer, to demonstrate working with NetDefinitionIterator &
+        # NetDefinition
+        for netdef_id, netdef in state.netdefs.items():
+            print('Netdef', netdef_id, 'is managed by:', netdef.backend)
+
+        # Write the new data from the YAML patch to disk, updating an
+        # existing Netdef, if file already exists, or FALLBACK_FILENAME
+        state._update_yaml_hierarchy(FALLBACK_FILENAME, rootdir='/')

gcovr.cfg

@@ -0,0 +1,2 @@
+filter = src/*
+filter = tests/ctests/*

include/parse-nm.h

@@ -23,6 +23,10 @@
 NETPLAN_PUBLIC gboolean
 netplan_parser_load_keyfile(NetplanParser* npp, const char* filename, NetplanError** error);
 
+//TODO: needs to be implemented
+//NETPLAN_PUBLIC gboolean
+//netplan_parser_load_keyfile_from_fd(NetplanParser* npp, int input_fd, NetplanError** error);
+
 /********** Old API below this ***********/
 
 NETPLAN_PUBLIC gboolean

include/parse.h

@@ -38,6 +38,9 @@ netplan_parser_load_yaml(NetplanParser* npp, const char* filename, NetplanError*
 NETPLAN_PUBLIC gboolean
 netplan_parser_load_yaml_from_fd(NetplanParser* npp, int input_fd, NetplanError** error);
 
+NETPLAN_PUBLIC gboolean
+netplan_parser_load_yaml_hierarchy(NetplanParser* npp, const char* rootdir, NetplanError** error);
+
 NETPLAN_PUBLIC gboolean
 netplan_parser_load_nullable_fields(NetplanParser* npp, int input_fd, NetplanError** error);
 
@@ -51,7 +54,7 @@ netplan_state_import_parser_results(NetplanState* np_state, NetplanParser* npp,
  * <constraint> only. */
 NETPLAN_PUBLIC gboolean
 netplan_parser_load_nullable_overrides(
-    NetplanParser* npp, int input_fd, const char* constraint, GError** error);
+    NetplanParser* npp, int input_fd, const char* constraint, NetplanError** error);
 
 /********** Old API below this ***********/
 

include/types.h

@@ -87,7 +87,7 @@ struct _NetplanStateIterator {
  * Errors and error domains
  *
  * NOTE: if new errors or domains are added,
- * netplan/libnetplan.py must be updated with the new entries.
+ * python-cffi/netplan/_utils.py must be updated with the new entries.
  */
 
 enum NETPLAN_ERROR_DOMAINS {

include/util.h

@@ -54,6 +54,12 @@ netplan_state_iterator_next(NetplanStateIterator* iter);
 NETPLAN_PUBLIC gboolean
 netplan_state_iterator_has_next(const NetplanStateIterator* iter);
 
+NETPLAN_PUBLIC gboolean
+netplan_util_create_yaml_patch(const char* conf_obj_path, const char* obj_payload, int out_fd, NetplanError** error);
+
+NETPLAN_PUBLIC gboolean
+netplan_util_dump_yaml_subtree(const char* prefix, int input_fd, int output_fd, NetplanError** error);
+
 /********** Old API below this ***********/
 
 NETPLAN_DEPRECATED NETPLAN_PUBLIC gchar*

meson.build

@@ -36,10 +36,12 @@ add_project_arguments(
     language: 'c')
 
 inc = include_directories('include')
+inc_internal = include_directories('src')
 subdir('include')
 subdir('src')
 subdir('dbus')
 subdir('netplan_cli')
+subdir('python-cffi')
 subdir('examples')
 subdir('doc')
 
@@ -60,7 +62,7 @@ install_data(
 # Testing #
 ###########
 test_env = [
-    'PYTHONPATH=' + meson.current_source_dir(),
+    'PYTHONPATH=' + join_paths(meson.current_build_dir(), 'python-cffi') + ':' + meson.current_source_dir(),
     'LD_LIBRARY_PATH=' + join_paths(meson.current_build_dir(), 'src'),
     'NETPLAN_GENERATE_PATH=' + join_paths(meson.current_build_dir(), 'src', 'generate'),
     'NETPLAN_DBUS_CMD=' + join_paths(meson.current_build_dir(), 'dbus', 'netplan-dbus'),
@@ -107,12 +109,18 @@ if get_option('b_coverage')
     gcovr = find_program('gcovr')
     ninja = find_program('ninja')
     grep  = find_program('grep')
+    cat  = find_program('cat')
     test('coverage-c-output',
          find_program('ninja'),
          args: ['-C', meson.current_build_dir(), 'coverage'],
          timeout: 60,
          priority: -90, # run before 'coverage-c'
          is_parallel: false)
+    test('coverage-c-cat',
+         cat,
+         args: [join_paths(meson.current_build_dir(), 'meson-logs', 'coverage.txt')],
+         priority: -98, # run before 'coverage-c'
+         is_parallel: false)
     test('coverage-c',
          grep,
          args: ['^TOTAL.*100%$', join_paths(meson.current_build_dir(), 'meson-logs', 'coverage.txt')],

netplan_cli/cli/commands/apply.py

@@ -368,7 +368,7 @@ def process_link_changes(interfaces, config_manager: ConfigManager):  # pragma:
             newname = netdef.set_name
             if not newname:
                 continue  # Skip if no new name needs to be set
-            if not netdef.has_match:
+            if not netdef._has_match:
                 continue  # Skip if no match for current name is given
             if NetplanApply.is_composite_member(composite_interfaces, netdef.id):
                 logging.debug('Skipping composite member {}'.format(netdef.id))

netplan_cli/cli/commands/set.py

@@ -22,7 +22,7 @@
 import io
 
 from ..utils import NetplanCommand
-from ... import libnetplan
+import netplan
 
 FALLBACK_FILENAME = '70-netplan-set.yaml'
 GLOBAL_KEYS = ['renderer', 'version']
@@ -67,9 +67,9 @@ def command_set(self):
         # Split the string into a list on the dot separators, and unescape the remaining dots
         yaml_path = [s.replace(r'\.', '.') for s in re.split(r'(?<!\\)\.', key)]
 
-        parser = libnetplan.Parser()
+        parser = netplan.Parser()
         with tempfile.TemporaryFile() as tmp:
-            libnetplan.create_yaml_patch(yaml_path, value, tmp)
+            netplan._create_yaml_patch(yaml_path, value, tmp)
             tmp.flush()
 
             # Load fields that are about to be deleted (e.g. some.setting=NULL)
@@ -85,11 +85,11 @@ def command_set(self):
             parser.load_yaml(tmp)
 
             # Validate the final parser state
-            state = libnetplan.State()
+            state = netplan.State()
             state.import_parser_results(parser)
 
             if filename:  # only act on the output file (a.k.a. "origin-hint")
-                parser_output_file = libnetplan.Parser()
+                parser_output_file = netplan.Parser()
 
                 # Load fields that are about to be deleted ("some.setting=NULL")
                 # Ignore those fields when parsing subsequent YAML files
@@ -103,7 +103,7 @@ def command_set(self):
                 # (a.k.a. "origin-hint", <filename>), have they been defined in
                 # pre-existing YAML files or not.
                 tmp.seek(0, io.SEEK_SET)
-                parser_output_file.load_nullable_overrides(tmp, constraint=filename)
+                parser_output_file._load_nullable_overrides(tmp, constraint=filename)
 
                 # Parse the full YAML hierarchy and new patch, ignoring any
                 # nullable overrides (netdefs/globals) from pre-existing files
@@ -122,8 +122,8 @@ def command_set(self):
                 # Import the partial parser state, ignoring duplicated netdefs
                 # from pre-existing YAML files, so we can force write the patch
                 # contents to the output file or update this file if exists.
-                state_output_file = libnetplan.State()
+                state_output_file = netplan.State()
                 state_output_file.import_parser_results(parser_output_file)
-                state_output_file.write_yaml_file(filename, self.root_dir)
+                state_output_file._write_yaml_file(filename, self.root_dir)
             else:
-                state.update_yaml_hierarchy(FALLBACK_FILENAME, self.root_dir)
+                state._update_yaml_hierarchy(FALLBACK_FILENAME, self.root_dir)