Got error when use fork and exec to start up python executable inside gramine

[santongding]

In my workflow, I need to use fork and execv in C++ code to create a gramine instance that runs a Python script, as shown in the code below:

        int py_pid = fork();
        if (!py_pid) {
            const char *const argv[] = {"/usr/bin/python3", py-server.py", NULL};
            if (execv(argv[0], (char **)(void *)argv)) {
                printf("Fail to execv, error:%s\n", strerror(errno));
                throw;
            }
        } 

There may be multiple (about 5) gramine instances running concurrently, and occasionally (maybe ten forks) there may be errors that cause the fork to fail, resulting in the following error report:

(libos_init.c:303:init_stack) [P2:T2:python3] debug: Allocated stack at 0x2666000 (size = 0x40000)
(libos_rtld.c:844:load_elf_object) [P2:T2:python3] debug: loading "file:/usr/bin/python3"
(libos_vma.c:245:_vma_bkeep_remove) [P2:T2:python3] warning: user app tried to free an internal vma!
(libos_rtld.c:262:execute_loadcmd) [P2:T2:python3] debug: failed to bookkeep address of segment
(libos_rtld.c:477:map_elf_object) [P2:T2:python3] debug: loading file:/usr/bin/python3: failed to execute load command (-13)
(libos_rtld.c:852:load_elf_object) [P2:T2:python3] error: Failed to map file:/usr/bin/python3.
(libos_exec.c:84:__libos_syscall_execve_rtld) [P2:T2:python3] error: execve failed with: Invalid argument (EINVAL)

After diving into the source code in libos_vma.c:245:_vma_bkeep_remove, I found that the code that triggered the error is as follows:

    while (vma && vma->begin < end) {
        if (!!(vma->flags & VMA_INTERNAL) != is_internal) {
            if (is_internal) {
                log_warning("LibOS tried to free a user vma!");
            } else {
                log_warning("user app tried to free an internal vma!");
            }
            return -EACCES;
        }

        vma = _get_next_vma(vma);
    }

I wonder how this happened and how to fix it. Any more detailed information will be provided if needed. Thanks a lot.

[dimakuv]

It looks like a problem with the enclave size (I assume you're talking about gramine-sgx, not gramine-direct).

Can you try to increase the sgx.enclave_size value and check again?

Also, can you show your manifest file?

[santongding]

Yes, the gramine-sgx.

I'm running on an i7-8559U, which does not support the edmm feature. So, if the sgx.enclave_size is too large, it may cause a failed boot due to OOM.

You inspired me to think that this may be a memory leaking problem. Please note that my code is running inside a Docker container, and I'm mounting the necessary device files (/dev/isgx and /var/run/aesmd/aesm.socket). I clean up the code by simply stopping the Docker container. Could this be causing the enclave resources to not be recycled correctly?

I have reduced the sgx.enclave_size from 1G to 128M to accelerate the boot speed and improve concurrency. I set it to 256M and so far, I'm testing it with no errors.

Also, below is the manifest, which is mainly copied from gramine/CI-Examples/ra-tls-mbedtls and modified according to gramine/CI-Examples/python.

# RA-TLS manifest file example

loader.entrypoint = "file:{{ gramine.libos }}"
libos.entrypoint = "/{{ exe_name }}"

loader.log_level = "{{ log_level }}"

loader.env.LD_LIBRARY_PATH = "/usr/local/lib:/lib:/lib:{{ arch_libdir }}:/usr/{{ arch_libdir }}:/usr/local/lib"
loader.env.PYTHONPATH = "/usr/local/lib/python3.8/dist-packages"
# python example: loader.env.LD_LIBRARY_PATH = "/lib:/lib:{{ arch_libdir }}:/usr/{{ arch_libdir }}"

loader.env.OMP_NUM_THREADS = "4"

loader.insecure__use_cmdline_argv = true
# loader.insecure__use_host_env = true
loader.env.in_sgx = "1" 

sys.enable_sigterm_injection = true
sys.insecure__allow_eventfd = true

fs.insecure__keys.default = "ffeeddccbbaa00887766554433221199"



fs.mounts = [
  { path = "{{ gramine.libos }}", uri = "file:{{ gramine.libos }}"},
  { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" },
  { path = "{{ gramine.runtimedir() }}", uri = "file:{{ gramine.runtimedir() }}" },
  { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" },
  { path = "/usr{{ arch_libdir }}", uri = "file:/usr{{ arch_libdir }}" },
  { path = "/etc", uri = "file:/etc" },
  { path = "/{{ exe_name }}", uri = "file:{{ exe_name }}" },
  { path = "/usr/local/lib/libhiredis.so.1.1.0", uri = "file:/usr/local/lib/libhiredis.so.1.1.0" },

  { type = "encrypted", path = "/tmp/", uri = "file:sgx_tmp/" },

  { path = "/usr/bin/python3", uri = "file:/usr/bin/python3"},
  { path = "/usr/local/lib/x86_64-linux-gnu/gramine/sgx/libpal.so", uri = "file:/usr/local/lib/x86_64-linux-gnu/gramine/sgx/libpal.so"},
  
  { path = "/sgx_resources", uri = "file:sgx_resources"},
  


{% for path in python.get_sys_path("/usr/bin/python3") %}
  { path = "{{ path }}", uri = "file:{{ path }}" },
{% endfor %}

  { path = "/bin/sh", uri = "file:/bin/sh"},

  { path = "/usr/bin/openssl", uri = "file:/usr/bin/openssl"}, 


]

sgx.debug = true
sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }}
sgx.enclave_size = "128M"
sgx.max_threads = 32

sgx.remote_attestation = "{{ ra_type }}"
sgx.ra_client_spid = "{{ ra_client_spid }}"
sgx.ra_client_linkable = {{ 'true' if ra_client_linkable == '1' else 'false' }}

sgx.nonpie_binary = true

sgx.trusted_files = [
  "file:{{ gramine.libos }}",
  "file:{{ exe_name }}",
  "file:{{ gramine.runtimedir() }}/",
  "file:{{ arch_libdir }}/",
  "file:/usr{{ arch_libdir }}/",
  "file:/usr/local/lib/libhiredis.so.1.1.0",
  "file:/usr/local/lib/x86_64-linux-gnu/gramine/sgx/libpal.so",
  "file:/usr/bin/python3",
{% for path in python.get_sys_path("/usr/bin/python3") %}
  "file:{{ path }}{{ '/' if path.is_dir() else '' }}",
{% endfor %}

  "file:/bin/sh",

  "file:/usr/bin/openssl",

  "file:sgx_resources/",
]

sgx.allowed_files = [
  "file:/etc/nsswitch.conf",
  "file:/etc/ethers",
  "file:/etc/hosts",
  "file:/etc/group",
  "file:/etc/passwd",
  "file:/etc/gai.conf",
  "file:/etc/mime.types",
  "file:/etc/resolv.conf",
]

[dimakuv]

I set it to 256M and so far, I'm testing it with no errors.

So looks like it's working now? And the problem was in the OOM killer in the host Linux (which killed some Gramine-SGX enclaves because you don't have too much system memory)?

[santongding]

Although its functionality works, I don't think it's an OOM problem. It generates errors when 'sgx.enclave_size' is set low, but runs properly when it is higher. I believe the memory allocation for the Python code is insufficient, but I don't understand why it crashes randomly.

[dimakuv]

I believe the memory allocation for the Python code is insufficient, but I don't understand why it crashes randomly.

I think the randomness is because ASLR (Address Space Layout Randomization) is enabled in Gramine by default. Try loader.insecure__disable_aslr = false, and you'll see more consistent behavior.

[santongding]

Try loader.insecure__disable_aslr = false

That's the solution! My programs remain stable even when sgx.enclave_size is set to a low value. Thanks a lot.