From 2fc926ad5003a24d14ca4ace36c5eb6fc2c71fbc Mon Sep 17 00:00:00 2001 From: Baptiste Jonglez Date: Fri, 22 Oct 2021 16:13:50 +0200 Subject: [PATCH 1/8] [oarsub] Use systemd-run when running processes on the frontend for deploy/cosystem jobs Currently, the user shell (for interactive jobs) or command (for passive jobs) that run on the frontend are part of the oar-node cgroup: frontend$ oarsub -t deploy -I frontend$ cat /proc/self/cgroup 0::/system.slice/oar-node.service Using systemd-run, we can create the processes inside the user-.slice cgroup of the target user: frontend$ oarsub -t deploy -I frontend$ cat /proc/self/cgroup 0::/user.slice/user-1101.slice/oar-1337.service This works both for cgroup v1 and v2 (the examples above are for cgroup v2). This is useful because we can then easily define per-user CPU/memory limits using these user slices, for instance with the following configuration: $ cat /etc/systemd/system/user-.slice.d/10-userlimits.conf [Slice] TasksMax=5000 MemoryHigh=1039534080 MemorySwapMax=0 CPUWeight=100 It would be cleaner to use machinectl instead of systemd-run: machinectl makes sure that a complete systemd session is created, while systemd-run only configures the cgroup of the process. However, machinectl is less widely available, for instance it is not installed by default on Debian 10 (the required package is systemd-container). So, let's use systemd-run. --- sources/core/qfunctions/oarsub | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/sources/core/qfunctions/oarsub b/sources/core/qfunctions/oarsub index e3307f8f..0fc56af8 100755 --- a/sources/core/qfunctions/oarsub +++ b/sources/core/qfunctions/oarsub @@ -165,6 +165,7 @@ sub connect_job($$$){ OAR::Sub::close_db_connection(); my @passinfo = getpwnam($lusr) or die("Cannot retreive system information for user $lusr\n"); my $shell=$passinfo[8]; + my $uid=$passinfo[2]; unless ((defined($xauth_path)) and (-x $xauth_path) and ($ENV{DISPLAY} =~ /^[\w.-]*:\d+(?:\.\d+)?$/)) { $ENV{DISPLAY}=""; } @@ -222,10 +223,18 @@ sub connect_job($$$){ if (defined($interactive_job_hook_file)) { $interactive_job_hook_cmd = "if test -x $interactive_job_hook_file; then oardodo $interactive_job_hook_file \$\$,\$PPID:$lusr:$job_id:$job->{name}:$job->{project}:$moldable->{moldable_walltime}; else true; fi && "; } + + my $oardodo_cmd; + if (defined($types->{cosystem}) or defined($types->{deploy}) or ($#hosts < 0)){ + $oardodo_cmd = "oardodo systemd-run -q --uid=$uid --pty --wait --collect --service-type=exec --slice=user-$uid.slice --unit=oar-$job_id /bin/bash --noprofile --norc -c '$str'"; + } else { + $oardodo_cmd = "OARDO_BECOME_USER=$job_user oardodo bash --noprofile --norc -c '$str'"; + } + if ($ENV{DISPLAY} ne ""){ - $cmd[$i] = "bash -c '${interactive_job_hook_cmd}echo \$PPID >> $oarsub_pids && ($xauth_path -q extract - \${DISPLAY/#localhost:/:} | OARDO_BECOME_USER=$lusr oardodo $xauth_path merge -) && [ \"$lusr\" != \"$job_user\" ] && OARDO_BECOME_USER=$lusr oardodo bash --noprofile --norc -c \"chmod 660 \\\$HOME/.Xauthority\" ;TTY=\$(tty) && test -e \$TTY && oardodo chown $job_user:oar \$TTY && oardodo chmod 660 \$TTY' && OARDO_BECOME_USER=$job_user oardodo bash --noprofile --norc -c '$str'";$i++; + $cmd[$i] = "bash -c '${interactive_job_hook_cmd}echo \$PPID >> $oarsub_pids && ($xauth_path -q extract - \${DISPLAY/#localhost:/:} | OARDO_BECOME_USER=$lusr oardodo $xauth_path merge -) && [ \"$lusr\" != \"$job_user\" ] && OARDO_BECOME_USER=$lusr oardodo bash --noprofile --norc -c \"chmod 660 \\\$HOME/.Xauthority\" ;TTY=\$(tty) && test -e \$TTY && oardodo chown $job_user:oar \$TTY && oardodo chmod 660 \$TTY' && $oardodo_cmd";$i++; }else{ - $cmd[$i] = "bash -c '${interactive_job_hook_cmd}echo \$PPID >> $oarsub_pids && TTY=\$(tty) && test -e \$TTY && oardodo chown $job_user:oar \$TTY && oardodo chmod 660 \$TTY' && OARDO_BECOME_USER=$job_user oardodo bash --noprofile --norc -c '$str'";$i++; + $cmd[$i] = "bash -c '${interactive_job_hook_cmd}echo \$PPID >> $oarsub_pids && TTY=\$(tty) && test -e \$TTY && oardodo chown $job_user:oar \$TTY && oardodo chmod 660 \$TTY' && $oardodo_cmd";$i++; } #essential: you become oar instead of the user From 85b4df9edabf20a221dbf7c731d4e27b5826f3ad Mon Sep 17 00:00:00 2001 From: Pierre Neyron Date: Wed, 3 Nov 2021 18:10:20 +0100 Subject: [PATCH 2/8] [oar.conf] add DEPLOY_COSYSTEM_JOB_EXEC_SYSTEM option --- sources/core/tools/oar.conf.in | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sources/core/tools/oar.conf.in b/sources/core/tools/oar.conf.in index a4fbca8b..dc5a0844 100644 --- a/sources/core/tools/oar.conf.in +++ b/sources/core/tools/oar.conf.in @@ -87,6 +87,17 @@ DEPLOY_HOSTNAME="127.0.0.1" COSYSTEM_HOSTNAME="127.0.0.1" # Configuration for module: server, user, api +# Specify what system to use for the execution on the deploy and cosystem frontends +# Possible values are: +# none: just run in a bash shell (default) +# systemd-run: use systemd-run +# machinectl: use machinectl +# systemd-run and machinectl place the job processes in systemd's slices, which +# allows for setting limits for instance. machinectl may not be available on +# some systems while systemd-run comes by default with systemd. +#DEPLOY_COSYSTEM_JOB_EXEC_SYSTEM="systemd-run" +# Configuration for module: server, user, api + # Specify the database field to use to fill the file on the first node of the # job in $OAR_NODE_FILE (default is 'network_address'). Only resources with # type=default are displayed in this file. From 03e4a33e44ab3a3228a413271c682ee4c53fbe54 Mon Sep 17 00:00:00 2001 From: Pierre Neyron Date: Thu, 4 Nov 2021 16:57:04 +0100 Subject: [PATCH 3/8] [bipbip] pass the DEPLOY_COSYSTEM_JOB_EXEC_SYSTEM option to oarexec --- sources/core/modules/runner/bipbip.in | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sources/core/modules/runner/bipbip.in b/sources/core/modules/runner/bipbip.in index 43615882..bd92cd70 100755 --- a/sources/core/modules/runner/bipbip.in +++ b/sources/core/modules/runner/bipbip.in @@ -382,11 +382,14 @@ my $epilogue_exec_file; if (is_conf("EPILOGUE_EXEC_FILE")){ $epilogue_exec_file = get_conf("EPILOGUE_EXEC_FILE"); } - my $passive_job_hook_exec_file; if (is_conf("PASSIVE_JOB_HOOK_EXEC_FILE")){ $passive_job_hook_exec_file = get_conf("PASSIVE_JOB_HOOK_EXEC_FILE"); } +my $deploy_cosystem_job_exec_system; +if (is_conf("DEPLOY_COSYSTEM_JOB_EXEC_SYSTEM")){ + $deploy_cosystem_job_exec_system = get_conf("DEPLOY_COSYSTEM_JOB_EXEC_SYSTEM"); +} my @oarexec_files = ($OAR_Tools_module_filepath,"$ENV{OARDIR}/oarexec"); my $host_to_connect_via_ssh = $hosts[0]; @@ -436,6 +439,7 @@ my %data_to_transfer = ( prologue => $prologue_exec_file, epilogue => $epilogue_exec_file, passive_job_hook => $passive_job_hook_exec_file, + deploy_cosystem_job_exec_system => $deploy_cosystem_job_exec_system, tmp_directory => OAR::Tools::get_default_oarexec_directory, detach_oarexec => $Detach_oarexec, cpuset_full_path => $oarexec_cpuset_path From 2ce05337ed74188d74d980724c17ef9c0fbe3313 Mon Sep 17 00:00:00 2001 From: Pierre Neyron Date: Wed, 3 Nov 2021 18:10:46 +0100 Subject: [PATCH 4/8] [oarexec] enable the use of systemd-run or machinectl for running deploy/cosystem jobs --- sources/core/modules/runner/oarexec | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sources/core/modules/runner/oarexec b/sources/core/modules/runner/oarexec index a170835d..b60ca890 100755 --- a/sources/core/modules/runner/oarexec +++ b/sources/core/modules/runner/oarexec @@ -305,6 +305,13 @@ if ( $Job->{mode} eq "PASSIVE" ){ my $str; ($str, $stdin_script_to_send) = OAR::Tools::get_oarexecuser_perl_script_for_oarexec($Node_file,$Job_id,$Job->{array_id},$Job->{array_index},$Job->{user},$shell,$Job->{launching_directory},$Job->{stdout_file},$Job->{stderr_file},$Res_file,$Job->{name},$Job->{project},$Job->{walltime},$Job->{walltime_seconds},$Job->{job_env},$Job->{command}); @cmd = ("oardodo","perl","-e",$str); + if ((defined($Job->{types}->{deploy}) or defined($Job->{types}->{cosystem})) and defined($Job->{deploy_cosystem_job_exec_system})) { + if ($Job->{deploy_cosystem_job_exec_system} eq "systemd-run") { + @cmd = ("oardodo","systemd-run","perl","-e",$str); + } elsif ($Job->{deploy_cosystem_job_exec_system} eq "machinectl") { + @cmd = ("oardodo","machinectl","perl","-e",$str); + } + } } #resolve terminal type problems From b2d42200699a6c71659bcc0ba4d62ef34bb71b20 Mon Sep 17 00:00:00 2001 From: Pierre Neyron Date: Thu, 4 Nov 2021 16:55:30 +0100 Subject: [PATCH 5/8] [oarsub] enable the use of systemd-run or machinectl for running interactive deploy/cosystem jobs --- sources/core/qfunctions/oarsub | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/sources/core/qfunctions/oarsub b/sources/core/qfunctions/oarsub index 0fc56af8..b01c4bd3 100755 --- a/sources/core/qfunctions/oarsub +++ b/sources/core/qfunctions/oarsub @@ -224,11 +224,14 @@ sub connect_job($$$){ $interactive_job_hook_cmd = "if test -x $interactive_job_hook_file; then oardodo $interactive_job_hook_file \$\$,\$PPID:$lusr:$job_id:$job->{name}:$job->{project}:$moldable->{moldable_walltime}; else true; fi && "; } - my $oardodo_cmd; - if (defined($types->{cosystem}) or defined($types->{deploy}) or ($#hosts < 0)){ - $oardodo_cmd = "oardodo systemd-run -q --uid=$uid --pty --wait --collect --service-type=exec --slice=user-$uid.slice --unit=oar-$job_id /bin/bash --noprofile --norc -c '$str'"; - } else { - $oardodo_cmd = "OARDO_BECOME_USER=$job_user oardodo bash --noprofile --norc -c '$str'"; + my $oardodo_cmd = "OARDO_BECOME_USER=$job_user oardodo bash --noprofile --norc -c '$str'"; + my $deploy_cosystem_job_exec_system = get_conf("DEPLOY_COSYSTEM_JOB_EXEC_SYSTEM"); + if ((defined($types->{cosystem}) or defined($types->{deploy}) or ($#hosts < 0)) and defined($deploy_cosystem_job_exec_system)) { + if ($deploy_cosystem_job_exec_system eq "systemd-run") { + $oardodo_cmd = "oardodo systemd-run -q --uid=$uid --pty --wait --collect --service-type=exec --slice=user-$uid.slice --unit=oar-$job_id /bin/bash --noprofile --norc -c '$str'"; + } elsif ($deploy_cosystem_job_exec_system eq "machinectl") { + $oardodo_cmd = "oardodo machinectl TODO /bin/bash --noprofile --norc -c '$str'"; + } } if ($ENV{DISPLAY} ne ""){ From 12e11581be81e9cf478c3ee988aedaa7eba959e4 Mon Sep 17 00:00:00 2001 From: Baptiste Jonglez Date: Thu, 4 Nov 2021 19:11:57 +0100 Subject: [PATCH 6/8] Add missing machinectl/systemd-run commands All working in vagrant-ci except machinectl in oarexec... --- sources/core/modules/runner/oarexec | 10 +++++++--- sources/core/qfunctions/oarsub | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/sources/core/modules/runner/oarexec b/sources/core/modules/runner/oarexec index b60ca890..04d93b8a 100755 --- a/sources/core/modules/runner/oarexec +++ b/sources/core/modules/runner/oarexec @@ -298,18 +298,22 @@ if (!defined($shell)){ print("[oarexec $Job_id] error: user $Job->{job_user} does not exist on this node, $host\n"); exit(7); } +my $uid = $pass_info[2]; my @cmd; my $stdin_script_to_send; +my $oardo_become_user = $Job->{job_user}; if ( $Job->{mode} eq "PASSIVE" ){ my $str; ($str, $stdin_script_to_send) = OAR::Tools::get_oarexecuser_perl_script_for_oarexec($Node_file,$Job_id,$Job->{array_id},$Job->{array_index},$Job->{user},$shell,$Job->{launching_directory},$Job->{stdout_file},$Job->{stderr_file},$Res_file,$Job->{name},$Job->{project},$Job->{walltime},$Job->{walltime_seconds},$Job->{job_env},$Job->{command}); @cmd = ("oardodo","perl","-e",$str); if ((defined($Job->{types}->{deploy}) or defined($Job->{types}->{cosystem})) and defined($Job->{deploy_cosystem_job_exec_system})) { if ($Job->{deploy_cosystem_job_exec_system} eq "systemd-run") { - @cmd = ("oardodo","systemd-run","perl","-e",$str); + $oardo_become_user = ""; + @cmd = ("oardodo","systemd-run","-q","--uid=$uid","--pty","--wait","--collect","--service-type=exec","--slice=user-$uid.slice","perl","-e",$str); } elsif ($Job->{deploy_cosystem_job_exec_system} eq "machinectl") { - @cmd = ("oardodo","machinectl","perl","-e",$str); + $oardo_become_user = ""; + @cmd = ("oardodo","machinectl","-q","--uid=$uid","shell",".host","perl","-e",$str); } } } @@ -470,7 +474,7 @@ if ($Job->{mode} eq "PASSIVE"){ close(STDIN); open(STDIN, "<& pipe_stdin_read"); - $ENV{OARDO_BECOME_USER} = $Job->{job_user}; + $ENV{OARDO_BECOME_USER} = $oardo_become_user; umask(oct($Old_umask)); exec(@cmd); warn("[ERROR] Cannot find @cmd\n"); diff --git a/sources/core/qfunctions/oarsub b/sources/core/qfunctions/oarsub index b01c4bd3..8046cb51 100755 --- a/sources/core/qfunctions/oarsub +++ b/sources/core/qfunctions/oarsub @@ -228,9 +228,9 @@ sub connect_job($$$){ my $deploy_cosystem_job_exec_system = get_conf("DEPLOY_COSYSTEM_JOB_EXEC_SYSTEM"); if ((defined($types->{cosystem}) or defined($types->{deploy}) or ($#hosts < 0)) and defined($deploy_cosystem_job_exec_system)) { if ($deploy_cosystem_job_exec_system eq "systemd-run") { - $oardodo_cmd = "oardodo systemd-run -q --uid=$uid --pty --wait --collect --service-type=exec --slice=user-$uid.slice --unit=oar-$job_id /bin/bash --noprofile --norc -c '$str'"; + $oardodo_cmd = "oardodo systemd-run -q --uid=$uid --pty --wait --collect --service-type=exec --slice=user-$uid.slice /bin/bash --noprofile --norc -c '$str'"; } elsif ($deploy_cosystem_job_exec_system eq "machinectl") { - $oardodo_cmd = "oardodo machinectl TODO /bin/bash --noprofile --norc -c '$str'"; + $oardodo_cmd = "oardodo machinectl -q --uid=$uid shell .host /bin/bash --noprofile --norc -c '$str'"; } } From 4d948e26706add6d126dad2058cdc6a4e30e5765 Mon Sep 17 00:00:00 2001 From: Baptiste Jonglez Date: Fri, 5 Nov 2021 16:56:28 +0100 Subject: [PATCH 7/8] Remove support for machinectl (does not work for non-interactive execution in oarexec) --- sources/core/modules/runner/oarexec | 7 +++---- sources/core/qfunctions/oarsub | 4 ++-- sources/core/tools/oar.conf.in | 6 ++---- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/sources/core/modules/runner/oarexec b/sources/core/modules/runner/oarexec index 04d93b8a..5b90db49 100755 --- a/sources/core/modules/runner/oarexec +++ b/sources/core/modules/runner/oarexec @@ -310,10 +310,9 @@ if ( $Job->{mode} eq "PASSIVE" ){ if ((defined($Job->{types}->{deploy}) or defined($Job->{types}->{cosystem})) and defined($Job->{deploy_cosystem_job_exec_system})) { if ($Job->{deploy_cosystem_job_exec_system} eq "systemd-run") { $oardo_become_user = ""; - @cmd = ("oardodo","systemd-run","-q","--uid=$uid","--pty","--wait","--collect","--service-type=exec","--slice=user-$uid.slice","perl","-e",$str); - } elsif ($Job->{deploy_cosystem_job_exec_system} eq "machinectl") { - $oardo_become_user = ""; - @cmd = ("oardodo","machinectl","-q","--uid=$uid","shell",".host","perl","-e",$str); + @cmd = ("oardodo","systemd-run","-q","--uid=$uid","--pipe","--wait","--collect","--service-type=exec","--slice=user-$uid.slice","perl","-e",$str); + } elsif ($Job->{deploy_cosystem_job_exec_system} eq "none") { + # Use regular command by default } } } diff --git a/sources/core/qfunctions/oarsub b/sources/core/qfunctions/oarsub index 8046cb51..806f3a68 100755 --- a/sources/core/qfunctions/oarsub +++ b/sources/core/qfunctions/oarsub @@ -229,8 +229,8 @@ sub connect_job($$$){ if ((defined($types->{cosystem}) or defined($types->{deploy}) or ($#hosts < 0)) and defined($deploy_cosystem_job_exec_system)) { if ($deploy_cosystem_job_exec_system eq "systemd-run") { $oardodo_cmd = "oardodo systemd-run -q --uid=$uid --pty --wait --collect --service-type=exec --slice=user-$uid.slice /bin/bash --noprofile --norc -c '$str'"; - } elsif ($deploy_cosystem_job_exec_system eq "machinectl") { - $oardodo_cmd = "oardodo machinectl -q --uid=$uid shell .host /bin/bash --noprofile --norc -c '$str'"; + } elsif ($deploy_cosystem_job_exec_system eq "none") { + # Use regular command by default } } diff --git a/sources/core/tools/oar.conf.in b/sources/core/tools/oar.conf.in index dc5a0844..c50d0de6 100644 --- a/sources/core/tools/oar.conf.in +++ b/sources/core/tools/oar.conf.in @@ -91,10 +91,8 @@ COSYSTEM_HOSTNAME="127.0.0.1" # Possible values are: # none: just run in a bash shell (default) # systemd-run: use systemd-run -# machinectl: use machinectl -# systemd-run and machinectl place the job processes in systemd's slices, which -# allows for setting limits for instance. machinectl may not be available on -# some systems while systemd-run comes by default with systemd. +# systemd-run executes the job processes in user slices managed by systemd, which +# could easily allow to set resource limits through cgroups. #DEPLOY_COSYSTEM_JOB_EXEC_SYSTEM="systemd-run" # Configuration for module: server, user, api From 0332e167a8027ffc0806ed7fc21653a1c6f7593f Mon Sep 17 00:00:00 2001 From: Baptiste Jonglez Date: Fri, 5 Nov 2021 17:00:08 +0100 Subject: [PATCH 8/8] Check for validity of DEPLOY_COSYSTEM_JOB_EXEC_SYSTEM --- sources/core/modules/runner/bipbip.in | 6 +++++- sources/core/qfunctions/oarsub | 16 ++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/sources/core/modules/runner/bipbip.in b/sources/core/modules/runner/bipbip.in index bd92cd70..46dd1e1f 100755 --- a/sources/core/modules/runner/bipbip.in +++ b/sources/core/modules/runner/bipbip.in @@ -388,7 +388,11 @@ if (is_conf("PASSIVE_JOB_HOOK_EXEC_FILE")){ } my $deploy_cosystem_job_exec_system; if (is_conf("DEPLOY_COSYSTEM_JOB_EXEC_SYSTEM")){ - $deploy_cosystem_job_exec_system = get_conf("DEPLOY_COSYSTEM_JOB_EXEC_SYSTEM"); + $deploy_cosystem_job_exec_system = get_conf("DEPLOY_COSYSTEM_JOB_EXEC_SYSTEM"); + if (($deploy_cosystem_job_exec_system ne "none") and ($deploy_cosystem_job_exec_system ne "systemd-run")) { + oar_error("[bipbip $Job_id] Invalid configuration for DEPLOY_COSYSTEM_JOB_EXEC_SYSTEM: '$deploy_cosystem_job_exec_system' is unsupported\n"); + # Don't exit, because it causes the job to be stuck in 'Launching' state... + } } my @oarexec_files = ($OAR_Tools_module_filepath,"$ENV{OARDIR}/oarexec"); diff --git a/sources/core/qfunctions/oarsub b/sources/core/qfunctions/oarsub index 806f3a68..90d4b1ad 100755 --- a/sources/core/qfunctions/oarsub +++ b/sources/core/qfunctions/oarsub @@ -52,6 +52,7 @@ my $remote_host; my $remote_port; my $Deploy_hostname; my $Cosystem_hostname; +my $Deploy_cosystem_job_exec_system; my $Cpuset_field; my $Cpuset_path; my $Host = hostname ; @@ -225,11 +226,10 @@ sub connect_job($$$){ } my $oardodo_cmd = "OARDO_BECOME_USER=$job_user oardodo bash --noprofile --norc -c '$str'"; - my $deploy_cosystem_job_exec_system = get_conf("DEPLOY_COSYSTEM_JOB_EXEC_SYSTEM"); - if ((defined($types->{cosystem}) or defined($types->{deploy}) or ($#hosts < 0)) and defined($deploy_cosystem_job_exec_system)) { - if ($deploy_cosystem_job_exec_system eq "systemd-run") { + if (defined($types->{cosystem}) or defined($types->{deploy}) or ($#hosts < 0)) { + if ($Deploy_cosystem_job_exec_system eq "systemd-run") { $oardodo_cmd = "oardodo systemd-run -q --uid=$uid --pty --wait --collect --service-type=exec --slice=user-$uid.slice /bin/bash --noprofile --norc -c '$str'"; - } elsif ($deploy_cosystem_job_exec_system eq "none") { + } elsif ($Deploy_cosystem_job_exec_system eq "none") { # Use regular command by default } } @@ -565,6 +565,14 @@ if (!defined($Cosystem_hostname)){ $Cosystem_hostname = $remote_host; } +$Deploy_cosystem_job_exec_system = get_conf("DEPLOY_COSYSTEM_JOB_EXEC_SYSTEM"); +if (!defined($Deploy_cosystem_job_exec_system)) { + $Deploy_cosystem_job_exec_system = "none"; +} +if (($Deploy_cosystem_job_exec_system ne "none") and ($Deploy_cosystem_job_exec_system ne "systemd-run")) { + die("[ERROR] Bad configuration for DEPLOY_COSYSTEM_JOB_EXEC_SYSTEM: '$Deploy_cosystem_job_exec_system' is unsupported\n"); +} + $Cpuset_field = get_conf("JOB_RESOURCE_MANAGER_PROPERTY_DB_FIELD"); $Cpuset_path = get_conf("CPUSET_PATH");