From 082d1e6ea47d359406a34ce37889cfe72891061f Mon Sep 17 00:00:00 2001 From: xunpan Date: Tue, 2 Nov 2021 22:25:14 -0400 Subject: [PATCH] lsf criu scripts --- README.md | 3 ++ criu/README.md | 41 +++++++++++++++++++++ criu/echkpnt.criu | 76 +++++++++++++++++++++++++++++++++++++++ criu/erestart.criu | 89 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 209 insertions(+) create mode 100644 criu/README.md create mode 100644 criu/echkpnt.criu create mode 100644 criu/erestart.criu diff --git a/README.md b/README.md index cc4a4c1..e6c3a3d 100644 --- a/README.md +++ b/README.md @@ -5,3 +5,6 @@ A setuid tool to submit LSF jobs on behalf of other users. - [LSF chatops](https://github.com/IBMSpectrumComputing/lsf-utils/tree/master/chatops/errbot) An `Errbot` plugin to help you talking with your `LSF` cluster by `Slack` from anywhere. + +- [LSF checkpoint with CRIU](https://github.com/IBMSpectrumComputing/lsf-utils/tree/master/criu) +Scripts to use `CRIU` for LSF checkpoint/restart job diff --git a/criu/README.md b/criu/README.md new file mode 100644 index 0000000..a43cac7 --- /dev/null +++ b/criu/README.md @@ -0,0 +1,41 @@ +# LSF checkpoint with CRIU + +## Usage + +To prepare checkpoint environment for [CRIU](https://criu.org/Main_Page), +echkpnt scripts should be downloaded and copy to your `LSF` environment. +``` +$ cp echkpnt.criu erestart.criu $LSF_SERVERDIR +``` + +To use `CRIU` checkpoint method: +1. submit a job with method `criu` +``` +$ bsub -k '/usr/local/work/chkpnt/ method=criu' ./counter +Job <477> is submitted to default queue . +``` + +2. checkpoint the job with killing +``` +$ bchkpnt -k 477 +Job <477> is being checkpointed +``` + +3. restart the checkpointed job +``` +$ brestart /usr/local/work/chkpnt/ 477 +Job <478> is submitted to queue . +``` +## Debug +By default, `CRIU` checkpoint scripts log debug information to `/tmp` directory. +``` +$ cat /tmp/lsf-job-477.cr.log +2021-11-02 19:09:27,035 lsf-criu-checkpoint[2670944]: start checkpointing ...... +2021-11-02 19:09:27,035 lsf-criu-checkpoint[2670944]: ['-c', '-k', '-d', '477.tmp', '2669404'] +2021-11-02 19:09:27,044 lsf-criu-checkpoint[2670944]: job process id: 2669408 +2021-11-02 19:09:27,044 lsf-criu-checkpoint[2670944]: ['criu', 'dump', '-D', '/usr/local/work/chkpnt//477', '-t', '2669408', '--shell-job'] +2021-11-02 19:09:27,864 lsf-criu-checkpoint[2670944]: b'' +2021-11-02 19:09:27,864 lsf-criu-checkpoint[2670944]: b'Warn (compel/arch/x86/src/lib/infect.c:340): Will restore 2669410 with interrupted system call\n' +2021-11-02 19:09:27,865 lsf-criu-checkpoint[2670944]: leave checkpointing ... +``` + diff --git a/criu/echkpnt.criu b/criu/echkpnt.criu new file mode 100644 index 0000000..61c4369 --- /dev/null +++ b/criu/echkpnt.criu @@ -0,0 +1,76 @@ +#!/usr/bin/python3 + +# Copyright International Business Machines Corp, 2021 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import os +import psutil +import logging +import subprocess + +log = None + +def set_log(logfile): + global log + + filelog = logging.FileHandler(logfile) + logformat = logging.Formatter("%(asctime)s %(name)s[%(process)d]: %(message)s") + filelog.setFormatter(logformat) + + log = logging.getLogger('lsf-criu-checkpoint') + log.setLevel(logging.DEBUG) + log.addHandler(filelog) + +def log_debug(message): + if log is not None : + log.debug(message) + +def run_it(cmd) : + proc = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE) + out, err = proc.communicate() + if out is not None and out.strip() != '': + log_debug(out) + if err is not None and err.strip() != '': + log_debug(err) + proc.wait() + +def main(argv): + # For debug purpose only. Comment it out if you don't want debug messages + set_log("/tmp/lsf-job-" + os.environ['LSB_JOBID'] + ".cr.log") + + log_debug("start checkpointing ......") + log_debug(argv) + + # get process id for running job to checkpoint + p = psutil.Process(int(argv[-1])) + jobpid = str(p.children()[0].pid) + log_debug('job process id: ' + jobpid) + + # make checkpoint command line + cmd = ["criu", "dump", "-D", os.environ['LSB_CHKPNT_DIR'] , "-t", jobpid, "--shell-job"]; + if '-k' not in argv: + cmd.append('--leave-running') + log_debug(cmd) + + # checkpoint + run_it(cmd) + + log_debug("leave checkpointing ...") + sys.exit(0) + + +if __name__ == "__main__": + main(sys.argv[1:]) + diff --git a/criu/erestart.criu b/criu/erestart.criu new file mode 100644 index 0000000..9637159 --- /dev/null +++ b/criu/erestart.criu @@ -0,0 +1,89 @@ +#!/usr/bin/python3 + +# Copyright International Business Machines Corp, 2021 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import os +import re +import logging +import subprocess + +log = None + +def set_log(logfile): + global log + + filelog = logging.FileHandler(logfile) + logformat = logging.Formatter("%(asctime)s %(name)s[%(process)d]: %(message)s") + filelog.setFormatter(logformat) + + log = logging.getLogger('lsf-criu-restart') + log.setLevel(logging.DEBUG) + log.addHandler(filelog) + +def log_debug(message): + if log is not None : + log.debug(message) + +def run_it(cmd) : + proc = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE) + out, err = proc.communicate() + if out is not None and out.strip() != '': + log_debug(out) + if err is not None and err.strip() != '': + log_debug(err) + proc.wait() + +def main(argv): + # for debug purpose only + set_log("/tmp/lsf-job-" + os.environ['LSB_JOBID'] + ".cr.log") + + log_debug("start restarting ......") + log_debug(argv) + + # get job cgroup from memory sub group + cgtop="/sys/fs/cgroup/memory/lsf/" + os.environ['LSF_CGROUP_TOPDIR_KEY'] + jobcg = "^job\." + os.environ['LSB_JOBID'] + "\..*" + jobcgre = re.compile(jobcg) + jobcgroup = "" + for f in os.listdir(cgtop): + if (jobcgre.match(f)): + jobcgroup = f + break; + + log_debug("job cgroup is <" + jobcgroup + ">") + + cgrp_options = "" + if jobcgroup != "": + cgrp_path= "/lsf/" + os.environ['LSF_CGROUP_TOPDIR_KEY'] + "/" + jobcgroup + "/" + mem_cgrp= "memory:" + cgrp_path + frz_cgrp= "freezer:" + cgrp_path + cpu_cgrp= "cpu,cpuacct:" + cgrp_path + cgrp_options= " --cgroup-root " + mem_cgrp + " --cgroup-root " + frz_cgrp + " --cgroup-root " + cpu_cgrp + + cmd = "LSB_RESTART_CMD=criu restore -D " + os.environ['LSB_CHKPNT_DIR'] + cgrp_options + " --shell-job \n" + log_debug(cmd) + + f = open(os.environ['LSB_CHKPNT_DIR'] + "/.restart_cmd", "w") + f.write(cmd); + f.close(); + + log_debug("leaving ...") + sys.exit(0); + + +if __name__ == "__main__": + main(sys.argv[1:]) +