From 337b24d36de02d531042caf5247685f5ddfaf924 Mon Sep 17 00:00:00 2001 From: Mikhail Elhimov Date: Mon, 9 Dec 2024 12:18:47 +0300 Subject: [PATCH] coredump: support options in `pack` subcommand -e,--executable path Tarantool executable path -p,--pid pid PID of the dumped process -t,--time time Time of dump (seconds since the Epoch) Closes #763 @TarantoolBot document Title: support options in `tt coredump pack` subcommand --- .../actions/prepare-ce-test-env/action.yml | 5 + .../actions/prepare-ee-test-env/action.yml | 5 + .github/workflows/full-ci.yml | 6 ++ CHANGELOG.md | 7 +- cli/cmd/coredump.go | 43 +++++--- cli/coredump/coredump.go | 12 ++- cli/coredump/scripts/tarabrt.sh | 22 +++-- test/integration/coredump/test_coredump.py | 99 +++++++++++++------ test/utils.py | 4 +- 9 files changed, 149 insertions(+), 54 deletions(-) diff --git a/.github/actions/prepare-ce-test-env/action.yml b/.github/actions/prepare-ce-test-env/action.yml index 3f72a1e6f..bb0a239d8 100644 --- a/.github/actions/prepare-ce-test-env/action.yml +++ b/.github/actions/prepare-ce-test-env/action.yml @@ -67,3 +67,8 @@ runs: TT_CLI_BUILD_SSL: 'static' run: mage build shell: bash + + - name: Install test requirements + run: | + sudo apt -y install gdb + shell: bash diff --git a/.github/actions/prepare-ee-test-env/action.yml b/.github/actions/prepare-ee-test-env/action.yml index d30c3ee8a..44e4d7a8e 100644 --- a/.github/actions/prepare-ee-test-env/action.yml +++ b/.github/actions/prepare-ee-test-env/action.yml @@ -91,3 +91,8 @@ runs: TT_CLI_BUILD_SSL: 'static' run: mage build shell: bash + + - name: Install test requirements + run: | + sudo apt -y install gdb + shell: bash diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml index 3a8ad37f4..ac2e4a347 100644 --- a/.github/workflows/full-ci.yml +++ b/.github/workflows/full-ci.yml @@ -51,6 +51,8 @@ jobs: run: sudo systemctl kill mono-xsp4 || true - name: Integration tests + env: + TT_ENABLE_COREDUMP_TESTS: '1' run: mage integrationfull full-ci-ce-linux-arm64: @@ -83,6 +85,8 @@ jobs: run: mage unit:fullSkipDocker - name: Integration tests + env: + TT_ENABLE_COREDUMP_TESTS: '1' run: mage integrationfullskipdocker full-ci-sdk: @@ -139,6 +143,8 @@ jobs: run: sudo systemctl kill mono-xsp4 || true - name: Integration tests + env: + TT_ENABLE_COREDUMP_TESTS: '1' run: mage integrationfull full-ci-macOS: diff --git a/CHANGELOG.md b/CHANGELOG.md index 50a6417e6..4ba1d1805 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,15 +14,20 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. an extra path with modules. - `tt play`: support connection to a target instance by `application` name or `application:instance` name. +- `tt coredump pack`: add options to customize coredump packing: + * `-e (--executable)`: specify Tarantool executable path. + * `-p (--pid)`: specify PID of the dumped process. + * `-t (--time)`: specify time of dump (seconds since the Epoch). ### Changed - `tt stop/kill/clean/logrotate`: no longer need: * Instances scripts for multi-instance applications. * Cluster config for tarantool3-based cluster applications. - - `tt logrotate`: don't exit at non-running instance, just warn and proceed with the other instances, like `tt stop` and `tt kill` do. +- `tt coredump pack`: by default tarantool executable path is obtained from + `PATH` instead of using the hardcoded path `/usr/bin/tarantool`. ### Fixed diff --git a/cli/cmd/coredump.go b/cli/cmd/coredump.go index bb5638c03..a84a1c22b 100644 --- a/cli/cmd/coredump.go +++ b/cli/cmd/coredump.go @@ -6,9 +6,17 @@ import ( "github.com/tarantool/tt/cli/util" ) +var ( + coredumpPackExecutable string + coredumpPackPID uint + coredumpPackTime string + coredumpPackOutputDirectory string + coredumpInspectSourceDir string +) + // NewCoredumpCmd creates coredump command. func NewCoredumpCmd() *cobra.Command { - var coredumpCmd = &cobra.Command{ + var cmd = &cobra.Command{ Use: "coredump", Short: "Perform manipulations with the tarantool coredumps", } @@ -17,12 +25,28 @@ func NewCoredumpCmd() *cobra.Command { Use: "pack COREDUMP", Short: "pack tarantool coredump into tar.gz archive", Run: func(cmd *cobra.Command, args []string) { - if err := coredump.Pack(args[0]); err != nil { + err := coredump.Pack(args[0], + coredumpPackExecutable, + coredumpPackOutputDirectory, + coredumpPackPID, + coredumpPackTime, + ) + if err != nil { util.HandleCmdErr(cmd, err) } }, Args: cobra.ExactArgs(1), } + packCmd.Flags().StringVarP(&coredumpPackExecutable, "executable", "e", "", + "Tarantool executable path") + packCmd.Flags().StringVarP(&coredumpPackOutputDirectory, "directory", "d", "", + "Directory the resulting archive is created in") + packCmd.Flags().StringVarP(&coredumpPackTime, "time", "t", "", + "Time of dump, expressed as seconds since the Epoch (1970-01-01 00:00 UTC)") + packCmd.Flags().UintVarP(&coredumpPackPID, "pid", "p", 0, + "PID of the dumped process, as seen in the PID namespace in which\n"+ + "the given process resides (see %p in core(5) for more info). This flag\n"+ + "is to be used when tt is used as kernel.core_pattern pipeline script") var unpackCmd = &cobra.Command{ Use: "unpack ARCHIVE", @@ -35,29 +59,24 @@ func NewCoredumpCmd() *cobra.Command { Args: cobra.ExactArgs(1), } - var sourceDir string var inspectCmd = &cobra.Command{ Use: "inspect {ARCHIVE|DIRECTORY}", Short: "inspect tarantool coredump", Run: func(cmd *cobra.Command, args []string) { - if err := coredump.Inspect(args[0], sourceDir); err != nil { + if err := coredump.Inspect(args[0], coredumpInspectSourceDir); err != nil { util.HandleCmdErr(cmd, err) } }, Args: cobra.ExactArgs(1), } - inspectCmd.Flags().StringVarP(&sourceDir, "sourcedir", "s", "", + inspectCmd.Flags().StringVarP(&coredumpInspectSourceDir, "sourcedir", "s", "", "Source directory") - subCommands := []*cobra.Command{ + cmd.AddCommand( packCmd, unpackCmd, inspectCmd, - } - - for _, cmd := range subCommands { - coredumpCmd.AddCommand(cmd) - } + ) - return coredumpCmd + return cmd } diff --git a/cli/coredump/coredump.go b/cli/coredump/coredump.go index 259c2cf0e..0469fe52d 100644 --- a/cli/coredump/coredump.go +++ b/cli/coredump/coredump.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "github.com/apex/log" @@ -24,7 +25,7 @@ const packEmbedPath = "scripts/tarabrt.sh" const inspectEmbedPath = "scripts/gdb.sh" // Pack packs coredump into a tar.gz archive. -func Pack(corePath string) error { +func Pack(corePath string, executable string, outputDir string, pid uint, time string) error { tmpDir, err := os.MkdirTemp(os.TempDir(), "tt-coredump-*") if err != nil { return fmt.Errorf("cannot create a temporary directory for archiving: %v", err) @@ -32,6 +33,15 @@ func Pack(corePath string) error { defer os.RemoveAll(tmpDir) // Clean up on function return. scriptArgs := []string{"-c", corePath} + if executable != "" { + scriptArgs = append(scriptArgs, "-e", executable) + } + if outputDir != "" { + scriptArgs = append(scriptArgs, "-d", outputDir) + } + if pid != 0 { + scriptArgs = append(scriptArgs, "-p", strconv.FormatUint(uint64(pid), 10)) + } // Prepare gdb wrapper for packing. inspectPath := filepath.Join(tmpDir, filepath.Base(inspectEmbedPath)) diff --git a/cli/coredump/scripts/tarabrt.sh b/cli/coredump/scripts/tarabrt.sh index d20ec8387..f626c53cf 100755 --- a/cli/coredump/scripts/tarabrt.sh +++ b/cli/coredump/scripts/tarabrt.sh @@ -26,10 +26,10 @@ Supported options are: within DIRECTORY. -e TARANTOOL Use file TARANTOOL as the executable file for - examining with a core dump COREDUMP. If PID is - specified, the one from /proc/PID/exe is chosen - (see proc(5) for more info). If TARANTOOL is - omitted, /usr/bin/tarantool is chosen. + examining with a core dump COREDUMP. By default + tarantool executable path is obtained from PATH. + If PID is specified, the one from /proc/PID/exe + is chosen (see proc(5) for more info). -g GDBWRAPPER Include GDB-wrapper script GDBWRAPPER into the archive. @@ -62,7 +62,7 @@ HELP ) # Assign configurable parameters with the default values. -BINARY=/usr/bin/tarantool +BINARY= COREDIR=${PWD} COREFILE= EXTS= @@ -90,6 +90,16 @@ while true; do esac done +if [ -z "${BINARY}" ]; then + if ! which tarantool; then + [ -t 1 ] && cat < Path: - coredump_tmpdir = tmp_path_factory.mktemp("coredump") +skip_coredump_cond = os.getenv('TT_ENABLE_COREDUMP_TESTS') is None +skip_coredump_reason = "Should be launched explicitly to control coredump it produces" + + +def generate_coredump(tmp_path_factory, tarantool_bin='tarantool') -> Path: + coredump_dir = tmp_path_factory.mktemp("coredump") with open('/proc/sys/kernel/core_pattern', 'r') as f: - core_pattern = f.read() + core_pattern = f.read().strip() + + def coredump_apport(core_source, outdir): + process = subprocess.run(['apport-unpack', core_source, outdir]) + assert process.returncode == 0 + return outdir / 'CoreDump' - def apport_crash_to_coredump(crash): - apport_unpack_dir = coredump_tmpdir / 'apport-unpack' - rc, output = run_command_and_get_output(['apport-unpack', crash, apport_unpack_dir]) - return apport_unpack_dir / 'CoreDump' + def coredump_systemd(core_source, outdir): + core = outdir / core_source.stem + process = subprocess.run(['coredumpctl', 'dump', f'--output={core}']) + assert process.returncode == 0 + return core to_coredump = None if not core_pattern.startswith('|'): - core_wildcard = core_pattern.strip().split('%')[0] + '*' + core_wildcard = core_pattern.replace('%%', '%') + core_wildcard = re.sub('%[cdeEghiIpPstu]', '*', core_wildcard) if not os.path.isabs(core_wildcard): - core_wildcard = coredump_tmpdir / core_wildcard + core_wildcard = str(coredump_dir / core_wildcard) elif re.search(r"apport", core_pattern): - core_wildcard = os.path.join('/var/crash', '*.crash') - to_coredump = apport_crash_to_coredump + core_wildcard = '/var/crash/*.crash' + to_coredump = coredump_apport elif re.search(r"systemd-coredump", core_pattern): - core_wildcard = os.path.join('/var/lib/systemd/coredump', '*') + core_wildcard = '/var/lib/systemd/coredump/*' + to_coredump = coredump_systemd else: assert False, "Unexpected core pattern '{}'".format(core_pattern) cores = set(glob.glob(core_wildcard)) @@ -53,8 +64,8 @@ def apport_crash_to_coredump(crash): if rlim_core_soft != resource.RLIM_INFINITY: resource.setrlimit(resource.RLIMIT_CORE, (resource.RLIM_INFINITY, rlim_core_hard)) # Crash tarantool. - cmd = ["tarantool", "-e", "require('ffi').cast('char *', 0)[0] = 42"] - rc, output = run_command_and_get_output(cmd, cwd=coredump_tmpdir) + cmd = [tarantool_bin, "-e", "require('ffi').cast('char *', 0)[0] = 42"] + rc, output = run_command_and_get_output(cmd, cwd=coredump_dir) # Restore ulimit -c. resource.setrlimit(resource.RLIMIT_CORE, (rlim_core_soft, rlim_core_hard)) assert rc != 0 @@ -63,14 +74,19 @@ def apport_crash_to_coredump(crash): # Find the newly generated coredump. new_cores = set(glob.glob(core_wildcard)) - cores assert len(new_cores) == 1 + core_path = Path(next(iter(new_cores))) - # And move it to the temporary directory (this directory is removed - # automatically, so there is no need to remove the coredump explicitly). - core_path = coredump_tmpdir / "core" - os.rename(next(iter(new_cores)), core_path) + return core_path if to_coredump is None else to_coredump(core_path, coredump_dir) - assert os.path.exists(core_path) - return core_path if to_coredump is None else to_coredump(core_path) + +@pytest.fixture(scope="session") +def coredump(tmp_path_factory) -> Path: + return generate_coredump(tmp_path_factory) + + +@pytest.fixture(scope="session") +def coredump_alt(tmp_path_factory, tmpdir_with_tarantool) -> Path: + return generate_coredump(tmp_path_factory, tmpdir_with_tarantool / 'bin' / 'tarantool') @pytest.fixture(scope="session") @@ -108,8 +124,7 @@ def test_coredump_pack_no_such_file(tt_cmd, tmp_path): assert re.search(r"pack script execution failed", output) -@pytest.mark.skipif(os.getenv('TT_ENABLE_COREDUMP_FIXTURE') is None, - reason="Should be launched explicitly to control coredump it produces") +@pytest.mark.skipif(skip_coredump_cond, reason=skip_coredump_reason) def test_coredump_pack(tt_cmd, tmp_path, coredump): cmd = [tt_cmd, "coredump", "pack", coredump] rc, output = run_command_and_get_output(cmd, cwd=tmp_path) @@ -117,6 +132,29 @@ def test_coredump_pack(tt_cmd, tmp_path, coredump): assert re.search(r"Core was successfully packed.", output) +@pytest.mark.skipif(skip_coredump_cond, reason=skip_coredump_reason) +@pytest.mark.slow +def test_coredump_pack_executable(tt_cmd, tmp_path, coredump_alt, tmpdir_with_tarantool): + tarantool_bin = tmpdir_with_tarantool / 'bin' / 'tarantool' + version_expected = get_tarantool_version(tarantool_bin) + + cmd = [tt_cmd, "coredump", "pack", "-e", tarantool_bin, coredump_alt] + rc, output = run_command_and_get_output(cmd, cwd=tmp_path) + assert rc == 0 + assert re.search(r"Core was successfully packed.", output) + archives = list(tmp_path.glob("*.tar.gz")) + assert len(archives) == 1 + + # Extract Tarantool executable and check its version. + rc = subprocess.run(["tar", "xzf", archives[0], "--wildcards", "*/tarantool"], + cwd=tmp_path).returncode + assert rc == 0 + unpacked_tarantools = list(tmp_path.glob("*/tarantool")) + assert len(unpacked_tarantools) == 1 + version = get_tarantool_version(unpacked_tarantools[0]) + assert version == version_expected + + def test_coredump_unpack_no_arg(tt_cmd, tmp_path): cmd = [tt_cmd, "coredump", "unpack"] rc, output = run_command_and_get_output(cmd, cwd=tmp_path) @@ -131,8 +169,7 @@ def test_coredump_unpack_no_such_file(tt_cmd, tmp_path): assert re.search(r"failed to unpack", output) -@pytest.mark.skipif(os.getenv('TT_ENABLE_COREDUMP_FIXTURE') is None, - reason="Should be launched explicitly to control coredump it produces") +@pytest.mark.skipif(skip_coredump_cond, reason=skip_coredump_reason) def test_coredump_unpack(tt_cmd, tmp_path, coredump_packed): cmd = [tt_cmd, "coredump", "unpack", coredump_packed] rc, output = run_command_and_get_output(cmd, cwd=tmp_path) @@ -154,8 +191,7 @@ def test_coredump_inspect_no_such_file(tt_cmd, tmp_path): assert re.search(r"failed to inspect", output) -@pytest.mark.skipif(os.getenv('TT_ENABLE_COREDUMP_FIXTURE') is None, - reason="Should be launched explicitly to control coredump it produces") +@pytest.mark.skipif(skip_coredump_cond, reason=skip_coredump_reason) def test_coredump_inspect_packed(tt_cmd, tmp_path, coredump_packed): cmd = [tt_cmd, "coredump", "inspect", coredump_packed] process = subprocess.run( @@ -167,8 +203,7 @@ def test_coredump_inspect_packed(tt_cmd, tmp_path, coredump_packed): assert process.returncode == 0 -@pytest.mark.skipif(os.getenv('TT_ENABLE_COREDUMP_FIXTURE') is None, - reason="Should be launched explicitly to control coredump it produces") +@pytest.mark.skipif(skip_coredump_cond, reason=skip_coredump_reason) def test_coredump_inspect_unpacked(tt_cmd, tmp_path, coredump_unpacked): cmd = [tt_cmd, "coredump", "inspect", coredump_unpacked] process = subprocess.run( diff --git a/test/utils.py b/test/utils.py index a2ef2926f..a438c54bb 100644 --- a/test/utils.py +++ b/test/utils.py @@ -518,10 +518,10 @@ def is_valid_tarantool_installed( return True -def get_tarantool_version(): +def get_tarantool_version(tarantool_bin="tarantool"): try: tt_process = subprocess.Popen( - ["tarantool", "--version"], + [tarantool_bin, "--version"], stderr=subprocess.STDOUT, stdout=subprocess.PIPE, text=True ) except FileNotFoundError: