From 1b1918f487c251ed2f5f01c1800b6b31adfd0320 Mon Sep 17 00:00:00 2001 From: Mikhail Elhimov Date: Mon, 9 Dec 2024 12:18:47 +0300 Subject: [PATCH] coredump: support options in `pack` subcommand -e,--executable path Tarantool executable path -p,--pid pid PID of the dumped process -t,--time time Time of dump (seconds since the Epoch) Closes #763 --- .github/workflows/full-ci.yml | 8 +++ CHANGELOG.md | 8 +++ cli/cmd/coredump.go | 43 ++++++++++---- cli/coredump/coredump.go | 12 +++- cli/coredump/scripts/tarabrt.sh | 12 ++-- test/integration/coredump/test_coredump.py | 69 +++++++++++++++------- 6 files changed, 112 insertions(+), 40 deletions(-) diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml index 3a8ad37f4..701c34db3 100644 --- a/.github/workflows/full-ci.yml +++ b/.github/workflows/full-ci.yml @@ -51,6 +51,8 @@ jobs: run: sudo systemctl kill mono-xsp4 || true - name: Integration tests + env: + TT_ENABLE_COREDUMP_TESTS: '1' run: mage integrationfull full-ci-ce-linux-arm64: @@ -83,6 +85,8 @@ jobs: run: mage unit:fullSkipDocker - name: Integration tests + env: + TT_ENABLE_COREDUMP_TESTS: '1' run: mage integrationfullskipdocker full-ci-sdk: @@ -139,6 +143,8 @@ jobs: run: sudo systemctl kill mono-xsp4 || true - name: Integration tests + env: + TT_ENABLE_COREDUMP_TESTS: '1' run: mage integrationfull full-ci-macOS: @@ -299,6 +305,7 @@ jobs: env: TT_CLI_EE_USERNAME: '${{ secrets.TT_EE_USERNAME }}' TT_CLI_EE_PASSWORD: '${{ secrets.TT_EE_PASSWORD }}' + TT_ENABLE_COREDUMP_TESTS: '1' run: mage integrationee # ce suffix means that the job will be run on CE tarantool. @@ -347,6 +354,7 @@ jobs: env: TT_CLI_EE_USERNAME: '${{ secrets.TT_EE_USERNAME }}' TT_CLI_EE_PASSWORD: '${{ secrets.TT_EE_PASSWORD }}' + TT_ENABLE_COREDUMP_TESTS: '1' run: mage integrationee # Integration tests on the system without Tarantool installed. diff --git a/CHANGELOG.md b/CHANGELOG.md index 29f128310..39cf57437 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added +- `tt coredump pack`: add several options to customize dump processing: + * `-e (--executable)`: Tarantool executable path. + * `-p (--pid)`: PID of the dumped process. + * `-t (--time)`: Time of dump (seconds since the Epoch). + ### Changed +- `tt coredump pack`: by default tarantool executable path is obtained from + `PATH` instead of using the hardcoded path `/usr/bin/tarantool`. + ### Fixed - `tt coredump inspect`: fails for tarantool-ee coredump archive if the source diff --git a/cli/cmd/coredump.go b/cli/cmd/coredump.go index bb5638c03..a84a1c22b 100644 --- a/cli/cmd/coredump.go +++ b/cli/cmd/coredump.go @@ -6,9 +6,17 @@ import ( "github.com/tarantool/tt/cli/util" ) +var ( + coredumpPackExecutable string + coredumpPackPID uint + coredumpPackTime string + coredumpPackOutputDirectory string + coredumpInspectSourceDir string +) + // NewCoredumpCmd creates coredump command. func NewCoredumpCmd() *cobra.Command { - var coredumpCmd = &cobra.Command{ + var cmd = &cobra.Command{ Use: "coredump", Short: "Perform manipulations with the tarantool coredumps", } @@ -17,12 +25,28 @@ func NewCoredumpCmd() *cobra.Command { Use: "pack COREDUMP", Short: "pack tarantool coredump into tar.gz archive", Run: func(cmd *cobra.Command, args []string) { - if err := coredump.Pack(args[0]); err != nil { + err := coredump.Pack(args[0], + coredumpPackExecutable, + coredumpPackOutputDirectory, + coredumpPackPID, + coredumpPackTime, + ) + if err != nil { util.HandleCmdErr(cmd, err) } }, Args: cobra.ExactArgs(1), } + packCmd.Flags().StringVarP(&coredumpPackExecutable, "executable", "e", "", + "Tarantool executable path") + packCmd.Flags().StringVarP(&coredumpPackOutputDirectory, "directory", "d", "", + "Directory the resulting archive is created in") + packCmd.Flags().StringVarP(&coredumpPackTime, "time", "t", "", + "Time of dump, expressed as seconds since the Epoch (1970-01-01 00:00 UTC)") + packCmd.Flags().UintVarP(&coredumpPackPID, "pid", "p", 0, + "PID of the dumped process, as seen in the PID namespace in which\n"+ + "the given process resides (see %p in core(5) for more info). This flag\n"+ + "is to be used when tt is used as kernel.core_pattern pipeline script") var unpackCmd = &cobra.Command{ Use: "unpack ARCHIVE", @@ -35,29 +59,24 @@ func NewCoredumpCmd() *cobra.Command { Args: cobra.ExactArgs(1), } - var sourceDir string var inspectCmd = &cobra.Command{ Use: "inspect {ARCHIVE|DIRECTORY}", Short: "inspect tarantool coredump", Run: func(cmd *cobra.Command, args []string) { - if err := coredump.Inspect(args[0], sourceDir); err != nil { + if err := coredump.Inspect(args[0], coredumpInspectSourceDir); err != nil { util.HandleCmdErr(cmd, err) } }, Args: cobra.ExactArgs(1), } - inspectCmd.Flags().StringVarP(&sourceDir, "sourcedir", "s", "", + inspectCmd.Flags().StringVarP(&coredumpInspectSourceDir, "sourcedir", "s", "", "Source directory") - subCommands := []*cobra.Command{ + cmd.AddCommand( packCmd, unpackCmd, inspectCmd, - } - - for _, cmd := range subCommands { - coredumpCmd.AddCommand(cmd) - } + ) - return coredumpCmd + return cmd } diff --git a/cli/coredump/coredump.go b/cli/coredump/coredump.go index 259c2cf0e..0469fe52d 100644 --- a/cli/coredump/coredump.go +++ b/cli/coredump/coredump.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "github.com/apex/log" @@ -24,7 +25,7 @@ const packEmbedPath = "scripts/tarabrt.sh" const inspectEmbedPath = "scripts/gdb.sh" // Pack packs coredump into a tar.gz archive. -func Pack(corePath string) error { +func Pack(corePath string, executable string, outputDir string, pid uint, time string) error { tmpDir, err := os.MkdirTemp(os.TempDir(), "tt-coredump-*") if err != nil { return fmt.Errorf("cannot create a temporary directory for archiving: %v", err) @@ -32,6 +33,15 @@ func Pack(corePath string) error { defer os.RemoveAll(tmpDir) // Clean up on function return. scriptArgs := []string{"-c", corePath} + if executable != "" { + scriptArgs = append(scriptArgs, "-e", executable) + } + if outputDir != "" { + scriptArgs = append(scriptArgs, "-d", outputDir) + } + if pid != 0 { + scriptArgs = append(scriptArgs, "-p", strconv.FormatUint(uint64(pid), 10)) + } // Prepare gdb wrapper for packing. inspectPath := filepath.Join(tmpDir, filepath.Base(inspectEmbedPath)) diff --git a/cli/coredump/scripts/tarabrt.sh b/cli/coredump/scripts/tarabrt.sh index d20ec8387..3760c135a 100755 --- a/cli/coredump/scripts/tarabrt.sh +++ b/cli/coredump/scripts/tarabrt.sh @@ -26,10 +26,10 @@ Supported options are: within DIRECTORY. -e TARANTOOL Use file TARANTOOL as the executable file for - examining with a core dump COREDUMP. If PID is - specified, the one from /proc/PID/exe is chosen - (see proc(5) for more info). If TARANTOOL is - omitted, /usr/bin/tarantool is chosen. + examining with a core dump COREDUMP. By default + tarantool executable path is obtained from PATH. + If PID is specified, the one from /proc/PID/exe + is chosen (see proc(5) for more info). -g GDBWRAPPER Include GDB-wrapper script GDBWRAPPER into the archive. @@ -62,7 +62,7 @@ HELP ) # Assign configurable parameters with the default values. -BINARY=/usr/bin/tarantool +BINARY=$(which tarantool) COREDIR=${PWD} COREFILE= EXTS= @@ -175,7 +175,7 @@ fi # behaviours for the check below. For more info, see the highly # detailed answer on Stack Overflow below. # https://stackoverflow.com/a/55704865/4609359 -if file "${BINARY}" | grep -qvE 'executable|shared object'; then +if file -L "${BINARY}" | grep -qvE 'executable|shared object'; then [ -t 1 ] && cat < Path: - coredump_tmpdir = tmp_path_factory.mktemp("coredump") +skip_coredump_cond = os.getenv('TT_ENABLE_COREDUMP_TESTS') is None +skip_coredump_reason = "Should be launched explicitly to control coredump it produces" + + +def generate_coredump(workdir, env=None) -> Path: + print(f"generate_coredump: workdir={workdir}") + print(f"generate_coredump: env={env}") with open('/proc/sys/kernel/core_pattern', 'r') as f: - core_pattern = f.read() + core_pattern = f.read().strip() def apport_crash_to_coredump(crash): - apport_unpack_dir = coredump_tmpdir / 'apport-unpack' + apport_unpack_dir = workdir / 'apport-unpack' rc, output = run_command_and_get_output(['apport-unpack', crash, apport_unpack_dir]) return apport_unpack_dir / 'CoreDump' to_coredump = None if not core_pattern.startswith('|'): - core_wildcard = core_pattern.strip().split('%')[0] + '*' + core_wildcard = core_pattern.replace('%%', '%') + core_wildcard = re.sub('%[cdeEghiIpPstu]', '*', core_wildcard) if not os.path.isabs(core_wildcard): - core_wildcard = coredump_tmpdir / core_wildcard + core_wildcard = str(workdir / core_wildcard) elif re.search(r"apport", core_pattern): - core_wildcard = os.path.join('/var/crash', '*.crash') + core_wildcard = '/var/crash/*.crash' to_coredump = apport_crash_to_coredump elif re.search(r"systemd-coredump", core_pattern): - core_wildcard = os.path.join('/var/lib/systemd/coredump', '*') + core_wildcard = '/var/lib/systemd/coredump/*' else: assert False, "Unexpected core pattern '{}'".format(core_pattern) + print(f"generate_coredump: core_wildcard={core_wildcard}") cores = set(glob.glob(core_wildcard)) + print(f"generate_coredump: cores={cores}") # Setup ulimit -c. rlim_core_soft, rlim_core_hard = resource.getrlimit(resource.RLIMIT_CORE) @@ -54,7 +61,7 @@ def apport_crash_to_coredump(crash): resource.setrlimit(resource.RLIMIT_CORE, (resource.RLIM_INFINITY, rlim_core_hard)) # Crash tarantool. cmd = ["tarantool", "-e", "require('ffi').cast('char *', 0)[0] = 42"] - rc, output = run_command_and_get_output(cmd, cwd=coredump_tmpdir) + rc, output = run_command_and_get_output(cmd, cwd=workdir, env=env) # Restore ulimit -c. resource.setrlimit(resource.RLIMIT_CORE, (rlim_core_soft, rlim_core_hard)) assert rc != 0 @@ -62,17 +69,30 @@ def apport_crash_to_coredump(crash): # Find the newly generated coredump. new_cores = set(glob.glob(core_wildcard)) - cores + print(f"generate_coredump: new_cores={new_cores}") assert len(new_cores) == 1 # And move it to the temporary directory (this directory is removed # automatically, so there is no need to remove the coredump explicitly). - core_path = coredump_tmpdir / "core" + core_path = workdir / "core" + print(f"generate_coredump: core_path={core_path}") os.rename(next(iter(new_cores)), core_path) assert os.path.exists(core_path) return core_path if to_coredump is None else to_coredump(core_path) +@pytest.fixture(scope="session") +def coredump(tmp_path_factory) -> Path: + return generate_coredump(tmp_path_factory.mktemp("coredump")) + + +@pytest.fixture(scope="session") +def coredump_alt(tmpdir_with_tarantool) -> Path: + bin_dir = os.path.join(tmpdir_with_tarantool, "bin") + return generate_coredump(tmpdir_with_tarantool, env=dict(PATH=bin_dir)) + + @pytest.fixture(scope="session") def coredump_packed(tt_cmd, coredump): cmd = [tt_cmd, "coredump", "pack", coredump.as_posix()] @@ -108,8 +128,7 @@ def test_coredump_pack_no_such_file(tt_cmd, tmp_path): assert re.search(r"pack script execution failed", output) -@pytest.mark.skipif(os.getenv('TT_ENABLE_COREDUMP_FIXTURE') is None, - reason="Should be launched explicitly to control coredump it produces") +@pytest.mark.skipif(skip_coredump_cond, reason=skip_coredump_reason) def test_coredump_pack(tt_cmd, tmp_path, coredump): cmd = [tt_cmd, "coredump", "pack", coredump] rc, output = run_command_and_get_output(cmd, cwd=tmp_path) @@ -117,6 +136,17 @@ def test_coredump_pack(tt_cmd, tmp_path, coredump): assert re.search(r"Core was successfully packed.", output) +@pytest.mark.skipif(skip_coredump_cond, reason=skip_coredump_reason) +@pytest.mark.slow +def test_coredump_pack_executable(tt_cmd, tmp_path, coredump_alt, tmpdir_with_tarantool): + executable = tmpdir_with_tarantool / "bin" / "tarantool" + print(f"test_coredump_pack_executable: executable={executable}") + cmd = [tt_cmd, "coredump", "pack", "-e", executable, coredump_alt] + rc, output = run_command_and_get_output(cmd, cwd=tmp_path) + assert rc == 0 + assert re.search(r"Core was successfully packed.", output) + + def test_coredump_unpack_no_arg(tt_cmd, tmp_path): cmd = [tt_cmd, "coredump", "unpack"] rc, output = run_command_and_get_output(cmd, cwd=tmp_path) @@ -131,8 +161,7 @@ def test_coredump_unpack_no_such_file(tt_cmd, tmp_path): assert re.search(r"failed to unpack", output) -@pytest.mark.skipif(os.getenv('TT_ENABLE_COREDUMP_FIXTURE') is None, - reason="Should be launched explicitly to control coredump it produces") +@pytest.mark.skipif(skip_coredump_cond, reason=skip_coredump_reason) def test_coredump_unpack(tt_cmd, tmp_path, coredump_packed): cmd = [tt_cmd, "coredump", "unpack", coredump_packed] rc, output = run_command_and_get_output(cmd, cwd=tmp_path) @@ -154,8 +183,7 @@ def test_coredump_inspect_no_such_file(tt_cmd, tmp_path): assert re.search(r"failed to inspect", output) -@pytest.mark.skipif(os.getenv('TT_ENABLE_COREDUMP_FIXTURE') is None, - reason="Should be launched explicitly to control coredump it produces") +@pytest.mark.skipif(skip_coredump_cond, reason=skip_coredump_reason) def test_coredump_inspect_packed(tt_cmd, tmp_path, coredump_packed): cmd = [tt_cmd, "coredump", "inspect", coredump_packed] process = subprocess.run( @@ -167,8 +195,7 @@ def test_coredump_inspect_packed(tt_cmd, tmp_path, coredump_packed): assert process.returncode == 0 -@pytest.mark.skipif(os.getenv('TT_ENABLE_COREDUMP_FIXTURE') is None, - reason="Should be launched explicitly to control coredump it produces") +@pytest.mark.skipif(skip_coredump_cond, reason=skip_coredump_reason) def test_coredump_inspect_unpacked(tt_cmd, tmp_path, coredump_unpacked): cmd = [tt_cmd, "coredump", "inspect", coredump_unpacked] process = subprocess.run(