[Enhancement] Fix Profile collect bug when log dir is removed (backport

#51027) (#51259)

Co-authored-by: gengjun-git <[email protected]>

fe/fe-core/src/main/java/com/starrocks/memory/ProcProfileCollector.java

@@ -28,13 +28,15 @@
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.lang.management.ManagementFactory;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
 public class ProcProfileCollector extends FrontendDaemon {
     private static final Logger LOG = LogManager.getLogger(ProcProfileCollector.class);
@@ -46,20 +48,15 @@ public class ProcProfileCollector extends FrontendDaemon {
 
     private long lastCollectTime = -1;
 
-    private boolean initialized = false;
-
     public ProcProfileCollector() {
         super("ProcProfileCollector");
         profileLogDir = Config.sys_log_dir + "/proc_profile";
     }
 
     @Override
     protected void runAfterCatalogReady() {
-        if (!initialized) {
-            File file = new File(profileLogDir);
-            file.mkdirs();
-            initialized = true;
-        }
+        File file = new File(profileLogDir);
+        file.mkdirs();
 
         if (lastCollectTime == -1L
                 || (System.currentTimeMillis() - lastCollectTime > Config.proc_profile_collect_interval_s * 1000)) {
@@ -112,22 +109,47 @@ private void collectProfile(String... command) {
         try {
             ProcessBuilder processBuilder = new ProcessBuilder(command);
             Process process = processBuilder.start();
-            try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
-                StringBuilder sb = new StringBuilder();
-                String line;
-                while ((line = reader.readLine()) != null) {
-                    sb.append(line).append("\n");
-                }
-                if (sb.length() > 0) {
-                    LOG.info("collect profile output: {}", sb.toString());
-                }
-            }
             process.waitFor();
+            if (process.exitValue() != 0) {
+                LOG.info("collect profile failed, stdout: {}, stderr: {}",
+                        getMsgFromInputStream(process.getInputStream()),
+                        getMsgFromInputStream(process.getErrorStream()));
+                stopProfile();
+            }
         } catch (IOException | InterruptedException e) {
             LOG.warn("collect profile failed", e);
         }
     }
 
+    private void stopProfile() {
+        try {
+            ProcessBuilder processBuilder = new ProcessBuilder(StarRocksFE.STARROCKS_HOME_DIR + "/bin/profiler.sh",
+                    "stop",
+                    getPid());
+            Process process = processBuilder.start();
+            boolean terminated = process.waitFor(10, TimeUnit.SECONDS);
+            if (!terminated) {
+                process.destroyForcibly();
+            }
+        } catch (IOException | InterruptedException e) {
+            LOG.warn("stop profile failed", e);
+        }
+    }
+
+    private String getMsgFromInputStream(InputStream inputStream) throws IOException {
+        if (inputStream == null) {
+            return "";
+        }
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
+            StringBuilder sb = new StringBuilder();
+            String line;
+            while ((line = reader.readLine()) != null) {
+                sb.append(line).append("\n");
+            }
+            return sb.toString();
+        }
+    }
+
     private void compressFile(String fileName) throws IOException {
         File sourceFile = new File(profileLogDir + "/" + fileName);
         try (FileOutputStream fileOutputStream = new FileOutputStream(profileLogDir + "/" + fileName + ".tar.gz");
@@ -153,7 +175,11 @@ private void deleteExpiredFiles() {
         File dir = new File(profileLogDir);
         List<File> validFiles = new ArrayList<>();
         long totalSize = 0;
-        for (File file : dir.listFiles()) {
+        File[] files = dir.listFiles();
+        if (files == null) {
+            return;
+        }
+        for (File file : files) {
             if (file.getName().startsWith(CPU_FILE_NAME_PREFIX)
                     || file.getName().startsWith(MEM_FILE_NAME_PREFIX)) {
                 validFiles.add(file);