To remove the dependency on the zip command and implement encryption …

…of zip files through code (#529)

* release 2.5.0

* release 2.5.0

* update readme cn

* Update Explanatory Diagram

* To remove the dependency on the zip command and implement encryption of zip files through code

* To remove the dependency on the zip command and implement encryption of zip files through code

* To remove the dependency on the zip command and implement encryption of zip files through code

* fix

* fix

* add obdiag build docker

* add obdiag build docker

* fix

* add build docker

* add build docker

.github/workflows/build_base_docker.yml

@@ -0,0 +1,35 @@
+name: build docker helper
+
+on: 
+  push:
+    tags: 
+      - 'helper-[0-9]+.[0-9]+.[0-9]+'
+
+jobs:
+  build-docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Docker hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Create and push manifests w/o cache
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64
+          file: build/DockerFile
+          push: true
+          tags: |
+            ${{ vars.DOCKER_PUSH_BASE }}/obdiag-builder:latest

.github/workflows/build_package.yml

@@ -86,4 +86,4 @@ jobs:
         with:
           name: obdiag-deb-package
           path: ./oceanbase-diagnostic-tool_*.deb
-          retention-days: 3
+          retention-days: 3

build/Dockerfile

@@ -0,0 +1,12 @@
+FROM centos:7
+RUN rm -rf /etc/yum.repos.d/*
+RUN curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
+RUN yum install -y wget rpm-build
+RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --no-check-certificate
+RUN sh Miniconda3-latest-Linux-x86_64.sh -p /opt/miniconda3 -b
+RUN export PATH=/opt/miniconda3/bin:$PATH
+RUN /opt/miniconda3/bin/conda init
+RUN /opt/miniconda3/bin/conda create --name obdiag python=3.8 -y
+RUN source /opt/miniconda3/bin/activate obdiag
+RUN /opt/miniconda3/envs/obdiag/bin/python3.8  -m pip install --upgrade pip setuptools wheel
+RUN yum install -y gcc gcc-c++ make

common/command.py

@@ -193,6 +193,20 @@ def zip_dir(ssh_client, father_dir, zip_dir, stdio=None):
     ssh_client.exec_cmd(cmd)
 
 
+def tar_gz_dir(ssh_client, father_dir, tar_dir, stdio=None):
+    """
+    Compress files through tar.gz on a remote server
+    :param ssh_client: An instance of SSHClient
+    :param father_dir: The parent directory containing the directory to be compressed
+    :param tar_dir: The directory to be compressed
+    :param stdio: An optional stream for logging output
+    :return: None
+    """
+    tar_gz_file = f"{tar_dir}.tar.gz"
+    cmd = f"cd {father_dir} && tar -zcvf {tar_gz_file} {tar_dir} && rm {tar_dir}"
+    ssh_client.exec_cmd(cmd)
+
+
 def zip_encrypt_dir(ssh_client, zip_password, father_dir, zip_dir, stdio=None):
     """
     Compress files by encryption

common/tool.py

@@ -46,6 +46,10 @@
 import ast
 import lzma
 import pymysql as mysql
+import shutil
+import tarfile
+import pyminizip
+import os
 from datetime import timedelta
 from random import choice
 from io import BytesIO
@@ -655,6 +659,61 @@ def write_append(filename, result, stdio=None):
         with io.open(filename, 'a', encoding='utf-8') as fileobj:
             fileobj.write(u'{}'.format(result))
 
+    def tar_gz_to_zip(temp_dir, tar_gz_file, output_zip, password, stdio):
+        extract_dir = os.path.join(temp_dir, 'extracted_files')
+
+        try:
+            # 1. Extract the tar.gz file
+            with tarfile.open(tar_gz_file, 'r:gz') as tar:
+                tar.extractall(path=extract_dir)
+            stdio.verbose("tar.gz file extracted to {0}".format(extract_dir))
+
+            # 2. Gather all extracted files and their relative paths
+            files_to_compress = []
+            base_paths = []
+            for root, dirs, files in os.walk(extract_dir):
+                for file in files:
+                    file_path = os.path.join(root, file)
+                    base_path = os.path.basename(root)
+                    files_to_compress.append(file_path)
+                    base_paths.append(base_path)
+
+            # 3. Compress the extracted files into a (possibly) encrypted zip file
+            if password:
+                # Use pyminizip to create the encrypted zip file
+                pyminizip.compress_multiple(files_to_compress, base_paths, output_zip, password, 5)  # 5 is the compression level
+                stdio.verbose("extracted files compressed into encrypted {0}".format(output_zip))
+            else:
+                # Create an unencrypted zip file
+                pyminizip.compress_multiple(files_to_compress, base_paths, output_zip, None, 5)
+                stdio.verbose("extracted files compressed into unencrypted {0}".format(output_zip))
+
+            # 4. Remove the extracted directory
+            shutil.rmtree(extract_dir)
+            stdio.verbose("extracted directory {0} removed".format(extract_dir))
+
+            # 5. Optionally remove the original tar.gz file
+            os.remove(tar_gz_file)
+            stdio.verbose("original tar.gz file {0} removed".format(tar_gz_file))
+
+        except tarfile.TarError as te:
+            stdio.exception("tar file error: {0}".format(te))
+            if os.path.exists(extract_dir):
+                shutil.rmtree(extract_dir)
+            return False
+        except pyminizip.compress_error as ce:
+            stdio.exception("compression error: {0}".format(ce))
+            if os.path.exists(extract_dir):
+                shutil.rmtree(extract_dir)
+            return False
+        except Exception as e:
+            stdio.exception("an error occurred: {0}".format(e))
+            if os.path.exists(extract_dir):
+                shutil.rmtree(extract_dir)
+            return False
+
+        return True
+
 
 class YamlLoader(YAML):
 

handler/gather/gather_log.py

@@ -23,7 +23,7 @@
 from handler.base_shell_handler import BaseShellHandler
 from common.obdiag_exception import OBDIAGFormatException
 from common.constant import const
-from common.command import get_file_size, download_file, is_empty_dir, rm_rf_file, get_logfile_name_list, mkdir, delete_empty_file, zip_encrypt_dir, zip_dir
+from common.command import get_file_size, download_file, is_empty_dir, rm_rf_file, get_logfile_name_list, mkdir, delete_empty_file, tar_gz_dir
 from common.command import SshClient
 from common.tool import TimeUtils
 from common.tool import Util
@@ -365,21 +365,20 @@ def __pharse_log(self, ssh_client, home_path, log_name, gather_path):
     def __handle_zip_file(self, ssh_client, resp, gather_dir_name, pack_dir_this_command):
         zip_password = ""
         gather_dir_full_path = "{0}/{1}".format(self.gather_ob_log_temporary_dir, gather_dir_name)
-        self.stdio.print('[ip: {0}] zip observer log start'.format(ssh_client.get_name()))
+        self.stdio.print('[ip: {0}] gather observer log start'.format(ssh_client.get_name()))
         if self.zip_encrypt:
             zip_password = Util.gen_password(16)
-            self.zip_password = zip_password
-            zip_encrypt_dir(ssh_client, zip_password, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio)
-        else:
-            zip_dir(ssh_client, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio)
-        self.stdio.print('[{0}] zip observer log end'.format(ssh_client.get_name()))
-        gather_package_dir = "{0}.zip".format(gather_dir_full_path)
+        tar_gz_dir(ssh_client, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio)
+        self.stdio.print('[{0}] gather observer log end'.format(ssh_client.get_name()))
+        gather_package_dir = "{0}.tar.gz".format(gather_dir_full_path)
         gather_log_file_size = get_file_size(ssh_client, gather_package_dir, self.stdio)
         self.stdio.print(FileUtil.show_file_size_tabulate(ssh_client, gather_log_file_size))
         local_store_path = ""
         if int(gather_log_file_size) < self.file_size_limit:
+            local_store_tar_gz_file = pack_dir_this_command + "/{0}.tar.gz".format(gather_dir_name)
+            download_file(ssh_client, gather_package_dir, local_store_tar_gz_file, self.stdio)
             local_store_path = pack_dir_this_command + "/{0}.zip".format(gather_dir_name)
-            download_file(ssh_client, gather_package_dir, local_store_path, self.stdio)
+            FileUtil.tar_gz_to_zip(pack_dir_this_command, local_store_tar_gz_file, local_store_path, zip_password, self.stdio)
             resp["error"] = ""
             resp["zip_password"] = zip_password
         else:

handler/gather/gather_obadmin.py

@@ -24,7 +24,7 @@
 from common.constant import const
 from common.command import LocalClient, SshClient, is_empty_dir
 from handler.base_shell_handler import BaseShellHandler
-from common.command import download_file, rm_rf_file, get_file_size, zip_encrypt_dir, zip_dir, get_observer_version
+from common.command import download_file, rm_rf_file, get_file_size, get_observer_version, tar_gz_dir
 from common.tool import TimeUtils
 from common.tool import StringUtils
 from common.tool import Util
@@ -221,15 +221,16 @@ def __handle_zip_file(self, ssh_client, resp, gather_dir_name, pack_dir_this_com
         gather_dir_full_path = "{0}/{1}".format(self.gather_ob_log_temporary_dir, gather_dir_name)
         if self.zip_encrypt:
             zip_password = Util.gen_password(16)
-            zip_encrypt_dir(ssh_client, zip_password, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio)
-        else:
-            zip_dir(ssh_client, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio)
-        gather_package_dir = "{0}.zip".format(gather_dir_full_path)
+        tar_gz_dir(ssh_client, self.gather_ob_log_temporary_dir, gather_dir_name, self.stdio)
+        gather_package_dir = "{0}.tar.gz".format(gather_dir_full_path)
         gather_log_file_size = get_file_size(ssh_client, gather_package_dir, self.stdio)
         self.stdio.print(FileUtil.show_file_size_tabulate(ssh_client, gather_log_file_size, self.stdio))
         local_path = ""
         if int(gather_log_file_size) < self.file_size_limit:
-            local_path = download_file(ssh_client, gather_package_dir, pack_dir_this_command, self.stdio)
+            local_store_tar_gz_file = pack_dir_this_command + "/{0}.tar.gz".format(gather_dir_name)
+            download_file(ssh_client, gather_package_dir, local_store_tar_gz_file, self.stdio)
+            local_path = pack_dir_this_command + "/{0}.zip".format(gather_dir_name)
+            FileUtil.tar_gz_to_zip(pack_dir_this_command, local_store_tar_gz_file, local_path, zip_password, self.stdio)
             resp["error"] = ""
             resp["zip_password"] = zip_password
         else:

handler/gather/gather_obproxy_log.py

@@ -26,7 +26,7 @@
 from common.obdiag_exception import OBDIAGFormatException
 from common.command import SshClient
 from common.constant import const
-from common.command import get_file_size, download_file, is_empty_dir, get_logfile_name_list, mkdir, delete_empty_file, rm_rf_file, zip_encrypt_dir, zip_dir
+from common.command import get_file_size, download_file, is_empty_dir, get_logfile_name_list, mkdir, delete_empty_file, rm_rf_file, tar_gz_dir
 from common.tool import Util
 from common.tool import DirectoryUtil
 from common.tool import FileUtil
@@ -306,17 +306,16 @@ def __handle_zip_file(self, ssh_client, resp, gather_dir_name, pack_dir_this_com
         gather_dir_full_path = "{0}/{1}".format(self.gather_log_temporary_dir, gather_dir_name)
         if self.zip_encrypt:
             zip_password = Util.gen_password(16)
-            zip_encrypt_dir(ssh_client, zip_password, self.gather_log_temporary_dir, gather_dir_name, self.stdio)
-        else:
-            zip_dir(ssh_client, self.gather_log_temporary_dir, gather_dir_name, self.stdio)
-        gather_package_dir = "{0}.zip".format(gather_dir_full_path)
-
+        tar_gz_dir(ssh_client, self.gather_log_temporary_dir, gather_dir_name, self.stdio)
+        gather_package_dir = "{0}.tar.gz".format(gather_dir_full_path)
         gather_log_file_size = get_file_size(ssh_client, gather_package_dir, self.stdio)
         self.stdio.print(FileUtil.show_file_size_tabulate(ssh_client, gather_log_file_size))
         local_path = ""
         if int(gather_log_file_size) < self.file_size_limit:
-            local_store_path = pack_dir_this_command + "/{0}.zip".format(gather_dir_name)
-            local_path = download_file(ssh_client, gather_package_dir, local_store_path, self.stdio)
+            local_store_tar_gz_file = pack_dir_this_command + "/{0}.tar.gz".format(gather_dir_name)
+            download_file(ssh_client, gather_package_dir, local_store_tar_gz_file, self.stdio)
+            local_path = pack_dir_this_command + "/{0}.zip".format(gather_dir_name)
+            FileUtil.tar_gz_to_zip(pack_dir_this_command, local_store_tar_gz_file, local_path, zip_password, self.stdio)
             resp["error"] = ""
             resp["zip_password"] = zip_password
         else:

handler/gather/gather_obstack2.py

@@ -22,7 +22,7 @@
 
 import tabulate
 
-from common.command import download_file, is_empty_dir, is_support_arch, get_observer_version, get_observer_pid, mkdir, zip_dir, get_file_size, delete_file_force, is_empty_file, upload_file
+from common.command import download_file, is_empty_dir, is_support_arch, get_observer_version, get_observer_pid, mkdir, tar_gz_dir, get_file_size, delete_file_force, is_empty_file, upload_file
 from common.constant import const
 from common.command import LocalClient, SshClient
 from handler.base_shell_handler import BaseShellHandler
@@ -167,15 +167,17 @@ def __handle_from_node(self, local_stored_path, node):
         if is_empty_dir(ssh_client, "/tmp/{0}".format(remote_dir_name), self.stdio):
             resp["error"] = "gather failed, folder is empty"
             return resp
+        tar_gz_dir(ssh_client, "/tmp", remote_dir_name, self.stdio)
+        remote_tar_file_path = "{0}.tar.gz".format(remote_dir_full_path)
 
-        zip_dir(ssh_client, "/tmp", remote_dir_name, self.stdio)
-        remote_zip_file_path = "{0}.zip".format(remote_dir_full_path)
-
-        file_size = get_file_size(ssh_client, remote_zip_file_path, self.stdio)
-        remote_file_full_path = "{0}.zip".format(remote_dir_full_path)
+        file_size = get_file_size(ssh_client, remote_tar_file_path, self.stdio)
+        remote_file_full_path = "{0}.tar.gz".format(remote_dir_full_path)
         if int(file_size) < self.file_size_limit:
-            local_file_path = "{0}/{1}.zip".format(local_stored_path, remote_dir_name)
-            download_file(ssh_client, remote_file_full_path, local_file_path, self.stdio)
+            local_tar_file_path = "{0}/{1}.tar.gz".format(local_stored_path, remote_dir_name)
+            self.stdio.verbose("local tar file path {0}...".format(local_tar_file_path))
+            download_file(ssh_client, remote_file_full_path, local_tar_file_path, self.stdio)
+            local_zip_file_path = local_stored_path + "/{0}.zip".format(remote_dir_name)
+            FileUtil.tar_gz_to_zip(local_stored_path, local_tar_file_path, local_zip_file_path, None, self.stdio)
             resp["error"] = ""
         else:
             resp["error"] = "File too large"

handler/gather/gather_perf.py

@@ -21,7 +21,7 @@
 
 import tabulate
 
-from common.command import get_observer_pid, mkdir, zip_dir, get_file_size, download_file, delete_file_force
+from common.command import get_observer_pid, mkdir, tar_gz_dir, get_file_size, download_file, delete_file_force
 from common.command import SshClient
 from common.constant import const
 from handler.base_shell_handler import BaseShellHandler
@@ -151,12 +151,15 @@ def __handle_from_node(self, node, local_stored_path):
                     self.__gather_perf_flame(ssh_client, remote_dir_full_path, pid_observer)
                 self.__gather_top(ssh_client, remote_dir_full_path, pid_observer)
 
-            zip_dir(ssh_client, "/tmp", remote_dir_name, self.stdio)
-            remote_file_full_path = "{0}.zip".format(remote_dir_full_path)
+            tar_gz_dir(ssh_client, "/tmp", remote_dir_name, self.stdio)
+            remote_file_full_path = "{0}.tar.gz".format(remote_dir_full_path)
             file_size = get_file_size(ssh_client, remote_file_full_path, self.stdio)
             if int(file_size) < self.file_size_limit:
-                local_file_path = "{0}/{1}.zip".format(local_stored_path, remote_dir_name)
-                download_file(ssh_client, remote_file_full_path, local_file_path, self.stdio)
+                local_tar_file_path = "{0}/{1}.tar.gz".format(local_stored_path, remote_dir_name)
+                self.stdio.verbose("local tar file path {0}...".format(local_tar_file_path))
+                download_file(ssh_client, remote_file_full_path, local_tar_file_path, self.stdio)
+                local_zip_file_path = local_stored_path + "/{0}.zip".format(remote_dir_name)
+                FileUtil.tar_gz_to_zip(local_stored_path, local_tar_file_path, local_zip_file_path, None, self.stdio)
                 resp["error"] = ""
             else:
                 resp["error"] = "File too large"

handler/gather/gather_sysstat.py

@@ -22,7 +22,7 @@
 import tabulate
 from common.constant import const
 from common.command import LocalClient, SshClient
-from common.command import get_file_size, download_file, mkdir, zip_dir
+from common.command import get_file_size, download_file, mkdir, tar_gz_dir
 from handler.base_shell_handler import BaseShellHandler
 from common.tool import Util
 from common.tool import DirectoryUtil
@@ -147,13 +147,15 @@ def __handle_from_node(self, node, local_stored_path):
                 self.__gather_io_info(ssh_client, remote_dir_full_path)
                 self.__gather_traffic_info(ssh_client, remote_dir_full_path)
                 self.__gather_tcp_udp_info(ssh_client, remote_dir_full_path)
-            zip_dir(ssh_client, "/tmp", remote_dir_name, self.stdio)
-            remote_file_full_path = "{0}.zip".format(remote_dir_full_path)
+            tar_gz_dir(ssh_client, "/tmp", remote_dir_name, self.stdio)
+            remote_file_full_path = "{0}.tar.gz".format(remote_dir_full_path)
             file_size = get_file_size(ssh_client, remote_file_full_path, self.stdio)
             if int(file_size) < self.file_size_limit:
-                local_file_path = "{0}/{1}.zip".format(local_stored_path, remote_dir_name)
-                self.stdio.verbose("local file path {0}...".format(local_file_path))
-                download_file(ssh_client, remote_file_full_path, local_file_path, self.stdio)
+                local_tar_file_path = "{0}/{1}.tar.gz".format(local_stored_path, remote_dir_name)
+                self.stdio.verbose("local tar file path {0}...".format(local_tar_file_path))
+                download_file(ssh_client, remote_file_full_path, local_tar_file_path, self.stdio)
+                local_zip_file_path = local_stored_path + "/{0}.zip".format(remote_dir_name)
+                FileUtil.tar_gz_to_zip(local_stored_path, local_tar_file_path, local_zip_file_path, None, self.stdio)
                 resp["error"] = ""
             else:
                 resp["error"] = "File too large"

requirements3.txt

@@ -39,3 +39,4 @@ netifaces==0.11.0
 netifaces==0.11.0
 kubernetes==30.1.0
 setuptools==65.6.3
+pyminizip==0.2.6