From 5e6e5cecef0d233c9709d3a08f7faee396d21cee Mon Sep 17 00:00:00 2001 From: krihy <51393259+krihy@users.noreply.github.com> Date: Fri, 17 May 2024 16:56:08 +0800 Subject: [PATCH 01/64] fix(taskframework): daemon job be fired at one time in cluster model (#2408) * rm datasource * add properties --- .../odc/service/task/caller/ProcessJobCaller.java | 9 +++++++-- .../odc/service/task/config/DefaultJobConfiguration.java | 3 +++ .../task/config/DefaultSpringJobConfiguration.java | 1 + .../odc/service/task/config/JobConfiguration.java | 3 +++ .../service/task/config/TaskFrameworkConfiguration.java | 5 +---- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ProcessJobCaller.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ProcessJobCaller.java index c2be5f0a6c..77bf4c14e6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ProcessJobCaller.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ProcessJobCaller.java @@ -19,6 +19,7 @@ import static com.oceanbase.odc.service.task.constants.JobConstants.ODC_EXECUTOR_CANNOT_BE_DESTROYED; import java.text.MessageFormat; +import java.util.Objects; import java.util.Optional; import com.oceanbase.odc.common.util.SystemUtils; @@ -75,7 +76,8 @@ protected ExecutorIdentifier doStart(JobContext context) throws JobException { pid, executorName); } - String portString = Optional.ofNullable(SystemUtils.getEnvOrProperty("server.port")) + JobConfiguration jobConfiguration = JobConfigurationHolder.getJobConfiguration(); + String portString = Optional.ofNullable(jobConfiguration.getHostProperties().getPort()) .orElse(DefaultExecutorIdentifier.DEFAULT_PORT + ""); // set process id as namespace return DefaultExecutorIdentifier.builder().host(SystemUtils.getLocalIpAddress()) @@ -99,7 +101,10 @@ protected void doDestroy(JobIdentity ji, ExecutorIdentifier ei) throws JobExcept return; } - if (SystemUtils.getLocalIpAddress().equals(ei.getHost())) { + JobConfiguration jobConfiguration = JobConfigurationHolder.getJobConfiguration(); + String portString = Optional.ofNullable(jobConfiguration.getHostProperties().getPort()) + .orElse(DefaultExecutorIdentifier.DEFAULT_PORT + ""); + if (SystemUtils.getLocalIpAddress().equals(ei.getHost()) && Objects.equals(portString, ei.getPort() + "")) { updateExecutorDestroyed(ji); return; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultJobConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultJobConfiguration.java index eaa9925d85..39c71748df 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultJobConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultJobConfiguration.java @@ -19,6 +19,7 @@ import org.quartz.Scheduler; import com.oceanbase.odc.common.event.EventPublisher; +import com.oceanbase.odc.service.common.model.HostProperties; import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.objectstorage.cloud.model.CloudEnvConfigurations; import com.oceanbase.odc.service.schedule.ScheduleTaskService; @@ -78,4 +79,6 @@ public abstract class DefaultJobConfiguration implements JobConfiguration { protected TaskFrameworkDisabledHandler taskFrameworkDisabledHandler; protected JasyptEncryptorConfigProperties jasyptEncryptorConfigProperties; + + protected HostProperties hostProperties; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultSpringJobConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultSpringJobConfiguration.java index 8842cb1dc1..9a88b47c04 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultSpringJobConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultSpringJobConfiguration.java @@ -74,6 +74,7 @@ public void afterPropertiesSet() { initJobRateLimiter(); setTaskFrameworkDisabledHandler(new DefaultTaskFrameworkDisabledHandler()); setJasyptEncryptorConfigProperties(ctx.getBean(JasyptEncryptorConfigProperties.class)); + setHostProperties(ctx.getBean(HostProperties.class)); } @Override diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/JobConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/JobConfiguration.java index 6db1ea0edf..0bfd463df5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/JobConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/JobConfiguration.java @@ -19,6 +19,7 @@ import org.quartz.Scheduler; import com.oceanbase.odc.common.event.EventPublisher; +import com.oceanbase.odc.service.common.model.HostProperties; import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.objectstorage.cloud.model.CloudEnvConfigurations; import com.oceanbase.odc.service.schedule.ScheduleTaskService; @@ -72,4 +73,6 @@ public interface JobConfiguration { TaskFrameworkDisabledHandler getTaskFrameworkDisabledHandler(); JasyptEncryptorConfigProperties getJasyptEncryptorConfigProperties(); + + HostProperties getHostProperties(); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkConfiguration.java index 0097db65ee..afe3564e7d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkConfiguration.java @@ -16,8 +16,6 @@ package com.oceanbase.odc.service.task.config; -import javax.sql.DataSource; - import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -75,11 +73,10 @@ public JasyptEncryptorConfigProperties JasyptEncryptorConfigProperties( @Lazy @Bean("taskFrameworkSchedulerFactoryBean") - public SchedulerFactoryBean taskFrameworkSchedulerFactoryBean(DataSource dataSource, + public SchedulerFactoryBean taskFrameworkSchedulerFactoryBean( TaskFrameworkProperties taskFrameworkProperties, @Qualifier("taskFrameworkMonitorExecutor") ThreadPoolTaskExecutor executor) { SchedulerFactoryBean schedulerFactoryBean = new SchedulerFactoryBean(); - schedulerFactoryBean.setDataSource(dataSource); String taskFrameworkSchedulerName = "TASK-FRAMEWORK-SCHEDULER"; schedulerFactoryBean.setSchedulerName(taskFrameworkSchedulerName); schedulerFactoryBean.setStartupDelay(taskFrameworkProperties.getQuartzStartDelaySeconds()); From 4687ef5d56f0154ecae5944323729e44bbca49ed Mon Sep 17 00:00:00 2001 From: XiaoYang Date: Mon, 20 May 2024 17:01:40 +0800 Subject: [PATCH 02/64] feat(table-permission): supports table level authority control (#2324) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(table-permission): support table level authority control (beta) (#2163) * feat(database-owner): 1. Add the owner role to the database, supporting approval workflow to use the owner role of the database. 2. Add variables for integrating external approval. such as: database.owner.accounts database.owner.ids database.owner.names database.name environment.name task.description risk.level 2024-02-29 * feat(database-owner): 1. Add the owner role to the database, supporting approval workflow to use the owner role of the database. 2. Add variables for integrating external approval. such as: database.owner.accounts database.owner.ids database.owner.names database.name environment.name task.description risk.level 2024-02-29 * feat(database-owner): 1. Add the owner role to the database, supporting approval workflow to use the owner role of the database. 2. Add variables for integrating external approval. such as: database.owner.accounts database.owner.ids database.owner.names database.name environment.name task.description risk.level 2024-02-29 * feat(database-owner): 1. Add the owner role to the database, supporting approval workflow to use the owner role of the database. 2. Add variables for integrating external approval. such as: database.owner.accounts database.owner.ids database.owner.names database.name environment.name task.description risk.level odc.task.url 3. Adjust the database permission request form, and split the form when applying for multiple databases. 2024-03-06 * feat(database-owner): 1. Add the owner role to the database, supporting approval workflow to use the owner role of the database. 2. Add variables for integrating external approval. such as: database.owner.accounts database.owner.ids database.owner.names database.name environment.name task.description risk.level odc.task.url 3. Adjust the database permission request form, and split the form when applying for multiple databases. 2024-03-06 * feat(database-owner): 1. Add the owner role to the database, supporting approval workflow to use the owner role of the database. 2. Add variables for integrating external approval. such as: database.owner.accounts database.owner.ids database.owner.names database.name environment.name task.description risk.level odc.task.url 3. Adjust the database permission request form, and split the form when applying for multiple databases. 2024-03-07 * feat(database-owner): 1. Add the owner role to the database, supporting approval workflow to use the owner role of the database. 2. Add variables for integrating external approval. such as: database.owner.accounts database.owner.ids database.owner.names database.name environment.name task.description risk.level odc.task.url 3. Adjust the database permission request form, and split the form when applying for multiple databases. 2024-03-08 * feat(database-owner): 1. Add the owner role to the database, supporting approval workflow to use the owner role of the database. 2. Add variables for integrating external approval. such as: database.owner.accounts database.owner.ids database.owner.names database.name environment.name task.description risk.level odc.task.url 3. Adjust the database permission request form, and split the form when applying for multiple databases. 2024-03-08 * feat(database-owner): 1. Add the owner role to the database, supporting approval workflow to use the owner role of the database. 2. Add variables for integrating external approval. such as: database.owner.accounts database.owner.ids database.owner.names database.name environment.name task.description risk.level odc.task.url 3. Adjust the database permission request form, and split the form when applying for multiple databases. 2024-03-08 * feat(permission-table): 1. 完成表存储模型改造 2. 完成表权限管理-批量授权 3. 完成表权限管理-批量回收权限 4. 完成表权限管理-列表查询 5. 完成表权限审批工单创建逻辑 2024-03-14 * feat(permission-table): 1. 完善表权限审批工单-详情页接口 2. 添加TableController，提供v2版本的table list接口 2024-03-19 * feat(permission-table): 1. 完善表权限审批工单-详情页接口 2. 添加TableController，提供v2版本的table list接口 2024-03-19 * feat(permission-table): a. 完成底层数据建模(在开发方案基础上进行了更优的性能设计,进入了单独存储表信息的数据库表。) b. 完成表权限管理与RBAC系统适配 c. 完成表权限元数据CRUD接口开发 d. 完成部分申请表权限的工单接口开发 e. 完成全新的带权限属性的/api/v2版本表List接口开发 f. 完成库权限属性适配，支持无库权限，但是有表权限依然能够展示和创建连接的能力 g. SQL窗口执行返回未授权接口支持表级别属性 2024-03-19 * feat(permission-table): a. 完成数据库变更工单创建，预检查鉴权 b. 完成数据库导出工单创建，预检查鉴权 c. 完成数据库结果集导出工单创建，预检查鉴权 d. 完成表权限相关审计日志功能 e. 完成表权限相关消息通知功能 f. 国际化适配完成 g. 提供一个申请权限使用的无鉴权数据表列表 h. 支持过期表权限清理 2024-04-07 * feat(permission-table): a. 完成数据库变更工单创建，预检查鉴权 b. 完成数据库导出工单创建，预检查鉴权 c. 完成数据库结果集导出工单创建，预检查鉴权 d. 完成表权限相关审计日志功能 e. 完成表权限相关消息通知功能 f. 国际化适配完成 g. 提供一个申请权限使用的无鉴权数据表列表 h. 支持过期表权限清理 2024-04-07 * feat(permission-table): a. 完成数据库变更工单创建，预检查鉴权 b. 完成数据库导出工单创建，预检查鉴权 c. 完成数据库结果集导出工单创建，预检查鉴权 d. 完成表权限相关审计日志功能 e. 完成表权限相关消息通知功能 f. 国际化适配完成 g. 提供一个申请权限使用的无鉴权数据表列表 h. 支持过期表权限清理 2024-04-08 * feat(permission-table): a. 完成数据库变更工单创建，预检查鉴权 b. 完成数据库导出工单创建，预检查鉴权 c. 完成数据库结果集导出工单创建，预检查鉴权 d. 完成表权限相关审计日志功能 e. 完成表权限相关消息通知功能 f. 国际化适配完成 g. 提供一个申请权限使用的无鉴权数据表列表 h. 支持过期表权限清理 2024-04-09 * feat(permission-table): a. 完成数据库变更工单创建，预检查鉴权 b. 完成数据库导出工单创建，预检查鉴权 c. 完成数据库结果集导出工单创建，预检查鉴权 d. 完成表权限相关审计日志功能 e. 完成表权限相关消息通知功能 f. 国际化适配完成 g. 提供一个申请权限使用的无鉴权数据表列表 h. 支持过期表权限清理 2024-04-09 * feat(permission-table): a. table list接口继承项目和库的权限属性 b. 修改.gitmodules，准备打包 2024-04-09 * feat(permission-table): a. 完成数据库变更工单创建，预检查鉴权 b. 完成数据库导出工单创建，预检查鉴权 c. 完成数据库结果集导出工单创建，预检查鉴权 d. 完成表权限相关审计日志功能 e. 完成表权限相关消息通知功能 f. 国际化适配完成 g. 提供一个申请权限使用的无鉴权数据表列表 h. 支持过期表权限清理 2024-04-09 * feat(permission-table): a. 完成数据库变更工单创建，预检查鉴权 b. 完成数据库导出工单创建，预检查鉴权 c. 完成数据库结果集导出工单创建，预检查鉴权 d. 完成表权限相关审计日志功能 e. 完成表权限相关消息通知功能 f. 国际化适配完成 g. 提供一个申请权限使用的无鉴权数据表列表 h. 支持过期表权限清理 2024-04-09 * feat(permission-table): a. 完成数据库变更工单创建，预检查鉴权 b. 完成数据库导出工单创建，预检查鉴权 c. 完成数据库结果集导出工单创建，预检查鉴权 d. 完成表权限相关审计日志功能 e. 完成表权限相关消息通知功能 f. 国际化适配完成 g. 提供一个申请权限使用的无鉴权数据表列表 h. 支持过期表权限清理 2024-04-09 * feat(permission-table): a. 修复单元测试 2024-04-10 * feat(permission-table): a. 修复单元测试 2024-04-10 * fix(permission-table): a. 修复非async类型工单，sql为空问题。 2024-04-10 * fix(permission-table): a. 修复SQL库和表识别``导致识别错误的问题 b. 过滤掉表继承库的ACCESS权限 2024-04-10 * fix(permission-table): a. format code 2024-04-10 * fix(permission-table): a. 修复申请项目外置审批流问题 2024-04-10 * fix(permission-table): a. 修复申请项目外置审批流问题 2024-04-11 * fix(permission-table): bugfix a. 修复过期表权限仍然生效 b. 修复库迁移项目，表权限未清理 c. 修复doris数据源获取表元数据失败 2024-04-15 * fix(permission-table): fix unit test a. 修复单元测试 2024-04-16 * fix(permission-table): fix format a. fix format 2024-04-16 * fix(permission-table): fix review comments a. fix review comments 2024-04-26 * fix(permission-table): fix review comments a. fix review comments 2024-04-26 * refactor: table permission control * response to CR --------- Co-authored-by: isadba --- .../service/db/DBSchemaIndexServiceTest.java | 6 +- .../service/flow/FlowInstanceServiceTest.java | 6 +- .../DatabasePermissionServiceTest.java | 2 + .../shared/constant/AuditEventAction.java | 12 + .../core/shared/constant/AuditEventType.java | 4 + .../core/shared/constant/ResourceType.java | 3 +- .../odc/core/shared/constant/TaskType.java | 4 + .../i18n/BusinessMessages.properties | 12 + .../i18n/BusinessMessages_zh_CN.properties | 12 + .../i18n/BusinessMessages_zh_TW.properties | 12 + .../init-config/default-audit-event-meta.yml | 18 +- .../init/notification-policy-metadata.yaml | 26 +- .../V_4_3_1_1__alter_iam_permission.sql | 7 + ...2__add_list_user_table_permission_view.sql | 85 ++++ ...troller.java => DBSchemaControllerV1.java} | 2 +- .../web/controller/v2/DBSchemaController.java | 60 +++ .../controller/v2/FlowInstanceController.java | 3 +- .../v2/TablePermissionController.java | 98 ++++ .../odc-server/src/main/resources/log4j2.xml | 55 +++ .../metadb/dbobject/DBObjectRepository.java | 2 + .../odc/metadb/iam/PermissionEntity.java | 9 + .../odc/metadb/iam/PermissionRepository.java | 8 + .../iam/UserDatabasePermissionSpec.java | 5 +- .../metadb/iam/UserTablePermissionEntity.java | 97 ++++ .../iam/UserTablePermissionRepository.java | 54 +++ .../metadb/iam/UserTablePermissionSpec.java | 126 ++++++ .../resourcerole/ResourceRoleRepository.java | 1 - .../UserResourceRoleRepository.java | 1 - .../service/audit/AuditEventMetaService.java | 3 +- .../odc/service/audit/util/AuditUtils.java | 11 + .../collaboration/project/ProjectService.java | 21 +- .../connection/database/DatabaseService.java | 119 ++--- .../connection/database/model/DBResource.java | 56 +++ .../connection/database/model/Database.java | 1 + .../model/UnauthorizedDBResource.java} | 34 +- .../connection/table/TableService.java | 171 +++++++ .../table/model/QueryTableParams.java | 36 ++ .../service/connection/table/model/Table.java | 83 ++++ .../oceanbase/odc/service/db/DBPLService.java | 8 +- .../db/schema/DBSchemaSyncService.java | 6 +- .../syncer/object/AbstractDBObjectSyncer.java | 9 +- .../schema/syncer/object/DBTableSyncer.java | 21 + .../odc/service/flow/FlowInstanceService.java | 76 +++- .../service/flow/FlowTaskInstanceService.java | 10 +- .../flow/model/CreateFlowInstanceReq.java | 2 + .../flow/model/FlowInstanceDetailResp.java | 4 + .../model/FlowNodeInstanceDetailResp.java | 6 +- .../task/PreCheckRuntimeFlowableTask.java | 43 +- .../PreCheckRuntimeFlowableTaskCopied.java | 2 +- .../task/mapper/OdcRuntimeDelegateMapper.java | 3 + .../model/DatabasePermissionCheckResult.java | 4 +- .../odc/service/flow/util/FlowTaskUtil.java | 6 + .../odc/service/iam/ResourceRoleService.java | 6 - .../notification/constant/EventLabelKeys.java | 1 + .../notification/helper/EventBuilder.java | 14 + .../DBResourcePermissionHelper.java | 417 ++++++++++++++++++ .../database/ApplyDatabaseFlowableTask.java | 2 + .../database/DatabasePermissionHelper.java | 165 ------- .../database/DatabasePermissionService.java | 2 + .../model/ApplyDatabaseParameter.java | 18 +- .../model/DatabasePermissionType.java | 13 +- .../model/QueryDatabasePermissionParams.java | 4 +- .../table/ApplyTableFlowableTask.java | 271 ++++++++++++ .../ApplyTablePermissionPreprocessor.java | 138 ++++++ .../table/TablePermissionService.java | 234 ++++++++++ .../table/UserTablePermissionMapper.java | 48 ++ .../table/model/ApplyTableParameter.java | 81 ++++ .../table/model/ApplyTableResult.java | 34 ++ .../table/model/CreateTablePermissionReq.java | 48 ++ .../model/QueryTablePermissionParams.java | 37 ++ .../table/model/UserTablePermission.java | 49 ++ .../risklevel/RiskLevelService.java | 2 +- .../session/ConnectConsoleService.java | 12 +- .../session/ConnectSessionService.java | 6 +- ...a => DBResourcePermissionInterceptor.java} | 42 +- .../SwitchDatabaseInterceptor.java | 4 +- .../session/model/SqlAsyncExecuteResp.java | 4 +- ...aExtractor.java => DBSchemaExtractor.java} | 271 +++++++----- .../odc/service/task/TaskService.java | 5 + .../service/task/runtime/PreCheckTask.java | 83 +--- .../session/DBSchemaExtractorTest.java | 74 ++++ .../service/session/SchemaExtractorTest.java | 252 ----------- .../session/test_db_schema_extractor.yaml | 194 ++++++++ 83 files changed, 3156 insertions(+), 840 deletions(-) create mode 100644 server/odc-migrate/src/main/resources/migrate/common/V_4_3_1_1__alter_iam_permission.sql create mode 100644 server/odc-migrate/src/main/resources/migrate/common/V_4_3_1_2__add_list_user_table_permission_view.sql rename server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/{DBSchemaController.java => DBSchemaControllerV1.java} (98%) create mode 100644 server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBSchemaController.java create mode 100644 server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/TablePermissionController.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserTablePermissionEntity.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserTablePermissionRepository.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserTablePermissionSpec.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/DBResource.java rename server/odc-service/src/main/java/com/oceanbase/odc/service/{permission/database/model/UnauthorizedDatabase.java => connection/database/model/UnauthorizedDBResource.java} (51%) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/QueryTableParams.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/Table.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java delete mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/DatabasePermissionHelper.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTableFlowableTask.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTablePermissionPreprocessor.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/TablePermissionService.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/UserTablePermissionMapper.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/model/ApplyTableParameter.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/model/ApplyTableResult.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/model/CreateTablePermissionReq.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/model/QueryTablePermissionParams.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/model/UserTablePermission.java rename server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/{DatabasePermissionInterceptor.java => DBResourcePermissionInterceptor.java} (70%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/{SchemaExtractor.java => DBSchemaExtractor.java} (70%) create mode 100644 server/odc-service/src/test/java/com/oceanbase/odc/service/session/DBSchemaExtractorTest.java delete mode 100644 server/odc-service/src/test/java/com/oceanbase/odc/service/session/SchemaExtractorTest.java create mode 100644 server/odc-service/src/test/resources/session/test_db_schema_extractor.yaml diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBSchemaIndexServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBSchemaIndexServiceTest.java index 746df25b5b..d36fdee1b3 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBSchemaIndexServiceTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBSchemaIndexServiceTest.java @@ -59,7 +59,7 @@ import com.oceanbase.odc.service.iam.ProjectPermissionValidator; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.iam.model.User; -import com.oceanbase.odc.service.permission.database.DatabasePermissionHelper; +import com.oceanbase.odc.service.permission.DBResourcePermissionHelper; import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; import com.oceanbase.odc.test.tool.TestRandom; import com.oceanbase.tools.dbbrowser.model.DBObjectType; @@ -88,7 +88,7 @@ public class DBSchemaIndexServiceTest extends ServiceTestEnv { @MockBean private ConnectionService connectionService; @MockBean - private DatabasePermissionHelper databasePermissionHelper; + private DBResourcePermissionHelper permissionHelper; @Rule public ExpectedException thrown = ExpectedException.none(); @@ -135,7 +135,7 @@ public void setUp() { Mockito.when(connectionService.getBasicWithoutPermissionCheck(Mockito.eq(CONNECTION_ID))) .thenReturn(getConnectionConfig()); Mockito.when(connectionService.mapByIdIn(Mockito.anySet())).thenReturn(getConnectionMap()); - Mockito.when(databasePermissionHelper.getPermissions(Mockito.anySet())).thenReturn( + Mockito.when(permissionHelper.getDBPermissions(Mockito.anySet())).thenReturn( getDatabaseId2PermissionTypes(databases.stream().map(Database::getId).collect(Collectors.toList()))); } diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowInstanceServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowInstanceServiceTest.java index fb8859a163..b0ea1f92c9 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowInstanceServiceTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowInstanceServiceTest.java @@ -98,7 +98,7 @@ import com.oceanbase.odc.service.iam.UserService; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.iam.model.User; -import com.oceanbase.odc.service.permission.database.DatabasePermissionHelper; +import com.oceanbase.odc.service.permission.DBResourcePermissionHelper; import com.oceanbase.odc.service.regulation.approval.ApprovalFlowConfigSelector; import com.oceanbase.odc.service.regulation.approval.model.ApprovalFlowConfig; import com.oceanbase.odc.service.regulation.approval.model.ApprovalNodeConfig; @@ -176,7 +176,7 @@ public class FlowInstanceServiceTest extends ServiceTestEnv { @Autowired private UserTaskInstanceCandidateRepository userTaskInstanceCandidateRepository; @MockBean - private DatabasePermissionHelper databasePermissionHelper; + private DBResourcePermissionHelper permissionHelper; @Before public void setUp() { @@ -200,7 +200,7 @@ public void setUp() { when(databaseService.detail(Mockito.anyLong())).thenReturn(database); when(riskLevelService.findDefaultRiskLevel()).thenReturn(getRiskLevel()); when(riskLevelService.list()).thenReturn(Arrays.asList(getRiskLevel(), getRiskLevel())); - doNothing().when(databasePermissionHelper).checkPermissions(Mockito.anyCollection(), Mockito.anyCollection()); + doNothing().when(permissionHelper).checkDBPermissions(Mockito.anyCollection(), Mockito.anyCollection()); } @Test diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/permission/DatabasePermissionServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/permission/DatabasePermissionServiceTest.java index c8b1f52a07..87434ea114 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/permission/DatabasePermissionServiceTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/permission/DatabasePermissionServiceTest.java @@ -276,6 +276,8 @@ private PermissionEntity createPermission(Long databaseId, String action, Author entity.setAuthorizationType(authorizationType); entity.setTicketId(ticketId); entity.setExpireTime(expireTime); + entity.setResourceType(ResourceType.ODC_DATABASE); + entity.setResourceId(databaseId); return permissionRepository.saveAndFlush(entity); } diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/AuditEventAction.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/AuditEventAction.java index 96021cb95f..0220a78757 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/AuditEventAction.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/AuditEventAction.java @@ -181,6 +181,8 @@ public enum AuditEventAction implements Translatable { CREATE_APPLY_DATABASE_PERMISSION_TASK, + CREATE_APPLY_TABLE_PERMISSION_TASK, + CREATE_SHADOWTABLE_SYNC_TASK, CREATE_STRUCTURE_COMPARISON_TASK, @@ -214,6 +216,8 @@ public enum AuditEventAction implements Translatable { STOP_APPLY_DATABASE_PERMISSION_TASK, + STOP_APPLY_TABLE_PERMISSION_TASK, + EXECUTE_ASYNC_TASK, EXECUTE_MOCKDATA_TASK, @@ -259,6 +263,8 @@ public enum AuditEventAction implements Translatable { APPROVE_APPLY_DATABASE_PERMISSION_TASK, + APPROVE_APPLY_TABLE_PERMISSION_TASK, + REJECT_ASYNC_TASK, REJECT_MOCKDATA_TASK, @@ -284,6 +290,8 @@ public enum AuditEventAction implements Translatable { REJECT_APPLY_DATABASE_PERMISSION_TASK, + REJECT_APPLY_TABLE_PERMISSION_TASK, + /** * 数据脱敏规则 */ @@ -350,6 +358,10 @@ public enum AuditEventAction implements Translatable { REVOKE_DATABASE_PERMISSION, + GRANT_TABLE_PERMISSION, + + REVOKE_TABLE_PERMISSION, + /** * Automation rule management */ diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/AuditEventType.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/AuditEventType.java index 60a9e8ae7f..865330a7ab 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/AuditEventType.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/AuditEventType.java @@ -64,6 +64,8 @@ public enum AuditEventType implements Translatable { APPLY_DATABASE_PERMISSION, + APPLY_TABLE_PERMISSION, + STRUCTURE_COMPARISON, DATA_MASKING_RULE, @@ -84,6 +86,8 @@ public enum AuditEventType implements Translatable { DATABASE_PERMISSION_MANAGEMENT, + TABLE_PERMISSION_MANAGEMENT, + ENVIRONMENT_MANAGEMENT, AUTOMATION_RULE_MANAGEMENT, diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ResourceType.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ResourceType.java index 6178b731b5..36e1075d11 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ResourceType.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ResourceType.java @@ -121,7 +121,8 @@ public enum ResourceType implements Translatable { ALIYUN_ACCOUNT, ALIYUN_SUB_ACCOUNT, - ODC_JOB; + ODC_JOB, + ODC_TABLE; @Override public String code() { diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/TaskType.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/TaskType.java index d5b63b762f..04f2c750ad 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/TaskType.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/TaskType.java @@ -91,6 +91,10 @@ public enum TaskType implements Translatable { * Apply database permission */ APPLY_DATABASE_PERMISSION, + /** + * Apply table permission + */ + APPLY_TABLE_PERMISSION, /** * Structure comparison */ diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages.properties index 006a7fa4dc..bf0e4ad8c1 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages.properties @@ -64,6 +64,7 @@ com.oceanbase.odc.ResourceType.ODC_PROJECT=Project com.oceanbase.odc.ResourceType.ODC_ENVIRONMENT=Environment com.oceanbase.odc.ResourceType.ODC_DATASOURCE=DataSource com.oceanbase.odc.ResourceType.ODC_DATABASE=Database +com.oceanbase.odc.ResourceType.ODC_TABLE=Table com.oceanbase.odc.ResourceType.ODC_RULESET=Ruleset com.oceanbase.odc.ResourceType.ODC_RULE=Rule com.oceanbase.odc.ResourceType.ODC_SENSITIVE_COLUMN=Sensitive Column @@ -205,6 +206,7 @@ com.oceanbase.odc.AuditEventAction.CREATE_ALTER_SCHEDULE_TASK=Create alter sched com.oceanbase.odc.AuditEventAction.CREATE_ONLINE_SCHEMA_CHANGE_TASK=Create online schema change task com.oceanbase.odc.AuditEventAction.CREATE_APPLY_PROJECT_PERMISSION_TASK=Create apply project permission task com.oceanbase.odc.AuditEventAction.CREATE_APPLY_DATABASE_PERMISSION_TASK=Create apply database permission task +com.oceanbase.odc.AuditEventAction.CREATE_APPLY_TABLE_PERMISSION_TASK=Create apply table permission task com.oceanbase.odc.AuditEventAction.STOP_ASYNC_TASK=Stop database Change task com.oceanbase.odc.AuditEventAction.STOP_MOCKDATA_TASK=Stop mock data task com.oceanbase.odc.AuditEventAction.STOP_IMPORT_TASK=Stop import task @@ -217,6 +219,7 @@ com.oceanbase.odc.AuditEventAction.STOP_ALTER_SCHEDULE_TASK=Stop alter schedule com.oceanbase.odc.AuditEventAction.STOP_ONLINE_SCHEMA_CHANGE_TASK=Stop online schema change task com.oceanbase.odc.AuditEventAction.STOP_APPLY_PROJECT_PERMISSION_TASK=Stop apply project permission task com.oceanbase.odc.AuditEventAction.STOP_APPLY_DATABASE_PERMISSION_TASK=Stop apply database permission task +com.oceanbase.odc.AuditEventAction.STOP_APPLY_TABLE_PERMISSION_TASK=Stop apply table permission task com.oceanbase.odc.AuditEventAction.EXECUTE_ASYNC_TASK=Execute database Change task com.oceanbase.odc.AuditEventAction.EXECUTE_MOCKDATA_TASK=Execute mock data task com.oceanbase.odc.AuditEventAction.EXECUTE_IMPORT_TASK=Execute import task @@ -240,6 +243,7 @@ com.oceanbase.odc.AuditEventAction.APPROVE_ALTER_SCHEDULE_TASK=Approve alter sch com.oceanbase.odc.AuditEventAction.APPROVE_ONLINE_SCHEMA_CHANGE_TASK=Approve online schema change task com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_PROJECT_PERMISSION_TASK=Approve apply project permission task com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_DATABASE_PERMISSION_TASK=Approve apply database permission task +com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_TABLE_PERMISSION_TASK=Approve apply table permission task com.oceanbase.odc.AuditEventAction.REJECT_ASYNC_TASK=Reject database Change task com.oceanbase.odc.AuditEventAction.REJECT_MOCKDATA_TASK=Reject mock data task com.oceanbase.odc.AuditEventAction.REJECT_IMPORT_TASK=Reject import task @@ -253,6 +257,7 @@ com.oceanbase.odc.AuditEventAction.REJECT_ALTER_SCHEDULE_TASK=Reject alter sched com.oceanbase.odc.AuditEventAction.REJECT_ONLINE_SCHEMA_CHANGE_TASK=Reject online schema change task com.oceanbase.odc.AuditEventAction.REJECT_APPLY_PROJECT_PERMISSION_TASK=Reject apply project permission task com.oceanbase.odc.AuditEventAction.REJECT_APPLY_DATABASE_PERMISSION_TASK=Reject apply database permission task +com.oceanbase.odc.AuditEventAction.REJECT_APPLY_TABLE_PERMISSION_TASK=Reject apply table permission task com.oceanbase.odc.AuditEventAction.CREATE_DATA_MASKING_RULE=Create data masking rule com.oceanbase.odc.AuditEventAction.UPDATE_DATA_MASKING_RULE=Update data masking rule com.oceanbase.odc.AuditEventAction.ENABLE_DATA_MASKING_RULE=Enable data masking rule @@ -274,6 +279,8 @@ com.oceanbase.odc.AuditEventAction.CREATE_PROJECT=Create project com.oceanbase.odc.AuditEventAction.UPDATE_SQL_SECURITY_RULE=Modify SQL security rule com.oceanbase.odc.AuditEventAction.GRANT_DATABASE_PERMISSION=Grant database permission com.oceanbase.odc.AuditEventAction.REVOKE_DATABASE_PERMISSION=Revoke database permission +com.oceanbase.odc.AuditEventAction.GRANT_TABLE_PERMISSION=Grant table permission +com.oceanbase.odc.AuditEventAction.REVOKE_TABLE_PERMISSION=Revoke table permission com.oceanbase.odc.AuditEventAction.CREATE_ENVIRONMENT=Create environment com.oceanbase.odc.AuditEventAction.UPDATE_ENVIRONMENT=Update environment com.oceanbase.odc.AuditEventAction.ENABLE_ENVIRONMENT=Enable environment @@ -319,6 +326,7 @@ com.oceanbase.odc.AuditEventType.ALTER_SCHEDULE=Alter schedule com.oceanbase.odc.AuditEventType.ONLINE_SCHEMA_CHANGE=Online schema change com.oceanbase.odc.AuditEventType.APPLY_PROJECT_PERMISSION=Apply project permission com.oceanbase.odc.AuditEventType.APPLY_DATABASE_PERMISSION=Apply database permission +com.oceanbase.odc.AuditEventType.APPLY_TABLE_PERMISSION=Apply table permission com.oceanbase.odc.AuditEventType.DATA_MASKING_RULE=Data masking rule com.oceanbase.odc.AuditEventType.DATA_MASKING_POLICY=Data masking policy com.oceanbase.odc.AuditEventType.PERMISSION_APPLY=Permission apply @@ -329,6 +337,7 @@ com.oceanbase.odc.AuditEventType.PROJECT_MANAGEMENT=Project management com.oceanbase.odc.AuditEventType.SQL_SECURITY_RULE_MANAGEMENT=SQL security rule management com.oceanbase.odc.AuditEventType.ENVIRONMENT_MANAGEMENT=Environment management com.oceanbase.odc.AuditEventType.DATABASE_PERMISSION_MANAGEMENT=Database permission management +com.oceanbase.odc.AuditEventType.TABLE_PERMISSION_MANAGEMENT=Table permission management com.oceanbase.odc.AuditEventType.AUTOMATION_RULE_MANAGEMENT=Automation rule management com.oceanbase.odc.AuditEventType.NOTIFICATION_MANAGEMENT=Notification management com.oceanbase.odc.AuditEventType.SENSITIVE_COLUMN_MANAGEMENT=Sensitive column management @@ -809,6 +818,7 @@ com.oceanbase.odc.TaskType.PRE_CHECK=Pre Check com.oceanbase.odc.TaskType.EXPORT_RESULT_SET=Export Result Set com.oceanbase.odc.TaskType.APPLY_PROJECT_PERMISSION=Apply Project Permission com.oceanbase.odc.TaskType.APPLY_DATABASE_PERMISSION=Apply Database Permission +com.oceanbase.odc.TaskType.APPLY_TABLE_PERMISSION=Apply Table Permission com.oceanbase.odc.TaskType.SQL_PLAN=Sql Plan com.oceanbase.odc.TaskType.DATA_ARCHIVE=Data Archive com.oceanbase.odc.TaskType.DATA_DELETE=Data Delete @@ -837,6 +847,7 @@ com.oceanbase.odc.notification.channel-test-message=【ODC】Channel verificatio # com.oceanbase.odc.builtin-resource.permission-apply.project.description=Requesting [{1}] permissions for project [{0}] com.oceanbase.odc.builtin-resource.permission-apply.database.description=Requesting [{0}] permissions for databases +com.oceanbase.odc.builtin-resource.permission-apply.table.description=Requesting [{0}] permissions for tables # # ResourceRoleName @@ -858,6 +869,7 @@ com.oceanbase.odc.generate-update-table-ddl-check.drop-and-create-index.message= com.oceanbase.odc.DatabasePermissionType.QUERY=Query com.oceanbase.odc.DatabasePermissionType.CHANGE=Change com.oceanbase.odc.DatabasePermissionType.EXPORT=Export +com.oceanbase.odc.DatabasePermissionType.ACCESS=Access com.oceanbase.odc.PartitionPlanVariableKey.INTERVAL=interval com.oceanbase.odc.PartitionPlanVariableKey.LAST_PARTITION_VALUE=The value of the positional expression corresponding to the last partition rule diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties index 90b9b0b0c6..90d9a5fbbc 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties @@ -64,6 +64,7 @@ com.oceanbase.odc.ResourceType.ODC_PROJECT=项目 com.oceanbase.odc.ResourceType.ODC_ENVIRONMENT=环境 com.oceanbase.odc.ResourceType.ODC_DATASOURCE=数据源 com.oceanbase.odc.ResourceType.ODC_DATABASE=数据库 +com.oceanbase.odc.ResourceType.ODC_TABLE=数据表 com.oceanbase.odc.ResourceType.ODC_RULESET=规则集 com.oceanbase.odc.ResourceType.ODC_RULE=规则 com.oceanbase.odc.ResourceType.ODC_SENSITIVE_COLUMN=敏感列 @@ -204,6 +205,7 @@ com.oceanbase.odc.AuditEventAction.CREATE_ALTER_SCHEDULE_TASK=创建修改调度 com.oceanbase.odc.AuditEventAction.CREATE_ONLINE_SCHEMA_CHANGE_TASK=创建无锁结构变更任务 com.oceanbase.odc.AuditEventAction.CREATE_APPLY_PROJECT_PERMISSION_TASK=创建申请项目权限任务 com.oceanbase.odc.AuditEventAction.CREATE_APPLY_DATABASE_PERMISSION_TASK=创建申请数据库权限任务 +com.oceanbase.odc.AuditEventAction.CREATE_APPLY_TABLE_PERMISSION_TASK=创建申请数据表权限任务 com.oceanbase.odc.AuditEventAction.STOP_ASYNC_TASK=停止数据库变更任务 com.oceanbase.odc.AuditEventAction.STOP_MOCKDATA_TASK=停止模拟数据任务 com.oceanbase.odc.AuditEventAction.STOP_IMPORT_TASK=停止导入任务 @@ -216,6 +218,7 @@ com.oceanbase.odc.AuditEventAction.STOP_ALTER_SCHEDULE_TASK=停止修改调度 com.oceanbase.odc.AuditEventAction.STOP_ONLINE_SCHEMA_CHANGE_TASK=停止无锁结构变更任务 com.oceanbase.odc.AuditEventAction.STOP_APPLY_PROJECT_PERMISSION_TASK=停止申请项目权限任务 com.oceanbase.odc.AuditEventAction.STOP_APPLY_DATABASE_PERMISSION_TASK=停止申请数据库权限任务 +com.oceanbase.odc.AuditEventAction.STOP_APPLY_TABLE_PERMISSION_TASK=停止申请数据表权限任务 com.oceanbase.odc.AuditEventAction.EXECUTE_ASYNC_TASK=执行数据库变更任务 com.oceanbase.odc.AuditEventAction.EXECUTE_MOCKDATA_TASK=执行模拟数据任务 com.oceanbase.odc.AuditEventAction.EXECUTE_IMPORT_TASK=执行导入任务 @@ -239,6 +242,7 @@ com.oceanbase.odc.AuditEventAction.APPROVE_ALTER_SCHEDULE_TASK=同意修改调 com.oceanbase.odc.AuditEventAction.APPROVE_ONLINE_SCHEMA_CHANGE_TASK=同意无锁结构变更任务 com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_PROJECT_PERMISSION_TASK=同意申请项目权限任务 com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_DATABASE_PERMISSION_TASK=同意申请数据库权限任务 +com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_TABLE_PERMISSION_TASK=同意申请数据表权限任务 com.oceanbase.odc.AuditEventAction.REJECT_ASYNC_TASK=拒绝数据库变更任务 com.oceanbase.odc.AuditEventAction.REJECT_MOCKDATA_TASK=拒绝模拟数据任务 com.oceanbase.odc.AuditEventAction.REJECT_IMPORT_TASK=拒绝导入任务 @@ -252,6 +256,7 @@ com.oceanbase.odc.AuditEventAction.REJECT_ALTER_SCHEDULE_TASK=拒绝修改调度 com.oceanbase.odc.AuditEventAction.REJECT_ONLINE_SCHEMA_CHANGE_TASK=拒绝无锁结构变更任务 com.oceanbase.odc.AuditEventAction.REJECT_APPLY_PROJECT_PERMISSION_TASK=拒绝申请项目权限任务 com.oceanbase.odc.AuditEventAction.REJECT_APPLY_DATABASE_PERMISSION_TASK=拒绝申请数据库权限任务 +com.oceanbase.odc.AuditEventAction.REJECT_APPLY_TABLE_PERMISSION_TASK=拒绝申请数据表权限任务 com.oceanbase.odc.AuditEventAction.CREATE_DATA_MASKING_RULE=创建脱敏规则 com.oceanbase.odc.AuditEventAction.UPDATE_DATA_MASKING_RULE=更新脱敏规则 com.oceanbase.odc.AuditEventAction.ENABLE_DATA_MASKING_RULE=启用脱敏规则 @@ -273,6 +278,8 @@ com.oceanbase.odc.AuditEventAction.CREATE_PROJECT=创建项目 com.oceanbase.odc.AuditEventAction.UPDATE_SQL_SECURITY_RULE=修改 SQL 安全规则 com.oceanbase.odc.AuditEventAction.GRANT_DATABASE_PERMISSION=新增库权限 com.oceanbase.odc.AuditEventAction.REVOKE_DATABASE_PERMISSION=回收库权限 +com.oceanbase.odc.AuditEventAction.GRANT_TABLE_PERMISSION=新增表权限 +com.oceanbase.odc.AuditEventAction.REVOKE_TABLE_PERMISSION=回收表权限 com.oceanbase.odc.AuditEventAction.CREATE_ENVIRONMENT=创建环境 com.oceanbase.odc.AuditEventAction.UPDATE_ENVIRONMENT=修改环境 @@ -322,6 +329,7 @@ com.oceanbase.odc.AuditEventType.ALTER_SCHEDULE=修改调度 com.oceanbase.odc.AuditEventType.ONLINE_SCHEMA_CHANGE=无锁结构变更 com.oceanbase.odc.AuditEventType.APPLY_PROJECT_PERMISSION=申请项目权限 com.oceanbase.odc.AuditEventType.APPLY_DATABASE_PERMISSION=申请数据库权限 +com.oceanbase.odc.AuditEventType.APPLY_TABLE_PERMISSION=申请数据表权限 com.oceanbase.odc.AuditEventType.DATA_MASKING_RULE=脱敏规则 com.oceanbase.odc.AuditEventType.DATA_MASKING_POLICY=脱敏策略 com.oceanbase.odc.AuditEventType.PERMISSION_APPLY=权限申请 @@ -331,6 +339,7 @@ com.oceanbase.odc.AuditEventType.DATASOURCE_MANAGEMENT=数据源管理 com.oceanbase.odc.AuditEventType.PROJECT_MANAGEMENT=项目管理 com.oceanbase.odc.AuditEventType.SQL_SECURITY_RULE_MANAGEMENT=SQL 安全规则管理 com.oceanbase.odc.AuditEventType.DATABASE_PERMISSION_MANAGEMENT=库权限管理 +com.oceanbase.odc.AuditEventType.TABLE_PERMISSION_MANAGEMENT=表权限管理 com.oceanbase.odc.AuditEventType.ENVIRONMENT_MANAGEMENT=环境管理 com.oceanbase.odc.AuditEventType.AUTOMATION_RULE_MANAGEMENT=自动授权规则管理 com.oceanbase.odc.AuditEventType.NOTIFICATION_MANAGEMENT=消息推送管理 @@ -744,6 +753,7 @@ com.oceanbase.odc.TaskType.PRE_CHECK=预检查 com.oceanbase.odc.TaskType.EXPORT_RESULT_SET=导出结果集 com.oceanbase.odc.TaskType.APPLY_PROJECT_PERMISSION=申请项目权限 com.oceanbase.odc.TaskType.APPLY_DATABASE_PERMISSION=申请数据库权限 +com.oceanbase.odc.TaskType.APPLY_TABLE_PERMISSION=申请数据表权限 com.oceanbase.odc.TaskType.SQL_PLAN=SQL 计划 com.oceanbase.odc.TaskType.DATA_ARCHIVE=数据归档 com.oceanbase.odc.TaskType.DATA_DELETE=数据清理 @@ -772,6 +782,7 @@ com.oceanbase.odc.notification.channel-test-message=【ODC】消息通道验证 # com.oceanbase.odc.builtin-resource.permission-apply.project.description=申请项目【{0}】的【{1}】权限 com.oceanbase.odc.builtin-resource.permission-apply.database.description=申请数据库的【{0}】权限 +com.oceanbase.odc.builtin-resource.permission-apply.table.description=申请数据表的【{0}】权限 # # ResourceRoleName @@ -793,6 +804,7 @@ com.oceanbase.odc.generate-update-table-ddl-check.drop-and-create-index.message= com.oceanbase.odc.DatabasePermissionType.QUERY=查询 com.oceanbase.odc.DatabasePermissionType.CHANGE=变更 com.oceanbase.odc.DatabasePermissionType.EXPORT=导出 +com.oceanbase.odc.DatabasePermissionType.ACCESS=访问 com.oceanbase.odc.PartitionPlanVariableKey.INTERVAL=间隔 com.oceanbase.odc.PartitionPlanVariableKey.LAST_PARTITION_VALUE=最后一个分区规则对应位置表达式的值 diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties index a10926cace..f379475be9 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties @@ -64,6 +64,7 @@ com.oceanbase.odc.ResourceType.ODC_PROJECT=項目 com.oceanbase.odc.ResourceType.ODC_ENVIRONMENT=環境 com.oceanbase.odc.ResourceType.ODC_DATASOURCE=數據源 com.oceanbase.odc.ResourceType.ODC_DATABASE=數據庫 +com.oceanbase.odc.ResourceType.ODC_TABLE=數據表 com.oceanbase.odc.ResourceType.ODC_RULESET=規則集 com.oceanbase.odc.ResourceType.ODC_RULE=規則 com.oceanbase.odc.ResourceType.ODC_SENSITIVE_COLUMN=敏感列 @@ -206,6 +207,7 @@ com.oceanbase.odc.AuditEventAction.CREATE_ALTER_SCHEDULE_TASK=創建修改調度 com.oceanbase.odc.AuditEventAction.CREATE_ONLINE_SCHEMA_CHANGE_TASK=創建無鎖結構變更任務 com.oceanbase.odc.AuditEventAction.CREATE_APPLY_PROJECT_PERMISSION_TASK=創建申請項目權限任務 com.oceanbase.odc.AuditEventAction.CREATE_APPLY_DATABASE_PERMISSION_TASK=創建申請數據庫權限任務 +com.oceanbase.odc.AuditEventAction.CREATE_APPLY_TABLE_PERMISSION_TASK=創建申請數據表權限任務 com.oceanbase.odc.AuditEventAction.STOP_ASYNC_TASK=停止數據庫變更任務 com.oceanbase.odc.AuditEventAction.STOP_MOCKDATA_TASK=停止模擬數據任務 com.oceanbase.odc.AuditEventAction.STOP_IMPORT_TASK=停止導入任務 @@ -218,6 +220,7 @@ com.oceanbase.odc.AuditEventAction.STOP_ALTER_SCHEDULE_TASK=停止修改調度 com.oceanbase.odc.AuditEventAction.STOP_ONLINE_SCHEMA_CHANGE_TASK=停止無鎖結構變更任務 com.oceanbase.odc.AuditEventAction.STOP_APPLY_PROJECT_PERMISSION_TASK=停止申請項目權限任務 com.oceanbase.odc.AuditEventAction.STOP_APPLY_DATABASE_PERMISSION_TASK=停止申請數據庫權限任務 +com.oceanbase.odc.AuditEventAction.STOP_APPLY_TABLE_PERMISSION_TASK=停止申請數據表權限任務 com.oceanbase.odc.AuditEventAction.EXECUTE_ASYNC_TASK=執行執行數據庫變更任務 com.oceanbase.odc.AuditEventAction.EXECUTE_MOCKDATA_TASK=執行模擬數據任務 com.oceanbase.odc.AuditEventAction.EXECUTE_IMPORT_TASK=執行導入任務 @@ -241,6 +244,7 @@ com.oceanbase.odc.AuditEventAction.APPROVE_ALTER_SCHEDULE_TASK=同意修改調 com.oceanbase.odc.AuditEventAction.APPROVE_ONLINE_SCHEMA_CHANGE_TASK=同意無鎖結構變更任務 com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_PROJECT_PERMISSION_TASK=同意申請項目權限任務 com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_DATABASE_PERMISSION_TASK=同意申請數據庫權限任務 +com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_TABLE_PERMISSION_TASK=同意申請數據表權限任務 com.oceanbase.odc.AuditEventAction.REJECT_ASYNC_TASK=拒絕數據庫變更任務 com.oceanbase.odc.AuditEventAction.REJECT_MOCKDATA_TASK=拒絕模擬數據任務 com.oceanbase.odc.AuditEventAction.REJECT_IMPORT_TASK=拒絕導入任務 @@ -254,6 +258,7 @@ com.oceanbase.odc.AuditEventAction.REJECT_ALTER_SCHEDULE_TASK=拒絕修改調度 com.oceanbase.odc.AuditEventAction.REJECT_ONLINE_SCHEMA_CHANGE_TASK=拒絕無鎖結構變更任務 com.oceanbase.odc.AuditEventAction.REJECT_APPLY_PROJECT_PERMISSION_TASK=拒絕申請項目權限任務 com.oceanbase.odc.AuditEventAction.REJECT_APPLY_DATABASE_PERMISSION_TASK=拒絕申請數據庫權限任務 +com.oceanbase.odc.AuditEventAction.REJECT_APPLY_TABLE_PERMISSION_TASK=拒絕申請數據表權限任務 com.oceanbase.odc.AuditEventAction.CREATE_DATA_MASKING_RULE=創建脫敏規則 com.oceanbase.odc.AuditEventAction.UPDATE_DATA_MASKING_RULE=更新脫敏規則 com.oceanbase.odc.AuditEventAction.ENABLE_DATA_MASKING_RULE=啓用脫敏規則 @@ -275,6 +280,8 @@ com.oceanbase.odc.AuditEventAction.CREATE_PROJECT=創建项目 com.oceanbase.odc.AuditEventAction.UPDATE_SQL_SECURITY_RULE=修改 SQL 安全規則 com.oceanbase.odc.AuditEventAction.GRANT_DATABASE_PERMISSION=新增庫權限 com.oceanbase.odc.AuditEventAction.REVOKE_DATABASE_PERMISSION=回收庫權限 +com.oceanbase.odc.AuditEventAction.GRANT_TABLE_PERMISSION=新增表權限 +com.oceanbase.odc.AuditEventAction.REVOKE_TABLE_PERMISSION=回收表權限 com.oceanbase.odc.AuditEventAction.CREATE_ENVIRONMENT=創建環境 com.oceanbase.odc.AuditEventAction.UPDATE_ENVIRONMENT=更新環境 com.oceanbase.odc.AuditEventAction.ENABLE_ENVIRONMENT=啓用環境 @@ -323,6 +330,7 @@ com.oceanbase.odc.AuditEventType.STRUCTURE_COMPARISON=結構比對 com.oceanbase.odc.AuditEventType.ONLINE_SCHEMA_CHANGE=無鎖結構變更 com.oceanbase.odc.AuditEventType.APPLY_PROJECT_PERMISSION=申請項目權限 com.oceanbase.odc.AuditEventType.APPLY_DATABASE_PERMISSION=申請數據庫權限 +com.oceanbase.odc.AuditEventType.APPLY_TABLE_PERMISSION=申請數據表權限 com.oceanbase.odc.AuditEventType.DATA_MASKING_RULE=脫敏規則 com.oceanbase.odc.AuditEventType.DATA_MASKING_POLICY=脫敏策略 com.oceanbase.odc.AuditEventType.PERMISSION_APPLY=權限申請 @@ -332,6 +340,7 @@ com.oceanbase.odc.AuditEventType.DATASOURCE_MANAGEMENT=数据源管理 com.oceanbase.odc.AuditEventType.PROJECT_MANAGEMENT=项目管理 com.oceanbase.odc.AuditEventType.SQL_SECURITY_RULE_MANAGEMENT=SQL 安全規則管理 com.oceanbase.odc.AuditEventType.DATABASE_PERMISSION_MANAGEMENT=庫權限管理 +com.oceanbase.odc.AuditEventType.TABLE_PERMISSION_MANAGEMENT=表權限管理 com.oceanbase.odc.AuditEventType.ENVIRONMENT_MANAGEMENT=環境管理 com.oceanbase.odc.AuditEventType.AUTOMATION_RULE_MANAGEMENT=自動授權規則管理 com.oceanbase.odc.AuditEventType.NOTIFICATION_MANAGEMENT=消息推送管理 @@ -813,6 +822,7 @@ com.oceanbase.odc.TaskType.PRE_CHECK=預檢查 com.oceanbase.odc.TaskType.EXPORT_RESULT_SET=導出結果集 com.oceanbase.odc.TaskType.APPLY_PROJECT_PERMISSION=申請項目權限 com.oceanbase.odc.TaskType.APPLY_DATABASE_PERMISSION=申請數據庫權限 +com.oceanbase.odc.TaskType.APPLY_TABLE_PERMISSION=申請數據表權限 com.oceanbase.odc.TaskType.SQL_PLAN=SQL 計劃 com.oceanbase.odc.TaskType.DATA_ARCHIVE=數據歸檔 com.oceanbase.odc.TaskType.DATA_DELETE=數據清理 @@ -841,6 +851,7 @@ com.oceanbase.odc.notification.channel-test-message=【ODC】消息通道驗證 # com.oceanbase.odc.builtin-resource.permission-apply.project.description=申請項目【{0}】的【{1}】權限 com.oceanbase.odc.builtin-resource.permission-apply.database.description=申請數據庫的【{0}】權限 +com.oceanbase.odc.builtin-resource.permission-apply.table.description=申請數據表的【{0}】權限 # # ResourceRoleName @@ -862,6 +873,7 @@ com.oceanbase.odc.generate-update-table-ddl-check.drop-and-create-index.message= com.oceanbase.odc.DatabasePermissionType.QUERY=查詢 com.oceanbase.odc.DatabasePermissionType.CHANGE=變更 com.oceanbase.odc.DatabasePermissionType.EXPORT=導出 +com.oceanbase.odc.DatabasePermissionType.ACCESS=訪問 com.oceanbase.odc.PartitionPlanVariableKey.INTERVAL=間隔 com.oceanbase.odc.PartitionPlanVariableKey.LAST_PARTITION_VALUE=最後一個分區規則對應位置表達式的值 diff --git a/server/odc-migrate/src/main/resources/init-config/default-audit-event-meta.yml b/server/odc-migrate/src/main/resources/init-config/default-audit-event-meta.yml index 7cace7468f..20e9ad9a19 100644 --- a/server/odc-migrate/src/main/resources/init-config/default-audit-event-meta.yml +++ b/server/odc-migrate/src/main/resources/init-config/default-audit-event-meta.yml @@ -512,4 +512,20 @@ sid_extract_expression: "" in_connection: 0 enabled: 1 - id: 95 \ No newline at end of file + id: 95 + +# TABLE_PERMISSION_MANAGEMENT +- type: "TABLE_PERMISSION_MANAGEMENT" + action: "GRANT_TABLE_PERMISSION" + method_signature: "com.oceanbase.odc.server.web.controller.v2.TablePermissionController.batchCreate" + sid_extract_expression: "" + in_connection: 0 + enabled: 1 + id: 96 +- type: "TABLE_PERMISSION_MANAGEMENT" + action: "REVOKE_TABLE_PERMISSION" + method_signature: "com.oceanbase.odc.server.web.controller.v2.TablePermissionController.batchRevoke" + sid_extract_expression: "" + in_connection: 0 + enabled: 1 + id: 97 \ No newline at end of file diff --git a/server/odc-migrate/src/main/resources/init-config/init/notification-policy-metadata.yaml b/server/odc-migrate/src/main/resources/init-config/init/notification-policy-metadata.yaml index b2794ca209..697d210224 100644 --- a/server/odc-migrate/src/main/resources/init-config/init/notification-policy-metadata.yaml +++ b/server/odc-migrate/src/main/resources/init-config/init/notification-policy-metadata.yaml @@ -353,4 +353,28 @@ - id: 89 event_category: "TASK" event_name: "${com.oceanbase.odc.TaskType.STRUCTURE_COMPARISON}-${com.oceanbase.odc.event.TASK.APPROVAL_REJECTION.name}" - match_expression: "taskType.equals('STRUCTURE_COMPARISON') && taskStatus.equals('APPROVAL_REJECTION')" \ No newline at end of file + match_expression: "taskType.equals('STRUCTURE_COMPARISON') && taskStatus.equals('APPROVAL_REJECTION')" +- id: 90 + event_category: "TASK" + event_name: "${com.oceanbase.odc.TaskType.APPLY_TABLE_PERMISSION}-${com.oceanbase.odc.event.TASK.EXECUTION_FAILED.name}" + match_expression: "taskType.equals('APPLY_TABLE_PERMISSION') && taskStatus.equals('EXECUTION_FAILED')" +- id: 91 + event_category: "TASK" + event_name: "${com.oceanbase.odc.TaskType.APPLY_TABLE_PERMISSION}-${com.oceanbase.odc.event.TASK.EXECUTION_TIMEOUT.name}" + match_expression: "taskType.equals('APPLY_TABLE_PERMISSION') && taskStatus.equals('EXECUTION_TIMEOUT')" +- id: 92 + event_category: "TASK" + event_name: "${com.oceanbase.odc.TaskType.APPLY_TABLE_PERMISSION}-${com.oceanbase.odc.event.TASK.PENDING_APPROVAL.name}" + match_expression: "taskType.equals('APPLY_TABLE_PERMISSION') && taskStatus.equals('PENDING_APPROVAL')" +- id: 93 + event_category: "TASK" + event_name: "${com.oceanbase.odc.TaskType.APPLY_TABLE_PERMISSION}-${com.oceanbase.odc.event.TASK.APPROVED.name}" + match_expression: "taskType.equals('APPLY_TABLE_PERMISSION') && taskStatus.equals('APPROVED')" +- id: 94 + event_category: "TASK" + event_name: "${com.oceanbase.odc.TaskType.APPLY_TABLE_PERMISSION}-${com.oceanbase.odc.event.TASK.APPROVAL_REJECTION.name}" + match_expression: "taskType.equals('APPLY_TABLE_PERMISSION') && taskStatus.equals('APPROVAL_REJECTION')" +- id: 95 + event_category: "TASK" + event_name: "${com.oceanbase.odc.TaskType.APPLY_TABLE_PERMISSION}-${com.oceanbase.odc.event.TASK.EXECUTION_SUCCEEDED.name}" + match_expression: "taskType.equals('APPLY_TABLE_PERMISSION') && taskStatus.equals('EXECUTION_SUCCEEDED')" \ No newline at end of file diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_4_3_1_1__alter_iam_permission.sql b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_1_1__alter_iam_permission.sql new file mode 100644 index 0000000000..33869949e5 --- /dev/null +++ b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_1_1__alter_iam_permission.sql @@ -0,0 +1,7 @@ +-- +-- Add columns in `iam_permission` table for table-level permission management +-- +alter table `iam_permission` add column `resource_type` varchar(64) default null comment 'Refer to ResourceType, optional values: ODC_DATABASE, ODC_TABLE, ODC_COLUMN, etc.'; +alter table `iam_permission` add column `resource_id` bigint(20) default null comment 'ID of the resource'; +alter table `iam_permission` add key `idx_iam_permission_resource_type_id` (resource_type, resource_id); +update `iam_permission` set `resource_type` = 'ODC_DATABASE', `resource_id` = substring( `resource_identifier` from 14 ) where `resource_identifier` like 'ODC_DATABASE:%' and `action` in ('query', 'change', 'export'); diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_4_3_1_2__add_list_user_table_permission_view.sql b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_1_2__add_list_user_table_permission_view.sql new file mode 100644 index 0000000000..398a0e6d4f --- /dev/null +++ b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_1_2__add_list_user_table_permission_view.sql @@ -0,0 +1,85 @@ +-- +-- optimize `list_user_database_permission_view` view for query user database permissions list +-- +create or replace view `list_user_database_permission_view` as +select + u_p.`id` as `id`, + u_p.`user_id` as `user_id`, + u_p.`action` as `action`, + u_p.`authorization_type` as `authorization_type`, + u_p.`ticket_id` as `ticket_id`, + u_p.`create_time` as `create_time`, + u_p.`expire_time` as `expire_time`, + u_p.`creator_id` as `creator_id`, + u_p.`organization_id` as `organization_id`, + c_d.`project_id` as `project_id`, + c_d.`id` as `database_id`, + c_d.`name` as `database_name`, + c_c.`id` as `data_source_id`, + c_c.`name` as `data_source_name` +from + ( + select + i_p.`id` as `id`, + i_up.`user_id` as `user_id`, + i_p.`action` as `action`, + i_p.`authorization_type` as `authorization_type`, + i_p.`ticket_id` as `ticket_id`, + i_p.`create_time` as `create_time`, + i_p.`expire_time` as `expire_time`, + i_p.`creator_id` as `creator_id`, + i_p.`organization_id` as `organization_id`, + i_p.`resource_id` as `resource_id` + from + `iam_permission` as i_p + inner join `iam_user_permission` as i_up on i_p.`id` = i_up.`permission_id` + where + i_p.`resource_type` = 'ODC_DATABASE' + ) as u_p + inner join `connect_database` as c_d on u_p.`resource_id` = c_d.`id` + inner join `connect_connection` as c_c on c_d.`connection_id` = c_c.`id`; + + +-- +-- Add `list_user_table_permission_view` view for query user table permissions list +-- +create or replace view `list_user_table_permission_view` as +select + u_p.`id` as `id`, + u_p.`user_id` as `user_id`, + u_p.`action` as `action`, + u_p.`authorization_type` as `authorization_type`, + u_p.`ticket_id` as `ticket_id`, + u_p.`create_time` as `create_time`, + u_p.`expire_time` as `expire_time`, + u_p.`creator_id` as `creator_id`, + u_p.`organization_id` as `organization_id`, + c_d.`project_id` as `project_id`, + d_so.`id` as `table_id`, + d_so.`name` as `table_name`, + c_d.`id` as `database_id`, + c_d.`name` as `database_name`, + c_c.`id` as `data_source_id`, + c_c.`name` as `data_source_name` +from + ( + select + i_p.`id` as `id`, + i_up.`user_id` as `user_id`, + i_p.`action` as `action`, + i_p.`authorization_type` as `authorization_type`, + i_p.`ticket_id` as `ticket_id`, + i_p.`create_time` as `create_time`, + i_p.`expire_time` as `expire_time`, + i_p.`creator_id` as `creator_id`, + i_p.`organization_id` as `organization_id`, + i_p.`resource_id` as `resource_id` + from + `iam_permission` as i_p + inner join `iam_user_permission` as i_up on i_p.`id` = i_up.`permission_id` + where + i_p.`resource_type` = 'ODC_TABLE' + ) as u_p + inner join `database_schema_object` as d_so on u_p.`resource_id` = d_so.`id` + inner join `connect_database` as c_d on d_so.`database_id` = c_d.`id` + inner join `connect_connection` as c_c on c_d.`connection_id` = c_c.`id`; diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/DBSchemaController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/DBSchemaControllerV1.java similarity index 98% rename from server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/DBSchemaController.java rename to server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/DBSchemaControllerV1.java index bd69c72264..d213affdd9 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/DBSchemaController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/DBSchemaControllerV1.java @@ -38,7 +38,7 @@ */ @RestController @RequestMapping("/api/v1/database") -public class DBSchemaController { +public class DBSchemaControllerV1 { @Autowired private DBSchemaService databaseService; diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBSchemaController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBSchemaController.java new file mode 100644 index 0000000000..24d908a351 --- /dev/null +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBSchemaController.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.server.web.controller.v2; + +import java.sql.SQLException; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import com.oceanbase.odc.service.common.response.ListResponse; +import com.oceanbase.odc.service.common.response.Responses; +import com.oceanbase.odc.service.connection.table.TableService; +import com.oceanbase.odc.service.connection.table.model.QueryTableParams; +import com.oceanbase.odc.service.connection.table.model.Table; + +import io.swagger.annotations.ApiOperation; + +/** + * + * @Author: fenghao + * @Create 2024/3/19 11:13 + * @Version 1.0 + */ +@RestController +@RequestMapping("/api/v2/databaseSchema") +public class DBSchemaController { + + @Autowired + private TableService tableService; + + @ApiOperation(value = "listTables", notes = "List tables with permitted actions") + @RequestMapping(value = "/tables", method = RequestMethod.GET) + public ListResponse list(@RequestParam(name = "databaseId") Long databaseId, + @RequestParam(name = "includePermittedAction", required = false, + defaultValue = "false") boolean includePermittedAction) + throws SQLException, InterruptedException { + QueryTableParams params = QueryTableParams.builder() + .databaseId(databaseId) + .includePermittedAction(includePermittedAction) + .build(); + return Responses.list(tableService.list(params)); + } + +} diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/FlowInstanceController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/FlowInstanceController.java index be649fb310..b9a7465ef7 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/FlowInstanceController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/FlowInstanceController.java @@ -87,8 +87,7 @@ public class FlowInstanceController { @ApiOperation(value = "createFlowInstance", notes = "创建流程实例，返回流程实例") @RequestMapping(value = "/", method = RequestMethod.POST) - public ListResponse createFlowInstance( - @RequestBody CreateFlowInstanceReq flowInstanceReq) { + public ListResponse createFlowInstance(@RequestBody CreateFlowInstanceReq flowInstanceReq) { flowInstanceReq.validate(); if (authenticationFacade.currentUser().getOrganizationType() == OrganizationType.INDIVIDUAL) { return Responses.list(flowInstanceService.createIndividualFlowInstance(flowInstanceReq)); diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/TablePermissionController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/TablePermissionController.java new file mode 100644 index 0000000000..d344022e64 --- /dev/null +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/TablePermissionController.java @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.server.web.controller.v2; + +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort.Direction; +import org.springframework.data.web.PageableDefault; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import com.oceanbase.odc.core.shared.constant.AuthorizationType; +import com.oceanbase.odc.service.common.response.ListResponse; +import com.oceanbase.odc.service.common.response.PaginatedResponse; +import com.oceanbase.odc.service.common.response.Responses; +import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; +import com.oceanbase.odc.service.permission.database.model.ExpirationStatusFilter; +import com.oceanbase.odc.service.permission.table.TablePermissionService; +import com.oceanbase.odc.service.permission.table.model.CreateTablePermissionReq; +import com.oceanbase.odc.service.permission.table.model.QueryTablePermissionParams; +import com.oceanbase.odc.service.permission.table.model.UserTablePermission; + +import io.swagger.annotations.ApiOperation; + +/** + * ClassName: TablePermissionController.java Package: com.oceanbase.odc.server.web.controller.v2 + * Description: + * + * @Author: fenghao + * @Create 2024/3/11 20:26 + * @Version 1.0 + */ +@RestController +@RequestMapping("/api/v2/collaboration/projects/{projectId:[\\d]+}/tablePermissions") +public class TablePermissionController { + + @Autowired + private TablePermissionService service; + + @ApiOperation(value = "listTablePermissions", notes = "List table permissions") + @RequestMapping(value = "", method = RequestMethod.GET) + public PaginatedResponse list(@PathVariable Long projectId, + @RequestParam(name = "userId", required = false) Long userId, + @RequestParam(name = "ticketId", required = false) Long ticketId, + @RequestParam(name = "tableName", required = false) String fuzzyTableName, + @RequestParam(name = "databaseName", required = false) String fuzzyDatabaseName, + @RequestParam(name = "dataSourceName", required = false) String fuzzyDataSourceName, + @RequestParam(name = "type", required = false) List types, + @RequestParam(name = "authorizationType", required = false) AuthorizationType authorizationType, + @RequestParam(name = "status", required = false) List statuses, + @PageableDefault(size = Integer.MAX_VALUE, sort = {"id"}, direction = Direction.DESC) Pageable pageable) { + QueryTablePermissionParams params = QueryTablePermissionParams.builder() + .userId(userId) + .ticketId(ticketId) + .fuzzyTableName(fuzzyTableName) + .fuzzyDatabaseName(fuzzyDatabaseName) + .fuzzyDataSourceName(fuzzyDataSourceName) + .types(types) + .authorizationType(authorizationType) + .statuses(statuses) + .build(); + return Responses.paginated(service.list(projectId, params, pageable)); + } + + @ApiOperation(value = "batchCreateTablePermissions", notes = "Batch create table permissions") + @RequestMapping(value = "/batchCreate", method = RequestMethod.POST) + public ListResponse batchCreate(@PathVariable Long projectId, + @RequestBody CreateTablePermissionReq req) { + return Responses.list(service.batchCreate(projectId, req)); + } + + @ApiOperation(value = "batchRevokeTablePermission", notes = "Batch revoke table permissions") + @RequestMapping(value = "/batchRevoke", method = RequestMethod.POST) + public ListResponse batchRevoke(@PathVariable Long projectId, + @RequestBody List ids) { + return Responses.list(service.batchRevoke(projectId, ids)); + } + +} diff --git a/server/odc-server/src/main/resources/log4j2.xml b/server/odc-server/src/main/resources/log4j2.xml index e26694a248..585c54ba14 100644 --- a/server/odc-server/src/main/resources/log4j2.xml +++ b/server/odc-server/src/main/resources/log4j2.xml @@ -358,6 +358,55 @@ + + + + + + + %d{yyyy-MM-dd HH:mm:ss} %p %c{1.} - %m%n + + + + + + + + + + + + + + + + + + + + %d{yyyy-MM-dd HH:mm:ss} %p %c{1.} - %m%n + + + + + + + + + + + + + + + + + + @@ -730,6 +779,12 @@ + + + + + + diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/dbobject/DBObjectRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/dbobject/DBObjectRepository.java index 2933d2cb5b..5859c07152 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/dbobject/DBObjectRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/dbobject/DBObjectRepository.java @@ -38,6 +38,8 @@ public interface DBObjectRepository extends OdcJpaRepository findByDatabaseIdAndType(Long databaseId, DBObjectType type); + List findByDatabaseIdInAndType(Collection databaseIds, DBObjectType type); + @Modifying @Transactional @Query(value = "delete from database_schema_object t where t.id in (:ids)", nativeQuery = true) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/PermissionEntity.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/PermissionEntity.java index 1caa1e9e5b..eb8fbe50c2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/PermissionEntity.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/PermissionEntity.java @@ -28,12 +28,14 @@ import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Table; +import javax.print.DocFlavor.STRING; import org.hibernate.annotations.Where; import com.oceanbase.odc.core.shared.PermissionConfiguration; import com.oceanbase.odc.core.shared.constant.AuthorizationType; import com.oceanbase.odc.core.shared.constant.PermissionType; +import com.oceanbase.odc.core.shared.constant.ResourceType; import lombok.AllArgsConstructor; import lombok.Data; @@ -95,6 +97,13 @@ public class PermissionEntity implements PermissionConfiguration { @Column(name = "ticket_id") private Long ticketId; + @Enumerated(value = EnumType.STRING) + @Column(name = "resource_type") + private ResourceType resourceType; + + @Column(name = "resource_id") + private Long resourceId; + @Override public String resourceIdentifier() { return this.resourceIdentifier; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/PermissionRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/PermissionRepository.java index b1a5a330bd..c2a0856ec7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/PermissionRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/PermissionRepository.java @@ -30,6 +30,7 @@ import com.oceanbase.odc.common.jpa.InsertSqlTemplateBuilder; import com.oceanbase.odc.config.jpa.OdcJpaRepository; import com.oceanbase.odc.core.shared.constant.PermissionType; +import com.oceanbase.odc.core.shared.constant.ResourceType; /** * Find all PermissionEntity for a specific User @@ -82,6 +83,8 @@ List findByUserIdsAndOrganizationId(@Param("userIds") Collecti List findByOrganizationId(Long organizationId); + List findByResourceTypeAndResourceIdIn(ResourceType resourceType, Collection resourceIds); + List findByOrganizationIdAndResourceIdentifier(Long organizationId, String resourceIdentifier); Optional findByOrganizationIdAndActionAndResourceIdentifier(Long organizationId, String action, @@ -115,6 +118,8 @@ default List batchCreate(List entities) { .field(PermissionEntity_.expireTime) .field(PermissionEntity_.authorizationType) .field(PermissionEntity_.ticketId) + .field(PermissionEntity_.resourceType) + .field(PermissionEntity_.resourceId) .build(); List> getter = valueGetterBuilder() .add(PermissionEntity::getAction) @@ -126,7 +131,10 @@ default List batchCreate(List entities) { .add(PermissionEntity::getExpireTime) .add((PermissionEntity e) -> e.getAuthorizationType().name()) .add(PermissionEntity::getTicketId) + .add((PermissionEntity e) -> e.getResourceType().name()) + .add(PermissionEntity::getResourceId) .build(); + return batchCreate(entities, sql, getter, PermissionEntity::setId); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserDatabasePermissionSpec.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserDatabasePermissionSpec.java index 249205f7ba..b0ea73ccbd 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserDatabasePermissionSpec.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserDatabasePermissionSpec.java @@ -73,9 +73,8 @@ public static Specification typeIn(Collection authorizationTypeEqual( - AuthorizationType authorizationTypes) { - return SpecificationUtil.columnEqual(UserDatabasePermissionEntity_.AUTHORIZATION_TYPE, authorizationTypes); + public static Specification authorizationTypeEqual(AuthorizationType type) { + return SpecificationUtil.columnEqual(UserDatabasePermissionEntity_.AUTHORIZATION_TYPE, type); } public static Specification filterByExpirationStatus( diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserTablePermissionEntity.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserTablePermissionEntity.java new file mode 100644 index 0000000000..1764a214e9 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserTablePermissionEntity.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.metadb.iam; + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import com.oceanbase.odc.core.shared.constant.AuthorizationType; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * + * @Author: fenghao + * @Create 2024/3/11 20:26 + * @Version 1.0 + */ +@Data +@Entity +@Table(name = "list_user_table_permission_view") +@EqualsAndHashCode(exclude = {"createTime"}) +public class UserTablePermissionEntity { + + @Id + @Column(name = "id", nullable = false, updatable = false) + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(name = "user_id") + private Long userId; + + @Column(name = "action") + private String action; + + @Column(name = "authorization_type") + @Enumerated(EnumType.STRING) + private AuthorizationType authorizationType; + + @Column(name = "ticket_id") + private Long ticketId; + + @Column(name = "create_time") + private Date createTime; + + @Column(name = "expire_time") + private Date expireTime; + + @Column(name = "creator_id") + private Long creatorId; + + @Column(name = "organization_id") + private Long organizationId; + + @Column(name = "project_id") + private Long projectId; + + @Column(name = "database_id") + private Long databaseId; + + @Column(name = "database_name") + private String databaseName; + + @Column(name = "data_source_id") + private Long dataSourceId; + + @Column(name = "data_source_name") + private String dataSourceName; + + @Column(name = "table_id") + private Long tableId; + + @Column(name = "table_name") + private String tableName; + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserTablePermissionRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserTablePermissionRepository.java new file mode 100644 index 0000000000..f71ba5c8b2 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserTablePermissionRepository.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.metadb.iam; + +import java.util.Collection; +import java.util.List; + +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +import com.oceanbase.odc.metadb.flow.ReadOnlyRepository; + +/** + * + * @Author: fenghao + * @Create 2024/3/11 20:26 + * @Version 1.0 + */ + +public interface UserTablePermissionRepository extends ReadOnlyRepository { + + List findByProjectIdAndIdIn(Long projectId, Collection ids); + + List findByDatabaseIdIn(Collection databaseIds); + + @Query(value = "select v.* from list_user_table_permission_view v where v.expire_time > now() " + + "and v.user_id = :userId and v.database_id in (:databaseIds)", nativeQuery = true) + List findNotExpiredByUserIdAndDatabaseIdIn(@Param("userId") Long userId, + @Param("databaseIds") Collection databaseIds); + + @Query(value = "select v.* from list_user_table_permission_view v where v.expire_time > now() " + + "and v.user_id = :userId and v.table_id in (:tableIds)", nativeQuery = true) + List findNotExpiredByUserIdAndTableIdIn(@Param("userId") Long userId, + @Param("tableIds") Collection tableIds); + + @Query(value = "select v.* from list_user_table_permission_view v where v.user_id = :userId " + + "and v.project_id = :projectId", nativeQuery = true) + List findByUserIdAndProjectId(@Param("userId") Long userId, + @Param("projectId") Long projectId); + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserTablePermissionSpec.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserTablePermissionSpec.java new file mode 100644 index 0000000000..574b68627d --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserTablePermissionSpec.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.metadb.iam; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang.time.DateUtils; +import org.springframework.data.jpa.domain.Specification; + +import com.oceanbase.odc.common.jpa.SpecificationUtil; +import com.oceanbase.odc.core.shared.constant.AuthorizationType; +import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; +import com.oceanbase.odc.service.permission.database.model.ExpirationStatusFilter; + +/** + * @author gaoda.xy + * @date 2024/1/11 17:49 + */ +public class UserTablePermissionSpec { + + private static final int EXPIRING_DAYS = 7; + + public static Specification projectIdEqual(Long projectId) { + return SpecificationUtil.columnEqual(UserTablePermissionEntity_.PROJECT_ID, projectId); + } + + public static Specification userIdEqual(Long userId) { + return SpecificationUtil.columnEqual(UserTablePermissionEntity_.USER_ID, userId); + } + + public static Specification organizationIdEqual(Long organizationId) { + return SpecificationUtil.columnEqual(UserTablePermissionEntity_.ORGANIZATION_ID, organizationId); + } + + public static Specification ticketIdEqual(Long ticketId) { + return SpecificationUtil.columnEqual(UserTablePermissionEntity_.TICKET_ID, ticketId); + } + + public static Specification databaseNameLike(String fuzzyDatabaseName) { + return SpecificationUtil.columnLike(UserTablePermissionEntity_.DATABASE_NAME, fuzzyDatabaseName); + } + + public static Specification tableNameLike(String fuzzyTableName) { + return SpecificationUtil.columnLike(UserTablePermissionEntity_.TABLE_NAME, fuzzyTableName); + } + + public static Specification dataSourceNameLike(String fuzzyDataSourceName) { + return SpecificationUtil.columnLike(UserTablePermissionEntity_.DATA_SOURCE_NAME, fuzzyDataSourceName); + } + + public static Specification dataSourceId(Long datasourceId) { + return SpecificationUtil.columnEqual(UserTablePermissionEntity_.DATA_SOURCE_ID, datasourceId); + } + + public static Specification typeIn(Collection types) { + Set actions = new HashSet<>(); + if (CollectionUtils.isNotEmpty(types)) { + types.forEach(type -> { + actions.add(type.getAction()); + }); + } + return SpecificationUtil.columnIn(UserTablePermissionEntity_.ACTION, actions); + } + + public static Specification authorizationTypeEqual(AuthorizationType type) { + return SpecificationUtil.columnEqual(UserTablePermissionEntity_.AUTHORIZATION_TYPE, type); + } + + public static Specification filterByExpirationStatus( + List statuses, Date expireTimeThreshold) { + if (CollectionUtils.isEmpty(statuses)) { + return (root, query, builder) -> builder.conjunction(); + } + List> expireSpecList = new ArrayList<>(); + for (ExpirationStatusFilter status : statuses) { + expireSpecList.add(getByExpirationStatusFilter(status, expireTimeThreshold)); + } + Specification expireSpec = expireSpecList.get(0); + for (int i = 1; i < expireSpecList.size(); i++) { + expireSpec = expireSpec.or(expireSpecList.get(i)); + } + return expireSpec; + } + + private static Specification getByExpirationStatusFilter( + ExpirationStatusFilter status, Date date) { + switch (status) { + case EXPIRED: + return expireTimeBefore(date); + case EXPIRING: + return expireTimeLate(date).and(expireTimeBefore(DateUtils.addDays(date, EXPIRING_DAYS))); + case NOT_EXPIRED: + return expireTimeLate(date); + default: + throw new IllegalArgumentException("Unknown status: " + status); + } + } + + private static Specification expireTimeBefore(Date date) { + return SpecificationUtil.columnBefore(UserTablePermissionEntity_.EXPIRE_TIME, date); + } + + private static Specification expireTimeLate(Date date) { + return SpecificationUtil.columnLate(UserTablePermissionEntity_.EXPIRE_TIME, date); + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/resourcerole/ResourceRoleRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/resourcerole/ResourceRoleRepository.java index 0729fe723f..1a33f77fb9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/resourcerole/ResourceRoleRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/resourcerole/ResourceRoleRepository.java @@ -28,5 +28,4 @@ public interface ResourceRoleRepository List findByResourceType(ResourceType resourceType); List findByResourceTypeIn(List resourceType); - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/resourcerole/UserResourceRoleRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/resourcerole/UserResourceRoleRepository.java index 3e98adb9ba..cfeb1a7437 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/resourcerole/UserResourceRoleRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/resourcerole/UserResourceRoleRepository.java @@ -123,5 +123,4 @@ default List batchCreate(List en .build(); return batchCreate(entities, sql, getter, UserResourceRoleEntity::setId); } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/audit/AuditEventMetaService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/audit/AuditEventMetaService.java index 499dbda650..fa41097315 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/audit/AuditEventMetaService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/audit/AuditEventMetaService.java @@ -84,7 +84,8 @@ public List listAllAuditEventMeta(@NotNull QueryAuditEventMetaPa AuditEventType.PARTITION_PLAN, AuditEventType.ALTER_SCHEDULE, AuditEventType.APPLY_PROJECT_PERMISSION, - AuditEventType.APPLY_DATABASE_PERMISSION))); + AuditEventType.APPLY_DATABASE_PERMISSION, + AuditEventType.APPLY_TABLE_PERMISSION))); continue; } if (AuditEventAction.OTHERS == entity.getAction()) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/audit/util/AuditUtils.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/audit/util/AuditUtils.java index cb2ea54a48..6a904b4685 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/audit/util/AuditUtils.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/audit/util/AuditUtils.java @@ -77,6 +77,9 @@ public static AuditEventType getEventTypeFromTaskType(TaskType taskType) { case APPLY_DATABASE_PERMISSION: type = AuditEventType.APPLY_DATABASE_PERMISSION; break; + case APPLY_TABLE_PERMISSION: + type = AuditEventType.APPLY_TABLE_PERMISSION; + break; case STRUCTURE_COMPARISON: type = AuditEventType.STRUCTURE_COMPARISON; break; @@ -181,6 +184,8 @@ public static AuditEventAction getActualActionForTask(AuditEventType type, Audit return AuditEventAction.CREATE_APPLY_PROJECT_PERMISSION_TASK; case APPLY_DATABASE_PERMISSION: return AuditEventAction.CREATE_APPLY_DATABASE_PERMISSION_TASK; + case APPLY_TABLE_PERMISSION: + return AuditEventAction.CREATE_APPLY_TABLE_PERMISSION_TASK; } } if (action == AuditEventAction.STOP_TASK) { @@ -209,6 +214,8 @@ public static AuditEventAction getActualActionForTask(AuditEventType type, Audit return AuditEventAction.STOP_APPLY_PROJECT_PERMISSION_TASK; case APPLY_DATABASE_PERMISSION: return AuditEventAction.STOP_APPLY_DATABASE_PERMISSION_TASK; + case APPLY_TABLE_PERMISSION: + return AuditEventAction.STOP_APPLY_TABLE_PERMISSION_TASK; } } if (action == AuditEventAction.EXECUTE_TASK) { @@ -263,6 +270,8 @@ public static AuditEventAction getActualActionForTask(AuditEventType type, Audit return AuditEventAction.APPROVE_APPLY_PROJECT_PERMISSION_TASK; case APPLY_DATABASE_PERMISSION: return AuditEventAction.APPROVE_APPLY_DATABASE_PERMISSION_TASK; + case APPLY_TABLE_PERMISSION: + return AuditEventAction.APPROVE_APPLY_TABLE_PERMISSION_TASK; } } if (action == AuditEventAction.REJECT) { @@ -293,6 +302,8 @@ public static AuditEventAction getActualActionForTask(AuditEventType type, Audit return AuditEventAction.REJECT_APPLY_PROJECT_PERMISSION_TASK; case APPLY_DATABASE_PERMISSION: return AuditEventAction.REJECT_APPLY_DATABASE_PERMISSION_TASK; + case APPLY_TABLE_PERMISSION: + return AuditEventAction.REJECT_APPLY_TABLE_PERMISSION_TASK; } } // 如果不是流程相关的 action，则返回原值 diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java index 44bc1a7b66..585d6f008d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java @@ -63,6 +63,8 @@ import com.oceanbase.odc.metadb.iam.UserEntity; import com.oceanbase.odc.metadb.iam.UserPermissionRepository; import com.oceanbase.odc.metadb.iam.UserRepository; +import com.oceanbase.odc.metadb.iam.UserTablePermissionEntity; +import com.oceanbase.odc.metadb.iam.UserTablePermissionRepository; import com.oceanbase.odc.metadb.iam.resourcerole.ResourceRoleEntity; import com.oceanbase.odc.metadb.iam.resourcerole.ResourceRoleRepository; import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleEntity; @@ -126,6 +128,9 @@ public class ProjectService { @Autowired private UserDatabasePermissionRepository userDatabasePermissionRepository; + @Autowired + private UserTablePermissionRepository userTablePermissionRepository; + @Autowired private PermissionRepository permissionRepository; @@ -353,6 +358,7 @@ public boolean deleteProjectMember(@NonNull Long projectId, @NonNull Long userId } checkMemberRoles(detail(projectId).getMembers()); deleteMemberRelatedDatabasePermissions(userId, projectId); + deleteMemberRelatedTablePermissions(userId, projectId); return true; } @@ -420,8 +426,8 @@ public Map> getProjectId2ResourceRoleNames() { @SkipAuthorize("internal usage") public Set getMemberProjectIds(Long userId) { - return resourceRoleService.listByUserId(userId).stream().map(UserResourceRole::getResourceId) - .collect(Collectors.toSet()); + return resourceRoleService.listByUserId(userId).stream().filter(UserResourceRole::isProjectMember) + .map(UserResourceRole::getResourceId).collect(Collectors.toSet()); } @SkipAuthorize("odc internal usage") @@ -494,7 +500,16 @@ private void checkMemberRoles(@NonNull List members) { private void deleteMemberRelatedDatabasePermissions(@NonNull Long userId, @NonNull Long projectId) { List permissionIds = userDatabasePermissionRepository.findByUserIdAndProjectId(userId, projectId).stream() .map(UserDatabasePermissionEntity::getId).collect(Collectors.toList()); - if (!CollectionUtils.isEmpty(permissionIds)) { + if (CollectionUtils.isNotEmpty(permissionIds)) { + permissionRepository.deleteByIds(permissionIds); + userPermissionRepository.deleteByPermissionIds(permissionIds); + } + } + + private void deleteMemberRelatedTablePermissions(@NonNull Long userId, @NonNull Long projectId) { + List permissionIds = userTablePermissionRepository.findByUserIdAndProjectId(userId, projectId).stream() + .map(UserTablePermissionEntity::getId).collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(permissionIds)) { permissionRepository.deleteByIds(permissionIds); userPermissionRepository.deleteByPermissionIds(permissionIds); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java index ead3c2c436..d82ce1f217 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java @@ -28,7 +28,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -65,7 +64,6 @@ import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.shared.PreConditions; -import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.constant.OrganizationType; import com.oceanbase.odc.core.shared.constant.ResourceRoleName; @@ -83,6 +81,8 @@ import com.oceanbase.odc.metadb.iam.UserDatabasePermissionEntity; import com.oceanbase.odc.metadb.iam.UserDatabasePermissionRepository; import com.oceanbase.odc.metadb.iam.UserPermissionRepository; +import com.oceanbase.odc.metadb.iam.UserTablePermissionEntity; +import com.oceanbase.odc.metadb.iam.UserTablePermissionRepository; import com.oceanbase.odc.service.collaboration.environment.EnvironmentService; import com.oceanbase.odc.service.collaboration.environment.model.Environment; import com.oceanbase.odc.service.collaboration.project.ProjectService; @@ -115,9 +115,8 @@ import com.oceanbase.odc.service.onlineschemachange.ddl.OscDBAccessor; import com.oceanbase.odc.service.onlineschemachange.ddl.OscDBAccessorFactory; import com.oceanbase.odc.service.onlineschemachange.rename.OscDBUserUtil; -import com.oceanbase.odc.service.permission.database.DatabasePermissionHelper; +import com.oceanbase.odc.service.permission.DBResourcePermissionHelper; import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; -import com.oceanbase.odc.service.permission.database.model.UnauthorizedDatabase; import com.oceanbase.odc.service.plugin.SchemaPluginUtil; import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; import com.oceanbase.odc.service.session.factory.OBConsoleDataSourceFactory; @@ -141,15 +140,6 @@ public class DatabaseService { private final DatabaseMapper databaseMapper = DatabaseMapper.INSTANCE; - private static final Set ORACLE_DATA_DICTIONARY = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - - private static final Set MYSQL_DATA_DICTIONARY = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - - static { - ORACLE_DATA_DICTIONARY.add("SYS"); - MYSQL_DATA_DICTIONARY.add("information_schema"); - } - @Autowired private DatabaseRepository databaseRepository; @@ -186,6 +176,9 @@ public class DatabaseService { @Autowired private UserDatabasePermissionRepository userDatabasePermissionRepository; + @Autowired + private UserTablePermissionRepository userTablePermissionRepository; + @Autowired private PermissionRepository permissionRepository; @@ -199,10 +192,7 @@ public class DatabaseService { private DBColumnRepository dbColumnRepository; @Autowired - private DatabasePermissionHelper databasePermissionHelper; - - @Autowired - private SecurityManager securityManager; + private DBResourcePermissionHelper permissionHelper; @Autowired private ResourceRoleService resourceRoleService; @@ -210,6 +200,9 @@ public class DatabaseService { @Autowired private UserService userService; + @Autowired + private SecurityManager securityManager; + @Autowired private DBSchemaSyncTaskManager dbSchemaSyncTaskManager; @@ -437,8 +430,7 @@ public boolean transfer(@NonNull @Valid TransferDatabasesReq req) { checkTransferable(entities, req); Set databaseIds = entities.stream().map(DatabaseEntity::getId).collect(Collectors.toSet()); databaseRepository.setProjectIdByIdIn(req.getProjectId(), databaseIds); - deleteDatabasePermissionByIds(databaseIds); - resourceRoleService.deleteByResourceTypeAndIdIn(ResourceType.ODC_DATABASE, databaseIds); + deleteDatabaseRelatedPermissionByIds(databaseIds); List userResourceRoles = buildUserResourceRoles(databaseIds, req.getOwnerIds()); resourceRoleService.saveAll(userResourceRoles); return true; @@ -458,7 +450,7 @@ public boolean deleteDatabases(@NonNull DeleteDatabasesReq req) { } saved.forEach(database -> checkPermission(database.getProjectId(), database.getConnectionId())); Set databaseIds = saved.stream().map(DatabaseEntity::getId).collect(Collectors.toSet()); - deleteDatabasePermissionByIds(databaseIds); + deleteDatabaseRelatedPermissionByIds(databaseIds); dbColumnRepository.deleteByDatabaseIdIn(req.getDatabaseIds()); dbObjectRepository.deleteByDatabaseIdIn(req.getDatabaseIds()); databaseRepository.deleteAll(saved); @@ -722,7 +714,7 @@ public int deleteByDataSourceIds(@NonNull Set dataSourceId) { if (CollectionUtils.isEmpty(databaseIds)) { return 0; } - deleteDatabasePermissionByIds(databaseIds); + deleteDatabaseRelatedPermissionByIds(databaseIds); dbColumnRepository.deleteByDatabaseIdIn(databaseIds); dbObjectRepository.deleteByDatabaseIdIn(databaseIds); return databaseRepository.deleteByConnectionIds(dataSourceId); @@ -736,77 +728,17 @@ public int deleteByDataSourceId(@NonNull Long dataSourceId) { if (CollectionUtils.isEmpty(databaseIds)) { return 0; } - deleteDatabasePermissionByIds(databaseIds); + deleteDatabaseRelatedPermissionByIds(databaseIds); dbColumnRepository.deleteByDatabaseIdIn(databaseIds); dbObjectRepository.deleteByDatabaseIdIn(databaseIds); return databaseRepository.deleteByConnectionId(dataSourceId); } - @SkipAuthorize("odc internal usage") - public List filterUnauthorizedDatabases( - Map> schemaName2PermissionTypes, @NotNull Long dataSourceId, - boolean ignoreDataDirectory) { - if (schemaName2PermissionTypes == null || schemaName2PermissionTypes.isEmpty()) { - return Collections.emptyList(); - } - ConnectionConfig dataSource = connectionService.getBasicWithoutPermissionCheck(dataSourceId); - List databases = listDatabasesByConnectionIds(Collections.singleton(dataSourceId)); - databases.forEach(d -> d.getDataSource().setName(dataSource.getName())); - Map name2Database = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - databases.forEach(d -> name2Database.put(d.getName(), d)); - Map> id2Types = databasePermissionHelper - .getPermissions(databases.stream().map(Database::getId).collect(Collectors.toList())); - List unauthorizedDatabases = new ArrayList<>(); - Set involvedProjectIds = projectService.getMemberProjectIds(authenticationFacade.currentUserId()); - for (Map.Entry> entry : schemaName2PermissionTypes.entrySet()) { - String schemaName = entry.getKey(); - Set needs = entry.getValue(); - if (CollectionUtils.isEmpty(needs)) { - continue; - } - if (name2Database.containsKey(schemaName)) { - Database database = name2Database.get(schemaName); - boolean applicable = - database.getProject() != null && involvedProjectIds.contains(database.getProject().getId()); - Set authorized = id2Types.get(database.getId()); - if (CollectionUtils.isEmpty(authorized)) { - unauthorizedDatabases.add(UnauthorizedDatabase.from(database, needs, applicable)); - } else { - Set unauthorized = - needs.stream().filter(p -> !authorized.contains(p)).collect(Collectors.toSet()); - if (CollectionUtils.isNotEmpty(unauthorized)) { - unauthorizedDatabases.add(UnauthorizedDatabase.from(database, unauthorized, applicable)); - } - } - } else { - Database unknownDatabase = new Database(); - unknownDatabase.setName(schemaName); - unknownDatabase.setDataSource(dataSource); - unauthorizedDatabases.add(UnauthorizedDatabase.from(unknownDatabase, needs, false)); - } - } - if (ignoreDataDirectory) { - DialectType dialectType = dataSource.getDialectType(); - if (dialectType != null) { - if (dialectType.isOracle()) { - unauthorizedDatabases = - unauthorizedDatabases.stream().filter(d -> !ORACLE_DATA_DICTIONARY.contains(d.getName())) - .collect(Collectors.toList()); - } else if (dialectType.isMysql()) { - unauthorizedDatabases = unauthorizedDatabases.stream() - .filter(d -> !MYSQL_DATA_DICTIONARY.contains(d.getName())) - .collect(Collectors.toList()); - } - } - } - return unauthorizedDatabases; - } - @SkipAuthorize("internal usage") public List getAllAuthorizedDatabases(@NonNull Long dataSourceId) { List databases = listDatabasesByConnectionIds(Collections.singleton(dataSourceId)); - Map> id2Types = databasePermissionHelper - .getPermissions(databases.stream().map(Database::getId).collect(Collectors.toList())); + Map> id2Types = permissionHelper + .getDBPermissions(databases.stream().map(Database::getId).collect(Collectors.toList())); return databases.stream().map(d -> new AuthorizedDatabase(d.getId(), d.getName(), id2Types.get(d.getId()))) .collect(Collectors.toList()); } @@ -855,7 +787,6 @@ public boolean modifyDatabasesOwners(@NotNull Long projectId, @NotNull @Valid Mo return true; } - @SkipAuthorize("odc internal usage") @Transactional(rollbackFor = Exception.class) public void updateObjectSyncStatus(@NotNull Collection databaseIds, @NotNull DBObjectSyncStatus status) { if (CollectionUtils.isEmpty(databaseIds)) { @@ -940,7 +871,7 @@ private Page entitiesToModels(Page entities, boolean i Map> databaseId2PermittedActions = new HashMap<>(); Set databaseIds = entities.stream().map(DatabaseEntity::getId).collect(Collectors.toSet()); if (includesPermittedAction) { - databaseId2PermittedActions = databasePermissionHelper.getPermissions(databaseIds); + databaseId2PermittedActions = permissionHelper.getDBPermissions(databaseIds); } Map> finalId2PermittedActions = databaseId2PermittedActions; Map> databaseId2UserResourceRole = new HashMap<>(); @@ -970,6 +901,7 @@ private Page entitiesToModels(Page entities, boolean i if (includesPermittedAction) { database.setAuthorizedPermissionTypes(finalId2PermittedActions.get(entity.getId())); } + // Set the owner of the database List resourceRoles = finalDatabaseId2UserResourceRole.get(entity.getId()); if (CollectionUtils.isNotEmpty(resourceRoles)) { @@ -998,7 +930,7 @@ private Database entityToModel(DatabaseEntity entity, boolean includesPermittedA model.setEnvironment(environmentService.detailSkipPermissionCheck(model.getDataSource().getEnvironmentId())); if (includesPermittedAction) { model.setAuthorizedPermissionTypes( - databasePermissionHelper.getPermissions(Collections.singleton(entity.getId())).get(entity.getId())); + permissionHelper.getDBPermissions(Collections.singleton(entity.getId())).get(entity.getId())); } return model; } @@ -1011,13 +943,18 @@ private void createDatabase(CreateDatabaseReq req, Connection conn, ConnectionCo SchemaPluginUtil.getDatabaseExtension(connection.getDialectType()).create(conn, db, connection.getPassword()); } - private void deleteDatabasePermissionByIds(Collection ids) { + private void deleteDatabaseRelatedPermissionByIds(Collection ids) { if (CollectionUtils.isEmpty(ids)) { return; } - List entities = userDatabasePermissionRepository.findByDatabaseIdIn(ids); - List permissionIds = - entities.stream().map(UserDatabasePermissionEntity::getId).collect(Collectors.toList()); + List permissionIds = userDatabasePermissionRepository.findByDatabaseIdIn(ids).stream() + .map(UserDatabasePermissionEntity::getId).collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(permissionIds)) { + permissionRepository.deleteByIds(permissionIds); + userPermissionRepository.deleteByPermissionIds(permissionIds); + } + permissionIds = userTablePermissionRepository.findByDatabaseIdIn(ids).stream() + .map(UserTablePermissionEntity::getId).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(permissionIds)) { permissionRepository.deleteByIds(permissionIds); userPermissionRepository.deleteByPermissionIds(permissionIds); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/DBResource.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/DBResource.java new file mode 100644 index 0000000000..3d1c0d9bb5 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/DBResource.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection.database.model; + +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.core.shared.constant.ResourceType; +import com.oceanbase.odc.service.connection.model.ConnectionConfig; + +import lombok.Data; + +/** + * @author gaoda.xy + * @date 2024/1/4 17:12 + */ +@Data +public class DBResource { + + private ResourceType type; + private DialectType dialectType; + private Long dataSourceId; + private String dataSourceName; + private Long databaseId; + private String databaseName; + private Long tableId; + private String tableName; + + public static DBResource from(ConnectionConfig dataSource, String databaseName, String tableName) { + DBResource obj = new DBResource(); + obj.setDataSourceId(dataSource.getId()); + obj.setDataSourceName(dataSource.getName()); + obj.setDialectType(dataSource.getDialectType()); + obj.setDatabaseName(databaseName); + obj.setTableName(tableName); + if (databaseName != null) { + obj.setType(ResourceType.ODC_DATABASE); + } + if (tableName != null) { + obj.setType(ResourceType.ODC_TABLE); + } + return obj; + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/Database.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/Database.java index 1217a8cfd7..29d342f5aa 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/Database.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/Database.java @@ -88,6 +88,7 @@ public class Database implements SecurityResource, OrganizationIsolated, Seriali @JsonProperty(access = Access.READ_ONLY) private Set authorizedPermissionTypes; + @JsonProperty(access = Access.READ_ONLY) private DBObjectSyncStatus objectSyncStatus; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/UnauthorizedDatabase.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/UnauthorizedDBResource.java similarity index 51% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/UnauthorizedDatabase.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/UnauthorizedDBResource.java index 43025f2a97..fa5d07f513 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/UnauthorizedDatabase.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/UnauthorizedDBResource.java @@ -13,35 +13,39 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.permission.database.model; +package com.oceanbase.odc.service.connection.database.model; import java.util.Set; -import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; import lombok.Data; import lombok.EqualsAndHashCode; /** - * @author gaoda.xy - * @date 2024/1/4 17:12 + * + * @Author: fenghao + * @Create 2024/3/20 13:54 + * @Version 1.0 */ @Data @EqualsAndHashCode(callSuper = true) -public class UnauthorizedDatabase extends Database { - - private static final long serialVersionUID = 2659094834615671659L; - - private Set unauthorizedPermissionTypes; +public class UnauthorizedDBResource extends DBResource { private Boolean applicable; + private Set unauthorizedPermissionTypes; - public static UnauthorizedDatabase from(Database database, Set types, boolean applicable) { - UnauthorizedDatabase obj = new UnauthorizedDatabase(); - obj.setId(database.getId()); - obj.setName(database.getName()); - obj.setDataSource(database.getDataSource()); - obj.setProject(database.getProject()); + public static UnauthorizedDBResource from(DBResource dbResource, Set types, + boolean applicable) { + UnauthorizedDBResource obj = new UnauthorizedDBResource(); + obj.setType(dbResource.getType()); + obj.setDialectType(dbResource.getDialectType()); + obj.setDataSourceId(dbResource.getDataSourceId()); + obj.setDataSourceName(dbResource.getDataSourceName()); + obj.setDatabaseId(dbResource.getDatabaseId()); + obj.setDatabaseName(dbResource.getDatabaseName()); + obj.setTableId(dbResource.getTableId()); + obj.setTableName(dbResource.getTableName()); obj.setUnauthorizedPermissionTypes(types); obj.setApplicable(applicable); return obj; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java new file mode 100644 index 0000000000..2186e72131 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection.table; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.stream.Collectors; + +import javax.validation.Valid; + +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.integration.jdbc.lock.JdbcLockRegistry; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.validation.annotation.Validated; + +import com.oceanbase.odc.core.authority.util.SkipAuthorize; +import com.oceanbase.odc.core.datasource.SingleConnectionDataSource; +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.core.shared.constant.ErrorCodes; +import com.oceanbase.odc.core.shared.constant.OrganizationType; +import com.oceanbase.odc.core.shared.constant.ResourceType; +import com.oceanbase.odc.core.shared.exception.ConflictException; +import com.oceanbase.odc.core.shared.exception.UnsupportedException; +import com.oceanbase.odc.metadb.dbobject.DBObjectEntity; +import com.oceanbase.odc.metadb.dbobject.DBObjectRepository; +import com.oceanbase.odc.plugin.schema.api.TableExtensionPoint; +import com.oceanbase.odc.service.connection.database.DatabaseService; +import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.connection.table.model.QueryTableParams; +import com.oceanbase.odc.service.connection.table.model.Table; +import com.oceanbase.odc.service.db.schema.DBSchemaSyncService; +import com.oceanbase.odc.service.db.schema.syncer.object.DBTableSyncer; +import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; +import com.oceanbase.odc.service.permission.DBResourcePermissionHelper; +import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; +import com.oceanbase.odc.service.plugin.SchemaPluginUtil; +import com.oceanbase.odc.service.session.factory.OBConsoleDataSourceFactory; +import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; + +import lombok.NonNull; + +/** + * + * @Author: fenghao + * @Create 2024/3/12 21:21 + * @Version 1.0 + */ +@Service +@Validated +public class TableService { + + @Autowired + private DatabaseService databaseService; + + @Autowired + private AuthenticationFacade authenticationFacade; + + @Autowired + private DBObjectRepository dbObjectRepository; + + @Autowired + private DBTableSyncer dbTableSyncer; + + @Autowired + private JdbcLockRegistry lockRegistry; + + @Autowired + private DBSchemaSyncService dbSchemaSyncService; + + @Autowired + private DBResourcePermissionHelper dbResourcePermissionHelper; + + @Transactional(rollbackFor = Exception.class) + @SkipAuthorize("permission check inside") + public List
list(@NonNull @Valid QueryTableParams params) throws SQLException, InterruptedException { + Database database = databaseService.detail(params.getDatabaseId()); + ConnectionConfig dataSource = database.getDataSource(); + OBConsoleDataSourceFactory factory = new OBConsoleDataSourceFactory(dataSource, true); + try (SingleConnectionDataSource ds = (SingleConnectionDataSource) factory.getDataSource(); + Connection conn = ds.getConnection()) { + TableExtensionPoint point = SchemaPluginUtil.getTableExtension(dataSource.getDialectType()); + Set latestTableNames = point.list(conn, database.getName()) + .stream().map(DBObjectIdentity::getName).collect(Collectors.toSet()); + if (authenticationFacade.currentUser().getOrganizationType() == OrganizationType.INDIVIDUAL) { + return latestTableNames.stream().map(tableName -> { + Table table = new Table(); + table.setName(tableName); + table.setAuthorizedPermissionTypes(new HashSet<>(DatabasePermissionType.all())); + return table; + }).collect(Collectors.toList()); + } + List tables = + dbObjectRepository.findByDatabaseIdAndType(params.getDatabaseId(), DBObjectType.TABLE); + Set existTableNames = tables.stream().map(DBObjectEntity::getName).collect(Collectors.toSet()); + if (latestTableNames.size() != existTableNames.size() || !existTableNames.containsAll(latestTableNames)) { + syncDBTables(conn, database, dataSource.getDialectType()); + tables = dbObjectRepository.findByDatabaseIdAndType(params.getDatabaseId(), DBObjectType.TABLE); + } + return entitiesToModels(tables, database, params.getIncludePermittedAction()); + } + } + + private void syncDBTables(Connection connection, Database database, DialectType dialectType) + throws InterruptedException { + Lock lock = lockRegistry + .obtain(dbSchemaSyncService.getSyncDBObjectLockKey(database.getDataSource().getId(), database.getId())); + if (!lock.tryLock(3, TimeUnit.SECONDS)) { + throw new ConflictException(ErrorCodes.ResourceSynchronizing, + new Object[] {ResourceType.ODC_TABLE.getLocalizedMessage()}, "Can not acquire jdbc lock"); + } + try { + if (dbTableSyncer.supports(dialectType)) { + dbTableSyncer.sync(connection, database, dialectType); + } else { + throw new UnsupportedException("Unsupported dialect type: " + dialectType); + } + } finally { + lock.unlock(); + } + } + + private List
entitiesToModels(Collection entities, Database database, + boolean includePermittedAction) { + List
tables = new ArrayList<>(); + if (CollectionUtils.isEmpty(entities)) { + return tables; + } + Map> id2Types = dbResourcePermissionHelper + .getTablePermissions(entities.stream().map(DBObjectEntity::getId).collect(Collectors.toSet())); + for (DBObjectEntity entity : entities) { + Table table = new Table(); + table.setId(entity.getId()); + table.setName(entity.getName()); + table.setDatabase(database); + table.setCreateTime(entity.getCreateTime()); + table.setUpdateTime(entity.getUpdateTime()); + table.setOrganizationId(entity.getOrganizationId()); + if (includePermittedAction) { + table.setAuthorizedPermissionTypes(id2Types.get(entity.getId())); + } + tables.add(table); + } + return tables; + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/QueryTableParams.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/QueryTableParams.java new file mode 100644 index 0000000000..78fe0e5eb7 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/QueryTableParams.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection.table.model; + +import javax.validation.constraints.NotNull; + +import lombok.Builder; +import lombok.Data; + +/** + * @author gaoda.xy + * @date 2024/4/28 19:17 + */ +@Data +@Builder +public class QueryTableParams { + + @NotNull + private Long databaseId; + @NotNull + private Boolean includePermittedAction; + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/Table.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/Table.java new file mode 100644 index 0000000000..449946b7a6 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/Table.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection.table.model; + +import java.io.Serializable; +import java.util.Date; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty.Access; +import com.oceanbase.odc.core.authority.model.SecurityResource; +import com.oceanbase.odc.core.shared.OrganizationIsolated; +import com.oceanbase.odc.core.shared.constant.ResourceType; +import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; + +import lombok.Data; + +/** + * + * @Author: fenghao + * @Create 2024/3/12 21:22 + * @Version 1.0 + */ +@Data +public class Table implements SecurityResource, OrganizationIsolated, Serializable { + + private static final long serialVersionUID = -3902642503472738041L; + + @JsonProperty(access = Access.READ_ONLY) + private Long id; + + @JsonProperty(access = Access.READ_ONLY) + private String name; + + @JsonProperty(access = Access.READ_ONLY) + private Database database; + + @JsonProperty(access = Access.READ_ONLY) + private Date createTime; + + @JsonProperty(access = Access.READ_ONLY) + private Date updateTime; + + @JsonProperty(access = Access.READ_ONLY) + private Long organizationId; + + @JsonProperty(access = Access.READ_ONLY) + private Set authorizedPermissionTypes; + + @Override + public String resourceId() { + return this.id == null ? null : this.id.toString(); + } + + @Override + public String resourceType() { + return ResourceType.ODC_TABLE.name(); + } + + @Override + public Long organizationId() { + return this.organizationId; + } + + @Override + public Long id() { + return this.id; + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLService.java index a7c86a5474..944153b03e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLService.java @@ -74,7 +74,7 @@ import com.oceanbase.odc.service.db.util.OBOracleCallFunctionBlockCallBack; import com.oceanbase.odc.service.db.util.OBOracleCallProcedureBlockCallBack; import com.oceanbase.odc.service.db.util.OBOracleCompilePLCallBack; -import com.oceanbase.odc.service.permission.database.DatabasePermissionHelper; +import com.oceanbase.odc.service.permission.DBResourcePermissionHelper; import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; import com.oceanbase.odc.service.session.ConnectConsoleService; import com.oceanbase.odc.service.session.SessionProperties; @@ -105,7 +105,7 @@ public class DBPLService { @Autowired private DatabaseRepository databaseRepository; @Autowired - private DatabasePermissionHelper databasePermissionHelper; + private DBResourcePermissionHelper permissionHelper; private static final Integer DEFAULT_MAX_CONCURRENT_BATCH_COMPILE_TASK_COUNT = 10; private final DefaultSqlExecuteTaskManager taskManager; @@ -242,7 +242,7 @@ public DBPLObjectIdentity parsePLNameType(@NonNull ConnectionSession session, @N public String callProcedure(@NonNull ConnectionSession session, @NonNull CallProcedureReq req) { Long databaseId = getDatabaseIdByConnectionSession(session); - databasePermissionHelper.checkPermissions(Collections.singleton(databaseId), + permissionHelper.checkDBPermissions(Collections.singleton(databaseId), Collections.singleton(DatabasePermissionType.CHANGE)); ConnectionCallback callback; DialectType dialectType = session.getDialectType(); @@ -277,7 +277,7 @@ public T getAsyncCallingResult(@NonNull ConnectionSession session, public String callFunction(@NonNull ConnectionSession session, @NonNull CallFunctionReq req) { Long databaseId = getDatabaseIdByConnectionSession(session); - databasePermissionHelper.checkPermissions(Collections.singleton(databaseId), + permissionHelper.checkDBPermissions(Collections.singleton(databaseId), Collections.singleton(DatabasePermissionType.CHANGE)); ConnectionCallback callback; DialectType dialectType = session.getDialectType(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncService.java index a5d96b13d4..2d00606786 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncService.java @@ -79,7 +79,7 @@ public void init() { public boolean sync(@NonNull Database database) throws InterruptedException, SQLException { PreConditions.notNull(database.getDataSource(), "database.dataSource"); Long dataSourceId = database.getDataSource().getId(); - Lock lock = jdbcLockRegistry.obtain("sync-datasource-" + dataSourceId + "-database-" + database.getId()); + Lock lock = jdbcLockRegistry.obtain(getSyncDBObjectLockKey(dataSourceId, database.getId())); if (!lock.tryLock(3, TimeUnit.SECONDS)) { throw new ConflictException(ErrorCodes.ResourceModifying, "Can not acquire jdbc lock"); } @@ -109,4 +109,8 @@ public boolean sync(@NonNull Database database) throws InterruptedException, SQL } } + public String getSyncDBObjectLockKey(@NonNull Long dataSourceId, @NonNull Long databaseId) { + return "sync-datasource-" + dataSourceId + "-database-" + databaseId; + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/AbstractDBObjectSyncer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/AbstractDBObjectSyncer.java index 2b412aeb5f..81451338c5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/AbstractDBObjectSyncer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/AbstractDBObjectSyncer.java @@ -42,10 +42,10 @@ public abstract class AbstractDBObjectSyncer implements DBSchemaSyncer { @Autowired - private DBObjectRepository dbObjectRepository; + protected DBObjectRepository dbObjectRepository; @Autowired - private DBColumnRepository dbColumnRepository; + protected DBColumnRepository dbColumnRepository; @Override public void sync(@NonNull Connection connection, @NonNull Database database, @NonNull DialectType dialectType) { @@ -77,6 +77,7 @@ public void sync(@NonNull Connection connection, @NonNull Database database, @No if (CollectionUtils.isNotEmpty(toBeDeleted)) { Set toBeDeletedIds = toBeDeleted.stream().map(DBObjectEntity::getId).collect(Collectors.toSet()); dbObjectRepository.deleteByIds(toBeDeletedIds); + preDelete(toBeDeletedIds); dbColumnRepository.deleteByDatabaseIdAndObjectIdIn(database.getId(), toBeDeletedIds); } } @@ -91,11 +92,13 @@ public int getOrder() { return Ordered.HIGHEST_PRECEDENCE; } - private T getExtensionPoint(@NonNull DialectType dialectType) { + protected T getExtensionPoint(@NonNull DialectType dialectType) { List points = SchemaPluginUtil.getExtensions(dialectType, getExtensionPointClass()); return CollectionUtils.isEmpty(points) ? null : points.get(0); } + void preDelete(@NonNull Set toBeDeletedIds) {} + abstract Set getLatestObjectNames(@NonNull T extensionPoint, @NonNull Connection connection, @NonNull Database database); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBTableSyncer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBTableSyncer.java index 943c47eb6d..5e43b29870 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBTableSyncer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBTableSyncer.java @@ -16,11 +16,17 @@ package com.oceanbase.odc.service.db.schema.syncer.object; import java.sql.Connection; +import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import com.oceanbase.odc.core.shared.constant.ResourceType; +import com.oceanbase.odc.metadb.iam.PermissionEntity; +import com.oceanbase.odc.metadb.iam.PermissionRepository; +import com.oceanbase.odc.metadb.iam.UserPermissionRepository; import com.oceanbase.odc.plugin.schema.api.TableExtensionPoint; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; @@ -35,6 +41,21 @@ @Component public class DBTableSyncer extends AbstractDBObjectSyncer { + @Autowired + private PermissionRepository permissionRepository; + + @Autowired + private UserPermissionRepository userPermissionRepository; + + @Override + protected void preDelete(@NonNull Set toBeDeletedIds) { + List permissions = + permissionRepository.findByResourceTypeAndResourceIdIn(ResourceType.ODC_TABLE, toBeDeletedIds); + Set permissionIds = permissions.stream().map(PermissionEntity::getId).collect(Collectors.toSet()); + permissionRepository.deleteByIds(permissionIds); + userPermissionRepository.deleteByPermissionIds(permissionIds); + } + @Override protected Set getLatestObjectNames(@NonNull TableExtensionPoint extensionPoint, @NonNull Connection connection, @NonNull Database database) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java index 12a6d1b8e9..4a281a5676 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java @@ -72,6 +72,7 @@ import com.oceanbase.odc.core.shared.constant.ResourceRoleName; import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.core.shared.constant.TaskType; +import com.oceanbase.odc.core.shared.exception.BadRequestException; import com.oceanbase.odc.core.shared.exception.NotFoundException; import com.oceanbase.odc.core.shared.exception.OverLimitException; import com.oceanbase.odc.core.shared.exception.UnsupportedException; @@ -101,7 +102,9 @@ import com.oceanbase.odc.service.connection.CloudMetadataClient.CloudPermissionAction; import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.connection.database.DatabaseService; +import com.oceanbase.odc.service.connection.database.model.DBResource; import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.odc.service.connection.database.model.UnauthorizedDBResource; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.connection.model.OBTenant; import com.oceanbase.odc.service.dispatch.DispatchResponse; @@ -152,10 +155,12 @@ import com.oceanbase.odc.service.notification.NotificationProperties; import com.oceanbase.odc.service.notification.helper.EventBuilder; import com.oceanbase.odc.service.notification.model.Event; -import com.oceanbase.odc.service.permission.database.DatabasePermissionHelper; +import com.oceanbase.odc.service.permission.DBResourcePermissionHelper; import com.oceanbase.odc.service.permission.database.model.ApplyDatabaseParameter; import com.oceanbase.odc.service.permission.database.model.ApplyDatabaseParameter.ApplyDatabase; import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; +import com.oceanbase.odc.service.permission.table.model.ApplyTableParameter; +import com.oceanbase.odc.service.permission.table.model.ApplyTableParameter.ApplyTable; import com.oceanbase.odc.service.regulation.approval.model.ApprovalFlowConfig; import com.oceanbase.odc.service.regulation.approval.model.ApprovalNodeConfig; import com.oceanbase.odc.service.regulation.risklevel.RiskLevelService; @@ -168,6 +173,7 @@ import com.oceanbase.odc.service.schedule.model.ScheduleStatus; import com.oceanbase.odc.service.task.TaskService; import com.oceanbase.odc.service.task.model.ExecutorInfo; +import com.oceanbase.tools.loaddump.common.enums.ObjectType; import lombok.AllArgsConstructor; import lombok.Data; @@ -240,7 +246,7 @@ public class FlowInstanceService { @Autowired private ResourceRoleService resourceRoleService; @Autowired - private DatabasePermissionHelper databasePermissionHelper; + private DBResourcePermissionHelper permissionHelper; @Autowired private NotificationProperties notificationProperties; @Autowired @@ -251,6 +257,8 @@ public class FlowInstanceService { private CloudMetadataClient cloudMetadataClient; @Autowired private EnvironmentRepository environmentRepository; + @Autowired + private DBResourcePermissionHelper dbResourcePermissionHelper; private final List> dataTransferTaskInitHooks = new ArrayList<>(); private final List> shadowTableComparingTaskHooks = new ArrayList<>(); @@ -315,6 +323,21 @@ public List create(@NotNull @Valid CreateFlowInstanceReq createReq.setParameters(parameter); return innerCreate(createReq); }).collect(Collectors.toList()).stream().flatMap(Collection::stream).collect(Collectors.toList()); + } else if (createReq.getTaskType() == TaskType.APPLY_TABLE_PERMISSION) { + ApplyTableParameter parameter = (ApplyTableParameter) createReq.getParameters(); + List tables = new ArrayList<>(parameter.getTables()); + Map> databaseId2Tables = + tables.stream().collect(Collectors.groupingBy(ApplyTable::getDatabaseId)); + if (CollectionUtils.isNotEmpty(databaseId2Tables.keySet()) + && databaseId2Tables.keySet().size() > MAX_APPLY_DATABASE_SIZE) { + throw new IllegalStateException("The number of databases to apply for exceeds the maximum limit"); + } + return databaseId2Tables.entrySet().stream().map(e -> { + parameter.setTables(new ArrayList<>(e.getValue())); + createReq.setDatabaseId(e.getKey()); + createReq.setParameters(parameter); + return innerCreate(createReq); + }).collect(Collectors.toList()).stream().flatMap(Collection::stream).collect(Collectors.toList()); } else { return innerCreate(createReq); } @@ -445,7 +468,8 @@ public Page listAll(@NotNull Pageable pageable, @NotNull Que TaskType.EXPORT_RESULT_SET, TaskType.APPLY_PROJECT_PERMISSION, TaskType.APPLY_DATABASE_PERMISSION, - TaskType.STRUCTURE_COMPARISON); + TaskType.STRUCTURE_COMPARISON, + TaskType.APPLY_TABLE_PERMISSION); specification = specification.and(FlowInstanceViewSpecs.taskTypeIn(types)); } @@ -734,6 +758,29 @@ private void checkCreateFlowInstancePermission(CreateFlowInstanceReq req) { if (authenticationFacade.currentUser().getOrganizationType() == OrganizationType.INDIVIDUAL) { return; } + if (req.getTaskType() == TaskType.EXPORT) { + DataTransferConfig parameters = (DataTransferConfig) req.getParameters(); + Map> resource2Types = new HashMap<>(); + if (CollectionUtils.isNotEmpty(parameters.getExportDbObjects())) { + ConnectionConfig config = connectionService.getBasicWithoutPermissionCheck(req.getConnectionId()); + parameters.getExportDbObjects().forEach(item -> { + if (item.getDbObjectType() == ObjectType.TABLE) { + resource2Types.put(DBResource.from(config, req.getDatabaseName(), item.getObjectName()), + DatabasePermissionType.from(TaskType.EXPORT)); + } + }); + } + List unauthorizedDBResources = + dbResourcePermissionHelper.filterUnauthorizedDBResources(resource2Types, false); + if (CollectionUtils.isNotEmpty(unauthorizedDBResources)) { + throw new BadRequestException(ErrorCodes.DatabaseAccessDenied, + new Object[] {unauthorizedDBResources.stream() + .map(UnauthorizedDBResource::getUnauthorizedPermissionTypes).flatMap(Collection::stream) + .map(DatabasePermissionType::getLocalizedMessage).collect(Collectors.joining(","))}, + "Lack permission for the database with id " + req.getDatabaseId()); + } + return; + } Set databaseIds = new HashSet<>(); if (Objects.nonNull(req.getDatabaseId())) { databaseIds.add(req.getDatabaseId()); @@ -770,7 +817,7 @@ private void checkCreateFlowInstancePermission(CreateFlowInstanceReq req) { databaseIds.add(p.getTargetDatabaseId()); databaseIds.add(p.getSourceDatabaseId()); } - databasePermissionHelper.checkPermissions(databaseIds, DatabasePermissionType.from(req.getTaskType())); + permissionHelper.checkDBPermissions(databaseIds, DatabasePermissionType.from(req.getTaskType())); } @@ -879,6 +926,7 @@ private FlowInstanceDetailResp buildFlowInstance(List riskLevels, Map variables = new HashMap<>(); FlowTaskUtil.setFlowInstanceId(variables, flowInstance.getId()); FlowTaskUtil.setTemplateVariables(variables, buildTemplateVariables(flowInstanceReq, connectionConfig)); + initVariables(variables, taskEntity, preCheckTaskEntity, connectionConfig, buildRiskLevelDescriber(flowInstanceReq)); flowInstance.start(variables); @@ -1041,6 +1089,9 @@ private TemplateVariables buildTemplateVariables(CreateFlowInstanceReq flowInsta for (Entry entry : config.getProperties().entrySet()) { variables.setAttribute(Variable.CONNECTION_PROPERTIES, entry.getKey(), entry.getValue()); } + } else { + variables.setAttribute(Variable.CONNECTION_NAME, ""); + variables.setAttribute(Variable.CONNECTION_TENANT, ""); } // set project related variables List projectOwners = new ArrayList<>(); @@ -1085,16 +1136,25 @@ private TemplateVariables buildTemplateVariables(CreateFlowInstanceReq flowInsta variables.setAttribute(Variable.DATABASE_OWNERS_ACCOUNTS, JsonUtils.toJson(ownerAccounts)); List ownerNames = databaseOwners.stream().map(User::getName).collect(Collectors.toList()); variables.setAttribute(Variable.DATABASE_OWNERS_NAMES, JsonUtils.toJson(ownerNames)); + } else { + variables.setAttribute(Variable.ENVIRONMENT_NAME, ""); + variables.setAttribute(Variable.DATABASE_NAME, ""); + variables.setAttribute(Variable.DATABASE_OWNERS_IDS, JsonUtils.toJson(Collections.emptyList())); + variables.setAttribute(Variable.DATABASE_OWNERS_ACCOUNTS, JsonUtils.toJson(Collections.emptyList())); + variables.setAttribute(Variable.DATABASE_OWNERS_NAMES, JsonUtils.toJson(Collections.emptyList())); } // set SQL content if task type is DatabaseChange if (taskType == TaskType.ASYNC) { DatabaseChangeParameters params = (DatabaseChangeParameters) flowInstanceReq.getParameters(); - String sqlContent = JsonUtils.toJson(params.getSqlContent()); - variables.setAttribute(Variable.SQL_CONTENT, sqlContent); - if (StringUtils.isNotBlank(sqlContent)) { - List splitSqlList = SqlUtils.split(config.getDialectType(), sqlContent, params.getDelimiter()); + variables.setAttribute(Variable.SQL_CONTENT, JsonUtils.toJson(params.getSqlContent())); + if (StringUtils.isNotBlank(params.getSqlContent())) { + List splitSqlList = + SqlUtils.split(config.getDialectType(), params.getSqlContent(), params.getDelimiter()); variables.setAttribute(Variable.SQL_CONTENT_JSON_ARRAY, JsonUtils.toJson(splitSqlList)); } + } else { + variables.setAttribute(Variable.SQL_CONTENT, ""); + variables.setAttribute(Variable.SQL_CONTENT_JSON_ARRAY, JsonUtils.toJson(Collections.emptyList())); } // set ODC URL site List configurations = systemConfigService.queryByKeyPrefix(ODC_SITE_URL); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceService.java index 93e76a9aba..7fa8cd9d5d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceService.java @@ -101,6 +101,7 @@ import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; import com.oceanbase.odc.service.permission.database.model.ApplyDatabaseResult; import com.oceanbase.odc.service.permission.project.ApplyProjectResult; +import com.oceanbase.odc.service.permission.table.model.ApplyTableResult; import com.oceanbase.odc.service.schedule.flowtask.AlterScheduleResult; import com.oceanbase.odc.service.session.model.SqlExecuteResult; import com.oceanbase.odc.service.task.TaskService; @@ -259,6 +260,8 @@ public List getTaskResultFromEntity(@NotNull TaskEntit results = getApplyProjectResult(taskEntity); } else if (taskEntity.getTaskType() == TaskType.APPLY_DATABASE_PERMISSION) { results = getApplyDatabaseResult(taskEntity); + } else if (taskEntity.getTaskType() == TaskType.APPLY_TABLE_PERMISSION) { + results = getApplyTableResult(taskEntity); } else if (taskEntity.getTaskType() == TaskType.STRUCTURE_COMPARISON) { results = getStructureComparisonResult(taskEntity); } else { @@ -676,6 +679,10 @@ private List getStructureComparisonResult(@NonN return innerGetResult(taskEntity, DBStructureComparisonTaskResult.class); } + private List getApplyTableResult(@NonNull TaskEntity taskEntity) { + return innerGetResult(taskEntity, ApplyTableResult.class); + } + private List innerGetResult(@NonNull TaskEntity taskEntity, @NonNull Class clazz) { String resultJson = taskEntity.getResultJson(); @@ -709,7 +716,8 @@ private Optional getDownloadableTaskEntity(@NonNull Long flowInstanc && instance.getTaskType() != TaskType.PRE_CHECK && instance.getTaskType() != TaskType.GENERATE_ROLLBACK && instance.getTaskType() != TaskType.APPLY_PROJECT_PERMISSION - && instance.getTaskType() != TaskType.APPLY_DATABASE_PERMISSION; + && instance.getTaskType() != TaskType.APPLY_DATABASE_PERMISSION + && instance.getTaskType() != TaskType.APPLY_TABLE_PERMISSION; } }); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/CreateFlowInstanceReq.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/CreateFlowInstanceReq.java index e77633fa09..f93cfd701a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/CreateFlowInstanceReq.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/CreateFlowInstanceReq.java @@ -38,6 +38,7 @@ import com.oceanbase.odc.service.partitionplan.model.PartitionPlanConfig; import com.oceanbase.odc.service.permission.database.model.ApplyDatabaseParameter; import com.oceanbase.odc.service.permission.project.ApplyProjectParameter; +import com.oceanbase.odc.service.permission.table.model.ApplyTableParameter; import com.oceanbase.odc.service.resultset.ResultSetExportTaskParameter; import com.oceanbase.odc.service.schedule.flowtask.AlterScheduleParameters; @@ -98,6 +99,7 @@ public class CreateFlowInstanceReq { @JsonSubTypes.Type(value = ResultSetExportTaskParameter.class, name = "EXPORT_RESULT_SET"), @JsonSubTypes.Type(value = ApplyProjectParameter.class, name = "APPLY_PROJECT_PERMISSION"), @JsonSubTypes.Type(value = ApplyDatabaseParameter.class, name = "APPLY_DATABASE_PERMISSION"), + @JsonSubTypes.Type(value = ApplyTableParameter.class, name = "APPLY_TABLE_PERMISSION"), @JsonSubTypes.Type(value = DBStructureComparisonParameter.class, name = "STRUCTURE_COMPARISON") }) private TaskParameters parameters; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowInstanceDetailResp.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowInstanceDetailResp.java index e5827afd9a..7a030cd79b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowInstanceDetailResp.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowInstanceDetailResp.java @@ -57,6 +57,7 @@ import com.oceanbase.odc.service.partitionplan.model.PartitionPlanConfig; import com.oceanbase.odc.service.permission.database.model.ApplyDatabaseParameter; import com.oceanbase.odc.service.permission.project.ApplyProjectParameter; +import com.oceanbase.odc.service.permission.table.model.ApplyTableParameter; import com.oceanbase.odc.service.regulation.risklevel.model.RiskLevel; import com.oceanbase.odc.service.resultset.ResultSetExportTaskParameter; import com.oceanbase.odc.service.schedule.flowtask.AlterScheduleParameters; @@ -314,6 +315,9 @@ public FlowInstanceDetailResp map(@NonNull FlowInstance flowInstance, @NonNull F case APPLY_DATABASE_PERMISSION: resp.setParameters(JsonUtils.fromJson(parameterJson, ApplyDatabaseParameter.class)); break; + case APPLY_TABLE_PERMISSION: + resp.setParameters(JsonUtils.fromJson(parameterJson, ApplyTableParameter.class)); + break; case STRUCTURE_COMPARISON: DBStructureComparisonParameter dbStructureComparisonParameter = JsonUtils.fromJson(parameterJson, DBStructureComparisonParameter.class); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowNodeInstanceDetailResp.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowNodeInstanceDetailResp.java index 220becd89e..1d3a1afbe8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowNodeInstanceDetailResp.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowNodeInstanceDetailResp.java @@ -31,11 +31,11 @@ import com.oceanbase.odc.metadb.iam.UserEntity; import com.oceanbase.odc.metadb.task.TaskEntity; import com.oceanbase.odc.service.common.model.InnerUser; +import com.oceanbase.odc.service.connection.database.model.UnauthorizedDBResource; import com.oceanbase.odc.service.flow.instance.BaseFlowNodeInstance; import com.oceanbase.odc.service.flow.instance.FlowApprovalInstance; import com.oceanbase.odc.service.flow.instance.FlowGatewayInstance; import com.oceanbase.odc.service.flow.instance.FlowTaskInstance; -import com.oceanbase.odc.service.permission.database.model.UnauthorizedDatabase; import lombok.Builder; import lombok.Data; @@ -63,7 +63,7 @@ public class FlowNodeInstanceDetailResp { private String externalApprovalName; private String externalFlowInstanceUrl; private Integer issueCount; - private List unauthorizedDatabases; + private List unauthorizedDBResources; private Boolean preCheckOverLimit; public static FlowNodeInstanceMapper mapper() { @@ -155,7 +155,7 @@ public FlowNodeInstanceDetailResp map(@NonNull FlowTaskInstance instance) { resp.setIssueCount(result.getSqlCheckResult().getIssueCount()); } if (Objects.nonNull(result.getPermissionCheckResult())) { - resp.setUnauthorizedDatabases(result.getPermissionCheckResult().getUnauthorizedDatabases()); + resp.setUnauthorizedDBResources(result.getPermissionCheckResult().getUnauthorizedDBResources()); } } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTask.java index faea530b73..a26a42a697 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTask.java @@ -51,7 +51,8 @@ import com.oceanbase.odc.service.common.FileManager; import com.oceanbase.odc.service.common.model.FileBucket; import com.oceanbase.odc.service.common.util.SqlUtils; -import com.oceanbase.odc.service.connection.database.DatabaseService; +import com.oceanbase.odc.service.connection.database.model.DBResource; +import com.oceanbase.odc.service.connection.database.model.UnauthorizedDBResource; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.flow.exception.ServiceTaskError; import com.oceanbase.odc.service.flow.model.FlowNodeStatus; @@ -65,15 +66,17 @@ import com.oceanbase.odc.service.flow.util.FlowTaskUtil; import com.oceanbase.odc.service.objectstorage.ObjectStorageFacade; import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeParameters; +import com.oceanbase.odc.service.permission.DBResourcePermissionHelper; import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; -import com.oceanbase.odc.service.permission.database.model.UnauthorizedDatabase; +import com.oceanbase.odc.service.permission.table.TablePermissionService; import com.oceanbase.odc.service.regulation.approval.ApprovalFlowConfigSelector; import com.oceanbase.odc.service.regulation.risklevel.model.RiskLevel; import com.oceanbase.odc.service.regulation.risklevel.model.RiskLevelDescriber; import com.oceanbase.odc.service.resultset.ResultSetExportTaskParameter; import com.oceanbase.odc.service.schedule.flowtask.AlterScheduleParameters; import com.oceanbase.odc.service.schedule.model.JobType; -import com.oceanbase.odc.service.session.util.SchemaExtractor; +import com.oceanbase.odc.service.session.util.DBSchemaExtractor; +import com.oceanbase.odc.service.session.util.DBSchemaExtractor.DBSchemaIdentity; import com.oceanbase.odc.service.sqlcheck.SqlCheckService; import com.oceanbase.odc.service.sqlcheck.model.CheckResult; import com.oceanbase.odc.service.sqlcheck.model.CheckViolation; @@ -105,7 +108,7 @@ public class PreCheckRuntimeFlowableTask extends BaseODCFlowTaskDelegate { @Autowired private ApprovalFlowConfigSelector approvalFlowConfigSelector; @Autowired - private DatabaseService databaseService; + private DBResourcePermissionHelper dbResourcePermissionHelper; @Autowired private SqlCheckService sqlCheckService; @Autowired @@ -113,6 +116,9 @@ public class PreCheckRuntimeFlowableTask extends BaseODCFlowTaskDelegate { @Autowired private ObjectStorageFacade storageFacade; + @Autowired + private TablePermissionService tablePermissionService; + private static final String CHECK_RESULT_FILE_NAME = "sql-check-result.json"; private final Map riskLevelResult = new HashMap<>(); @@ -247,7 +253,8 @@ private void preCheck(TaskEntity taskEntity, TaskEntity preCheckTaskEntity, Risk return; } } - doSqlCheckAndDatabasePermissionCheck(preCheckTaskEntity, riskLevelDescriber, taskType); + // doSqlCheckAndDatabasePermissionCheck(preCheckTaskEntity, riskLevelDescriber, taskType); + doSqlCheckAndResourcePermissionCheck(preCheckTaskEntity, riskLevelDescriber, taskType); if (isIntercepted(this.sqlCheckResult, this.permissionCheckResult)) { throw new ServiceTaskError(new RuntimeException()); } @@ -269,40 +276,42 @@ private boolean isIntercepted(SqlCheckTaskResult sqlCheckResult, } } if (Objects.nonNull(permissionCheckResult)) { - return CollectionUtils.isNotEmpty(permissionCheckResult.getUnauthorizedDatabases()); + return CollectionUtils.isNotEmpty(permissionCheckResult.getUnauthorizedDBResources()); } return false; } - private void doSqlCheckAndDatabasePermissionCheck(TaskEntity preCheckTaskEntity, RiskLevelDescriber describer, + private void doSqlCheckAndResourcePermissionCheck(TaskEntity preCheckTaskEntity, RiskLevelDescriber describer, TaskType taskType) { List sqls = new ArrayList<>(); this.overLimit = getSqlContentUntilOverLimit(sqls, preCheckTaskProperties.getMaxSqlContentBytes()); - List unauthorizedDatabases = new ArrayList<>(); + List unauthorizedDBResource = new ArrayList<>(); List violations = new ArrayList<>(); if (CollectionUtils.isNotEmpty(sqls)) { violations.addAll(this.sqlCheckService.check(Long.valueOf(describer.getEnvironmentId()), describer.getDatabaseName(), sqls, connectionConfig)); - Map> schemaName2SqlTypes = SchemaExtractor.listSchemaName2SqlTypes( + Map> identity2Types = DBSchemaExtractor.listDBSchemasWithSqlTypes( sqls.stream().map(e -> SqlTuple.newTuple(e.getStr())).collect(Collectors.toList()), - preCheckTaskEntity.getDatabaseName(), this.connectionConfig.getDialectType()); - Map> schemaName2PermissionTypes = new HashMap<>(); - for (Entry> entry : schemaName2SqlTypes.entrySet()) { + this.connectionConfig.getDialectType(), preCheckTaskEntity.getDatabaseName()); + Map> resource2PermissionTypes = new HashMap<>(); + for (Entry> entry : identity2Types.entrySet()) { + DBSchemaIdentity identity = entry.getKey(); Set sqlTypes = entry.getValue(); if (CollectionUtils.isNotEmpty(sqlTypes)) { Set permissionTypes = sqlTypes.stream().map(DatabasePermissionType::from) .filter(Objects::nonNull).collect(Collectors.toSet()); permissionTypes.addAll(DatabasePermissionType.from(taskType)); if (CollectionUtils.isNotEmpty(permissionTypes)) { - schemaName2PermissionTypes.put(entry.getKey(), permissionTypes); + resource2PermissionTypes.put( + DBResource.from(connectionConfig, identity.getSchema(), identity.getTable()), + permissionTypes); } } } - unauthorizedDatabases = - databaseService.filterUnauthorizedDatabases(schemaName2PermissionTypes, connectionConfig.getId(), - true); + unauthorizedDBResource = + dbResourcePermissionHelper.filterUnauthorizedDBResources(resource2PermissionTypes, false); } - this.permissionCheckResult = new DatabasePermissionCheckResult(unauthorizedDatabases); + this.permissionCheckResult = new DatabasePermissionCheckResult(unauthorizedDBResource); this.sqlCheckResult = SqlCheckTaskResult.success(violations); try { storeTaskResultToFile(preCheckTaskEntity.getId(), this.sqlCheckResult); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTaskCopied.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTaskCopied.java index 796761280e..2b09313c17 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTaskCopied.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTaskCopied.java @@ -290,7 +290,7 @@ private boolean isIntercepted(@NonNull PreCheckTaskResult result) { } } if (Objects.nonNull(permissionCheckResult)) { - return CollectionUtils.isNotEmpty(permissionCheckResult.getUnauthorizedDatabases()); + return CollectionUtils.isNotEmpty(permissionCheckResult.getUnauthorizedDBResources()); } return false; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/mapper/OdcRuntimeDelegateMapper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/mapper/OdcRuntimeDelegateMapper.java index 15362427e3..93609d0f09 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/mapper/OdcRuntimeDelegateMapper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/mapper/OdcRuntimeDelegateMapper.java @@ -31,6 +31,7 @@ import com.oceanbase.odc.service.onlineschemachange.OnlineSchemaChangeFlowableTask; import com.oceanbase.odc.service.permission.database.ApplyDatabaseFlowableTask; import com.oceanbase.odc.service.permission.project.ApplyProjectFlowableTask; +import com.oceanbase.odc.service.permission.table.ApplyTableFlowableTask; import com.oceanbase.odc.service.resultset.ResultSetExportFlowableTask; import com.oceanbase.odc.service.schedule.flowtask.AlterScheduleTask; @@ -73,6 +74,8 @@ public Class> map(@NonNull TaskType tas return ApplyProjectFlowableTask.class; case APPLY_DATABASE_PERMISSION: return ApplyDatabaseFlowableTask.class; + case APPLY_TABLE_PERMISSION: + return ApplyTableFlowableTask.class; case STRUCTURE_COMPARISON: return DBStructureComparisonFlowableTask.class; default: diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/model/DatabasePermissionCheckResult.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/model/DatabasePermissionCheckResult.java index 09e9850dff..f65d599396 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/model/DatabasePermissionCheckResult.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/model/DatabasePermissionCheckResult.java @@ -17,7 +17,7 @@ import java.util.List; -import com.oceanbase.odc.service.permission.database.model.UnauthorizedDatabase; +import com.oceanbase.odc.service.connection.database.model.UnauthorizedDBResource; import lombok.AllArgsConstructor; import lombok.Builder; @@ -34,5 +34,5 @@ @NoArgsConstructor @AllArgsConstructor public class DatabasePermissionCheckResult { - private List unauthorizedDatabases; + private List unauthorizedDBResources; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/util/FlowTaskUtil.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/util/FlowTaskUtil.java index 0d1e4240a6..9ff9c02b3d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/util/FlowTaskUtil.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/util/FlowTaskUtil.java @@ -53,6 +53,7 @@ import com.oceanbase.odc.service.partitionplan.model.PartitionPlanConfig; import com.oceanbase.odc.service.permission.database.model.ApplyDatabaseParameter; import com.oceanbase.odc.service.permission.project.ApplyProjectParameter; +import com.oceanbase.odc.service.permission.table.model.ApplyTableParameter; import com.oceanbase.odc.service.plugin.ConnectionPluginUtil; import com.oceanbase.odc.service.regulation.risklevel.model.RiskLevelDescriber; import com.oceanbase.odc.service.resultset.ResultSetExportTaskParameter; @@ -86,6 +87,11 @@ public static ApplyDatabaseParameter getApplyDatabaseParameter(@NonNull Delegate () -> new VerifyException("ApplyDatabaseParameter is absent")); } + public static ApplyTableParameter getApplyTableParameter(@NonNull DelegateExecution execution) { + return internalGetParameter(execution, ApplyTableParameter.class).orElseThrow( + () -> new VerifyException("ApplyTableParameter is absent")); + } + public static ApplyProjectParameter getApplyProjectParameter(@NonNull DelegateExecution execution) { return internalGetParameter(execution, ApplyProjectParameter.class).orElseThrow( () -> new VerifyException("ApplyProjectParameter is absent")); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java index d409ca2237..0d282fe9f1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java @@ -143,12 +143,6 @@ public List listByResourceTypeAndIdIn(ResourceType resourceTyp return fromEntities(userResourceRoleRepository.listByResourceTypeAndIdIn(resourceType, resourceIds)); } - @Transactional(rollbackFor = Exception.class) - @SkipAuthorize("internal usage") - public int deleteByResourceTypeAndId(@NonNull ResourceType resourceType, @NonNull Long resourceId) { - return userResourceRoleRepository.deleteByResourceTypeAndId(resourceType, resourceId); - } - @Transactional(rollbackFor = Exception.class) @SkipAuthorize("internal usage") public int deleteByResourceTypeAndIdIn(@NonNull ResourceType resourceType, @NotEmpty Collection resourceIds) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/constant/EventLabelKeys.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/constant/EventLabelKeys.java index 3180ba94a3..f0cf85c406 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/constant/EventLabelKeys.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/constant/EventLabelKeys.java @@ -40,6 +40,7 @@ public class EventLabelKeys { public static final String TENANT_NAME = "tenantName"; public static final String DATABASE_ID = "databaseId"; public static final String DATABASE_NAME = "databaseName"; + public static final String TABLE_NAME = "tableName"; /** * collaboration info diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/EventBuilder.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/EventBuilder.java index 7870498bbf..db2ebfbd9a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/EventBuilder.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/EventBuilder.java @@ -27,6 +27,7 @@ import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.PROJECT_ID; import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.PROJECT_NAME; import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.REGION; +import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.TABLE_NAME; import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.TASK_ENTITY_ID; import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.TASK_ID; import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.TASK_STATUS; @@ -78,6 +79,8 @@ import com.oceanbase.odc.service.permission.database.model.ApplyDatabaseParameter; import com.oceanbase.odc.service.permission.database.model.ApplyDatabaseParameter.ApplyDatabase; import com.oceanbase.odc.service.permission.project.ApplyProjectParameter; +import com.oceanbase.odc.service.permission.table.model.ApplyTableParameter; +import com.oceanbase.odc.service.permission.table.model.ApplyTableParameter.ApplyTable; import com.oceanbase.odc.service.schedule.flowtask.AlterScheduleParameters; import com.oceanbase.odc.service.schedule.model.JobType; @@ -203,6 +206,17 @@ private Event ofTask(TaskEntity task, TaskEvent status) { labels.putIfNonNull(DATABASE_NAME, dbNames); projectId = parameter.getProject().getId(); labels.putIfNonNull(PROJECT_ID, projectId); + } else if (task.getTaskType() == TaskType.APPLY_TABLE_PERMISSION) { + ApplyTableParameter parameter = + JsonUtils.fromJson(task.getParametersJson(), ApplyTableParameter.class); + List dbNames = + parameter.getTables().stream().map(ApplyTable::getDatabaseName).collect(Collectors.toList()); + labels.putIfNonNull(DATABASE_NAME, dbNames); + projectId = parameter.getProject().getId(); + labels.putIfNonNull(PROJECT_ID, projectId); + List tableNames = + parameter.getTables().stream().map(ApplyTable::getTableName).collect(Collectors.toList()); + labels.putIfNonNull(TABLE_NAME, tableNames); } else if (task.getTaskType() == TaskType.APPLY_PROJECT_PERMISSION) { ApplyProjectParameter parameter = JsonUtils.fromJson(task.getParametersJson(), ApplyProjectParameter.class); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java new file mode 100644 index 0000000000..3222853675 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java @@ -0,0 +1,417 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.permission; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.stream.Collectors; + +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.oceanbase.odc.core.authority.util.SkipAuthorize; +import com.oceanbase.odc.core.shared.constant.ErrorCodes; +import com.oceanbase.odc.core.shared.constant.OrganizationType; +import com.oceanbase.odc.core.shared.constant.ResourceRoleName; +import com.oceanbase.odc.core.shared.constant.ResourceType; +import com.oceanbase.odc.core.shared.exception.AccessDeniedException; +import com.oceanbase.odc.core.shared.exception.BadRequestException; +import com.oceanbase.odc.metadb.connection.DatabaseEntity; +import com.oceanbase.odc.metadb.connection.DatabaseRepository; +import com.oceanbase.odc.metadb.dbobject.DBObjectEntity; +import com.oceanbase.odc.metadb.dbobject.DBObjectRepository; +import com.oceanbase.odc.metadb.iam.UserDatabasePermissionEntity; +import com.oceanbase.odc.metadb.iam.UserDatabasePermissionRepository; +import com.oceanbase.odc.metadb.iam.UserTablePermissionEntity; +import com.oceanbase.odc.metadb.iam.UserTablePermissionRepository; +import com.oceanbase.odc.service.collaboration.project.ProjectService; +import com.oceanbase.odc.service.connection.database.model.DBResource; +import com.oceanbase.odc.service.connection.database.model.UnauthorizedDBResource; +import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; +import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; + +/** + * @author gaoda.xy + * @date 2024/4/28 16:02 + */ +@Component +@SkipAuthorize("odc internal usage") +public class DBResourcePermissionHelper { + + @Autowired + private ProjectService projectService; + + @Autowired + private AuthenticationFacade authenticationFacade; + + @Autowired + private DatabaseRepository databaseRepository; + + @Autowired + private UserDatabasePermissionRepository userDatabasePermissionRepository; + + @Autowired + private UserTablePermissionRepository userTablePermissionRepository; + + @Autowired + private DBObjectRepository dbObjectRepository; + + private static final Set ORACLE_DATA_DICTIONARY = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + + private static final Set MYSQL_DATA_DICTIONARY = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + + static { + ORACLE_DATA_DICTIONARY.add("SYS"); + MYSQL_DATA_DICTIONARY.add("information_schema"); + } + + /** + * Check user authority for the databases. Throw exception if the user has no specified permission. + * + * @param databaseIds database ids + * @param permissionTypes permission types + */ + public void checkDBPermissions(Collection databaseIds, Collection permissionTypes) { + if (CollectionUtils.isEmpty(databaseIds) || CollectionUtils.isEmpty(permissionTypes)) { + return; + } + if (authenticationFacade.currentUser().getOrganizationType() == OrganizationType.INDIVIDUAL) { + return; + } + List entities = databaseRepository.findByIdIn(databaseIds); + List toCheckDatabaseIds = new ArrayList<>(); + Set projectIds = getPermittedProjectIds(); + for (DatabaseEntity e : entities) { + if (e.getProjectId() == null) { + throw new AccessDeniedException("Database is not belong to any project"); + } + if (!projectIds.contains(e.getProjectId())) { + toCheckDatabaseIds.add(e.getId()); + } + } + Map> id2PermissionTypes = getInnerDBPermissionTypes(databaseIds); + for (Long databaseId : toCheckDatabaseIds) { + Set authorized = id2PermissionTypes.get(databaseId); + Set unAuthorized = + permissionTypes.stream().filter(e -> CollectionUtils.isEmpty(authorized) || !authorized.contains(e)) + .collect(Collectors.toSet()); + if (CollectionUtils.isNotEmpty(unAuthorized)) { + throw new BadRequestException(ErrorCodes.DatabaseAccessDenied, + new Object[] {unAuthorized.stream().map(DatabasePermissionType::getLocalizedMessage) + .collect(Collectors.joining(","))}, + "Lack permission for the database with id " + databaseId); + } + } + } + + /** + * Get user authority for the databases. + * + * @param databaseIds database ids + * @return database id to permission types + */ + public Map> getDBPermissions(Collection databaseIds) { + Map> ret = new HashMap<>(); + if (CollectionUtils.isEmpty(databaseIds)) { + return ret; + } + if (authenticationFacade.currentUser().getOrganizationType() == OrganizationType.INDIVIDUAL) { + for (Long databaseId : databaseIds) { + ret.put(databaseId, new HashSet<>(DatabasePermissionType.all())); + } + return ret; + } + List entities = databaseRepository.findByIdIn(databaseIds); + Set projectIds = getPermittedProjectIds(); + Map> id2PermissionTypes = getInnerDBPermissionTypes(databaseIds); + for (DatabaseEntity e : entities) { + if (e.getProjectId() == null) { + ret.put(e.getId(), new HashSet<>()); + } else if (projectIds.contains(e.getProjectId())) { + ret.put(e.getId(), new HashSet<>(DatabasePermissionType.all())); + } else { + if (id2PermissionTypes.containsKey(e.getId())) { + ret.put(e.getId(), id2PermissionTypes.get(e.getId())); + } else { + ret.put(e.getId(), new HashSet<>()); + } + } + } + return ret; + } + + /** + * Get user authority for the tables. + * + * @param tableIds table ids + * @return table id to permission types + */ + public Map> getTablePermissions(Collection tableIds) { + Map> ret = new HashMap<>(); + if (CollectionUtils.isEmpty(tableIds)) { + return ret; + } + if (authenticationFacade.currentUser().getOrganizationType() == OrganizationType.INDIVIDUAL) { + for (Long tableId : tableIds) { + ret.put(tableId, new HashSet<>(DatabasePermissionType.all())); + } + return ret; + } + List tbEntities = dbObjectRepository.findByIdIn(tableIds); + Set dbIds = tbEntities.stream().map(DBObjectEntity::getDatabaseId).collect(Collectors.toSet()); + Map dbId2Entity = databaseRepository.findByIdIn(dbIds).stream() + .collect(Collectors.toMap(DatabaseEntity::getId, e -> e)); + Set projectIds = getPermittedProjectIds(); + Map> dbId2Permissions = userDatabasePermissionRepository + .findNotExpiredByUserIdAndDatabaseIdIn(authenticationFacade.currentUserId(), dbIds) + .stream().collect(Collectors.groupingBy(UserDatabasePermissionEntity::getDatabaseId)); + Map> tbId2Permissions = userTablePermissionRepository + .findNotExpiredByUserIdAndTableIdIn(authenticationFacade.currentUserId(), tableIds) + .stream().collect(Collectors.groupingBy(UserTablePermissionEntity::getTableId)); + for (DBObjectEntity e : tbEntities) { + if (!dbId2Entity.containsKey(e.getDatabaseId()) + || dbId2Entity.get(e.getDatabaseId()).getProjectId() == null) { + ret.put(e.getId(), new HashSet<>()); + } else if (projectIds.contains(dbId2Entity.get(e.getDatabaseId()).getProjectId())) { + ret.put(e.getId(), new HashSet<>(DatabasePermissionType.all())); + } else { + Set authorized = new HashSet<>(); + List dbPermissions = dbId2Permissions.get(e.getDatabaseId()); + if (CollectionUtils.isNotEmpty(dbPermissions)) { + authorized.addAll(dbPermissions.stream().map(p -> DatabasePermissionType.from(p.getAction())) + .collect(Collectors.toSet())); + } + List permissions = tbId2Permissions.get(e.getId()); + if (CollectionUtils.isNotEmpty(permissions)) { + authorized.addAll(permissions.stream().map(p -> DatabasePermissionType.from(p.getAction())) + .collect(Collectors.toSet())); + } + ret.put(e.getId(), authorized); + } + } + return ret; + } + + /** + * Filter and return unauthorized database resources (ODC_DATABASE, ODC_TABLE) and permission types. + * + * @param resource2Types Resource to permission types. The resource is defined by {@link DBResource} + * and warning that only {@link DBResource#getDataSourceId()}, + * {@link DBResource#getDatabaseName()}, {@link DBResource#getTableName()} and + * {@link DBResource#getType()} are used. Other fields are autofilled by this method. + * {@link DBResource#getDataSourceId()}, {@link DBResource#getDatabaseName()} and + * {@link DBResource#getTableName()} could not be null. + * @param ignoreDataDictionary Ignore data dictionary schema + * @return Unauthorized database resources and permission types + */ + public List filterUnauthorizedDBResources( + Map> resource2Types, boolean ignoreDataDictionary) { + if (resource2Types == null || resource2Types.isEmpty()) { + return Collections.emptyList(); + } + // Pre-handle DBResource (fill in databaseId, databaseName, tableId, tableName) + Set dsIds = resource2Types.keySet().stream().map(DBResource::getDataSourceId).collect(Collectors.toSet()); + List dbEntities = databaseRepository.findByConnectionIdIn(dsIds); + Map dbId2Entity = + dbEntities.stream().collect(Collectors.toMap(DatabaseEntity::getId, e -> e)); + Map> dsId2dbName2dbEntity = new HashMap<>(); + for (DatabaseEntity dbEntity : dbEntities) { + dsId2dbName2dbEntity + .computeIfAbsent(dbEntity.getConnectionId(), k -> new TreeMap<>(String.CASE_INSENSITIVE_ORDER)) + .put(dbEntity.getName(), dbEntity); + } + resource2Types.forEach((k, v) -> { + if (dsId2dbName2dbEntity.containsKey(k.getDataSourceId())) { + DatabaseEntity dbEntity = dsId2dbName2dbEntity.get(k.getDataSourceId()).get(k.getDatabaseName()); + if (dbEntity != null) { + k.setDatabaseId(dbEntity.getId()); + k.setDatabaseName(dbEntity.getName()); + } + } + }); + Set dbIds = resource2Types.keySet().stream().map(DBResource::getDatabaseId).collect(Collectors.toSet()); + List tbEntities = dbObjectRepository.findByDatabaseIdInAndType(dbIds, DBObjectType.TABLE); + Map tbId2Entity = + tbEntities.stream().collect(Collectors.toMap(DBObjectEntity::getId, e -> e)); + Map> dbId2tbName2tbEntity = new HashMap<>(); + for (DBObjectEntity tbEntity : tbEntities) { + dbId2tbName2tbEntity + .computeIfAbsent(tbEntity.getDatabaseId(), k -> new TreeMap<>(String.CASE_INSENSITIVE_ORDER)) + .put(tbEntity.getName(), tbEntity); + } + resource2Types.forEach((k, v) -> { + if (k.getTableName() != null && k.getDatabaseId() != null + && dbId2tbName2tbEntity.containsKey(k.getDatabaseId())) { + DBObjectEntity tbEntity = dbId2tbName2tbEntity.get(k.getDatabaseId()).get(k.getTableName()); + if (tbEntity != null) { + k.setTableId(tbEntity.getId()); + k.setTableName(tbEntity.getName()); + } + } + }); + Set tbIds = resource2Types.keySet().stream().map(DBResource::getTableId).collect(Collectors.toSet()); + // Get permitted and involved project ids + Map> projectIds2Roles = projectService.getProjectId2ResourceRoleNames(); + Set involvedProjectIds = projectIds2Roles.keySet(); + Set permittedProjectIds = projectIds2Roles.entrySet().stream() + .filter(e -> e.getValue().contains(ResourceRoleName.OWNER) + || e.getValue().contains(ResourceRoleName.DBA) + || e.getValue().contains(ResourceRoleName.DEVELOPER)) + .map(Map.Entry::getKey).collect(Collectors.toSet()); + // Get user permissions + Map> dbId2Permissions = userDatabasePermissionRepository + .findNotExpiredByUserIdAndDatabaseIdIn(authenticationFacade.currentUserId(), dbIds) + .stream().collect(Collectors.groupingBy(UserDatabasePermissionEntity::getDatabaseId)); + Map> tbId2Permissions = userTablePermissionRepository + .findNotExpiredByUserIdAndTableIdIn(authenticationFacade.currentUserId(), tbIds) + .stream().collect(Collectors.groupingBy(UserTablePermissionEntity::getTableId)); + // Filter unauthorized resources + List unauthorizedDBResources = new ArrayList<>(); + for (Map.Entry> entry : resource2Types.entrySet()) { + DBResource resource = entry.getKey(); + Set needs = entry.getValue(); + if (CollectionUtils.isEmpty(needs)) { + continue; + } + if (resource.getType() == ResourceType.ODC_DATABASE) { + if (resource.getDatabaseId() == null) { + unauthorizedDBResources.add(UnauthorizedDBResource.from(resource, needs, false)); + continue; + } + DatabaseEntity database = dbId2Entity.get(resource.getDatabaseId()); + Set authorized = new HashSet<>(); + if (permittedProjectIds.contains(database.getProjectId())) { + authorized.addAll(DatabasePermissionType.all()); + } else { + List permissions = dbId2Permissions.get(database.getId()); + if (CollectionUtils.isNotEmpty(permissions)) { + authorized.addAll(permissions.stream().map(e -> DatabasePermissionType.from(e.getAction())) + .collect(Collectors.toSet())); + } + if (dbId2tbName2tbEntity.get(database.getId()) != null) { + Collection tbs = dbId2tbName2tbEntity.get(database.getId()).values(); + for (DBObjectEntity tbEntity : tbs) { + List tablePermissions = tbId2Permissions.get(tbEntity.getId()); + if (CollectionUtils.isNotEmpty(tablePermissions)) { + authorized.add(DatabasePermissionType.ACCESS); + break; + } + } + } + } + Set unauthorized = + needs.stream().filter(p -> !authorized.contains(p)).collect(Collectors.toSet()); + if (CollectionUtils.isNotEmpty(unauthorized)) { + unauthorizedDBResources.add(UnauthorizedDBResource.from(resource, unauthorized, + database.getProjectId() != null && involvedProjectIds.contains(database.getProjectId()))); + } + } else if (resource.getType() == ResourceType.ODC_TABLE) { + if (resource.getDatabaseId() == null) { + unauthorizedDBResources.add(UnauthorizedDBResource.from(resource, needs, false)); + continue; + } + Set authorized = new HashSet<>(); + DatabaseEntity database = dbId2Entity.get(resource.getDatabaseId()); + if (permittedProjectIds.contains(database.getProjectId())) { + authorized.addAll(DatabasePermissionType.all()); + } else { + List dbPermissions = dbId2Permissions.get(database.getId()); + if (CollectionUtils.isNotEmpty(dbPermissions)) { + authorized.addAll(dbPermissions.stream().map(e -> DatabasePermissionType.from(e.getAction())) + .collect(Collectors.toSet())); + } + if (resource.getTableId() != null) { + DBObjectEntity table = tbId2Entity.get(resource.getTableId()); + List permissions = tbId2Permissions.get(table.getId()); + if (CollectionUtils.isNotEmpty(permissions)) { + authorized.addAll(permissions.stream().map(e -> DatabasePermissionType.from(e.getAction())) + .collect(Collectors.toSet())); + } + } + } + Set unauthorized = + needs.stream().filter(p -> !authorized.contains(p)).collect(Collectors.toSet()); + if (CollectionUtils.isNotEmpty(unauthorized)) { + unauthorizedDBResources.add(UnauthorizedDBResource.from(resource, unauthorized, + database.getProjectId() != null && involvedProjectIds.contains(database.getProjectId()))); + } + } else { + throw new IllegalStateException("Unsupported resource type: " + resource.getType()); + } + } + // Filter data dictionary schema + if (ignoreDataDictionary) { + unauthorizedDBResources = unauthorizedDBResources.stream().filter(e -> { + if (e.getDialectType().isOracle()) { + return !ORACLE_DATA_DICTIONARY.contains(e.getDatabaseName()); + } else if (e.getDialectType().isMysql()) { + return !MYSQL_DATA_DICTIONARY.contains(e.getDatabaseName()); + } else { + return true; + } + }).collect(Collectors.toList()); + } + return unauthorizedDBResources; + } + + private Set getPermittedProjectIds() { + // OWNER, DBA or DEVELOPER can access all databases inner the project + Map> projectIds2Roles = projectService.getProjectId2ResourceRoleNames(); + return projectIds2Roles.entrySet().stream() + .filter(e -> e.getValue().contains(ResourceRoleName.OWNER) + || e.getValue().contains(ResourceRoleName.DBA) + || e.getValue().contains(ResourceRoleName.DEVELOPER)) + .map(Map.Entry::getKey).collect(Collectors.toSet()); + } + + private Map> getInnerDBPermissionTypes(Collection databaseIds) { + Map> typesFromDatabase = userDatabasePermissionRepository + .findNotExpiredByUserIdAndDatabaseIdIn(authenticationFacade.currentUserId(), databaseIds) + .stream().collect(Collectors.toMap( + UserDatabasePermissionEntity::getDatabaseId, + e -> { + Set list = new HashSet<>(); + list.add(DatabasePermissionType.from(e.getAction())); + return list; + }, + (e1, e2) -> { + e1.addAll(e2); + return e1; + })); + Map> typesFromTable = userTablePermissionRepository + .findNotExpiredByUserIdAndDatabaseIdIn(authenticationFacade.currentUserId(), databaseIds) + .stream().collect(Collectors.toMap( + UserTablePermissionEntity::getDatabaseId, + e -> Collections.singleton(DatabasePermissionType.ACCESS), + (e1, e2) -> e1)); + typesFromTable.forEach((k, v) -> typesFromDatabase.merge(k, v, (v1, v2) -> { + v1.addAll(v2); + return v1; + })); + return typesFromDatabase; + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/ApplyDatabaseFlowableTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/ApplyDatabaseFlowableTask.java index f888058110..e780818282 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/ApplyDatabaseFlowableTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/ApplyDatabaseFlowableTask.java @@ -114,6 +114,8 @@ protected ApplyDatabaseResult start(Long taskId, TaskService taskService, Delega permissionEntity.setExpireTime(parameter.getExpireTime()); permissionEntity.setAuthorizationType(AuthorizationType.TICKET_APPLICATION); permissionEntity.setTicketId(FlowTaskUtil.getFlowInstanceId(execution)); + permissionEntity.setResourceType(ResourceType.ODC_DATABASE); + permissionEntity.setResourceId(database.getId()); permissionEntities.add(permissionEntity); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/DatabasePermissionHelper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/DatabasePermissionHelper.java deleted file mode 100644 index 42636ac558..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/DatabasePermissionHelper.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.permission.database; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import org.apache.commons.collections4.CollectionUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import com.oceanbase.odc.core.shared.constant.ErrorCodes; -import com.oceanbase.odc.core.shared.constant.OrganizationType; -import com.oceanbase.odc.core.shared.constant.ResourceRoleName; -import com.oceanbase.odc.core.shared.exception.AccessDeniedException; -import com.oceanbase.odc.core.shared.exception.BadRequestException; -import com.oceanbase.odc.metadb.connection.DatabaseEntity; -import com.oceanbase.odc.metadb.connection.DatabaseRepository; -import com.oceanbase.odc.metadb.iam.UserDatabasePermissionEntity; -import com.oceanbase.odc.metadb.iam.UserDatabasePermissionRepository; -import com.oceanbase.odc.service.collaboration.project.ProjectService; -import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; -import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; - -/** - * @author gaoda.xy - * @date 2024/1/15 10:58 - */ -@Component -public class DatabasePermissionHelper { - - @Autowired - private ProjectService projectService; - - @Autowired - private AuthenticationFacade authenticationFacade; - - @Autowired - private DatabaseRepository databaseRepository; - - @Autowired - private UserDatabasePermissionRepository userDatabasePermissionRepository; - - /** - * Check whether the current user has the permission to access the database - * - * @param databaseIds database ids - * @param permissionTypes permission types - */ - public void checkPermissions(Collection databaseIds, Collection permissionTypes) { - if (authenticationFacade.currentUser().getOrganizationType() == OrganizationType.INDIVIDUAL) { - return; - } - if (CollectionUtils.isEmpty(databaseIds) || CollectionUtils.isEmpty(permissionTypes)) { - return; - } - List entities = databaseRepository.findByIdIn(databaseIds); - List toCheckDatabaseIds = new ArrayList<>(); - Set projectIds = getPermittedProjectIds(); - for (DatabaseEntity e : entities) { - if (e.getProjectId() == null) { - throw new AccessDeniedException("Database is not belong to any project"); - } - if (!projectIds.contains(e.getProjectId())) { - toCheckDatabaseIds.add(e.getId()); - } - } - Map> id2PermissionTypes = getDatabaseId2PermissionTypes(databaseIds); - for (Long databaseId : toCheckDatabaseIds) { - // TODO: may use Permission#implies() to check permission - Set authorized = id2PermissionTypes.get(databaseId); - Set unAuthorized = - permissionTypes.stream().filter(e -> CollectionUtils.isEmpty(authorized) || !authorized.contains(e)) - .collect(Collectors.toSet()); - if (CollectionUtils.isNotEmpty(unAuthorized)) { - throw new BadRequestException(ErrorCodes.DatabaseAccessDenied, - new Object[] {unAuthorized.stream().map(DatabasePermissionType::getLocalizedMessage) - .collect(Collectors.joining(","))}, - "Lack permission for the database with id " + databaseId); - } - } - } - - /** - * Get the DB access permissions of the databases - * - * @param databaseIds database ids - * @return database id to permission types - */ - public Map> getPermissions(Collection databaseIds) { - Map> ret = new HashMap<>(); - if (CollectionUtils.isEmpty(databaseIds)) { - return ret; - } - if (authenticationFacade.currentUser().getOrganizationType() == OrganizationType.INDIVIDUAL) { - for (Long databaseId : databaseIds) { - ret.put(databaseId, new HashSet<>(DatabasePermissionType.all())); - } - return ret; - } - List entities = databaseRepository.findByIdIn(databaseIds); - Set projectIds = getPermittedProjectIds(); - Map> id2PermissionTypes = getDatabaseId2PermissionTypes(databaseIds); - for (DatabaseEntity e : entities) { - if (e.getProjectId() == null) { - ret.put(e.getId(), new HashSet<>()); - } else if (projectIds.contains(e.getProjectId())) { - ret.put(e.getId(), new HashSet<>(DatabasePermissionType.all())); - } else { - if (id2PermissionTypes.containsKey(e.getId())) { - ret.put(e.getId(), id2PermissionTypes.get(e.getId())); - } else { - ret.put(e.getId(), new HashSet<>()); - } - } - } - return ret; - } - - private Set getPermittedProjectIds() { - // OWNER, DBA or DEVELOPER of a project can access all databases inner the project - Map> projectIds2Roles = projectService.getProjectId2ResourceRoleNames(); - return projectIds2Roles.entrySet().stream() - .filter(e -> e.getValue().contains(ResourceRoleName.OWNER) - || e.getValue().contains(ResourceRoleName.DBA) - || e.getValue().contains(ResourceRoleName.DEVELOPER)) - .map(Map.Entry::getKey).collect(Collectors.toSet()); - } - - private Map> getDatabaseId2PermissionTypes(Collection databaseIds) { - return userDatabasePermissionRepository - .findNotExpiredByUserIdAndDatabaseIdIn(authenticationFacade.currentUserId(), databaseIds).stream() - .collect(Collectors.toMap( - UserDatabasePermissionEntity::getDatabaseId, - e -> { - Set list = new HashSet<>(); - list.add(DatabasePermissionType.from(e.getAction())); - return list; - }, - (e1, e2) -> { - e1.addAll(e2); - return e1; - })); - } - -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/DatabasePermissionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/DatabasePermissionService.java index 38fb637029..e8c1df3bfd 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/DatabasePermissionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/DatabasePermissionService.java @@ -180,6 +180,8 @@ public List batchCreate(@NotNull Long projectId, entity.setBuiltIn(false); entity.setExpireTime(expireTime); entity.setAuthorizationType(AuthorizationType.USER_AUTHORIZATION); + entity.setResourceType(ResourceType.ODC_DATABASE); + entity.setResourceId(databaseId); permissionEntities.add(entity); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/ApplyDatabaseParameter.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/ApplyDatabaseParameter.java index 7e97604655..7dd16d166b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/ApplyDatabaseParameter.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/ApplyDatabaseParameter.java @@ -18,10 +18,8 @@ import java.io.Serializable; import java.util.Date; import java.util.List; -import java.util.Objects; import com.oceanbase.odc.core.flow.model.TaskParameters; -import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.permission.project.ApplyProjectParameter.ApplyProject; import lombok.Data; @@ -36,7 +34,7 @@ public class ApplyDatabaseParameter implements Serializable, TaskParameters { private static final long serialVersionUID = -2482302525012272875L; /** - * Project to be applied for, the + * Project to be applied for, required */ private ApplyProject project; /** @@ -66,20 +64,6 @@ public static class ApplyDatabase implements Serializable { private Long dataSourceId; private String dataSourceName; - public static ApplyDatabase from(Database database) { - ApplyDatabase applyDatabase = new ApplyDatabase(); - if (Objects.isNull(database)) { - return null; - } - applyDatabase.setId(database.getId()); - applyDatabase.setName(database.getName()); - if (Objects.nonNull(database.getDataSource())) { - applyDatabase.setDataSourceId(database.getDataSource().getId()); - applyDatabase.setDataSourceName(database.getDataSource().getName()); - } - return applyDatabase; - } - } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/DatabasePermissionType.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/DatabasePermissionType.java index 36cd665512..d7179e50ae 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/DatabasePermissionType.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/DatabasePermissionType.java @@ -39,7 +39,9 @@ public enum DatabasePermissionType implements Translatable { QUERY("query"), CHANGE("change"), - EXPORT("export"); + EXPORT("export"), + // The user has no database permission but has permissions to access the inner tables + ACCESS("access"); private final String action; @@ -52,15 +54,18 @@ public static List all() { } public static DatabasePermissionType from(String action) { - if ("query".equalsIgnoreCase(action)) { + if (QUERY.action.equalsIgnoreCase(action)) { return DatabasePermissionType.QUERY; } - if ("change".equalsIgnoreCase(action)) { + if (CHANGE.action.equalsIgnoreCase(action)) { return DatabasePermissionType.CHANGE; } - if ("export".equalsIgnoreCase(action)) { + if (EXPORT.action.equalsIgnoreCase(action)) { return DatabasePermissionType.EXPORT; } + if (ACCESS.action.equalsIgnoreCase(action)) { + return DatabasePermissionType.ACCESS; + } return null; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/QueryDatabasePermissionParams.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/QueryDatabasePermissionParams.java index 0076d4f9fa..71621425e3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/QueryDatabasePermissionParams.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/QueryDatabasePermissionParams.java @@ -19,15 +19,15 @@ import com.oceanbase.odc.core.shared.constant.AuthorizationType; -import lombok.Builder; import lombok.Data; +import lombok.experimental.SuperBuilder; /** * @author gaoda.xy * @date 2024/1/4 13:59 */ @Data -@Builder +@SuperBuilder public class QueryDatabasePermissionParams { private Long userId; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTableFlowableTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTableFlowableTask.java new file mode 100644 index 0000000000..3eac86a7ee --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTableFlowableTask.java @@ -0,0 +1,271 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.permission.table; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.flowable.engine.delegate.DelegateExecution; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.support.TransactionTemplate; + +import com.oceanbase.odc.common.trace.TaskContextHolder; +import com.oceanbase.odc.core.shared.constant.AuthorizationType; +import com.oceanbase.odc.core.shared.constant.ErrorCodes; +import com.oceanbase.odc.core.shared.constant.FlowStatus; +import com.oceanbase.odc.core.shared.constant.PermissionType; +import com.oceanbase.odc.core.shared.constant.ResourceType; +import com.oceanbase.odc.core.shared.exception.NotFoundException; +import com.oceanbase.odc.core.shared.exception.UnsupportedException; +import com.oceanbase.odc.metadb.collaboration.ProjectRepository; +import com.oceanbase.odc.metadb.connection.DatabaseEntity; +import com.oceanbase.odc.metadb.connection.DatabaseRepository; +import com.oceanbase.odc.metadb.dbobject.DBObjectEntity; +import com.oceanbase.odc.metadb.dbobject.DBObjectRepository; +import com.oceanbase.odc.metadb.iam.PermissionEntity; +import com.oceanbase.odc.metadb.iam.PermissionRepository; +import com.oceanbase.odc.metadb.iam.UserPermissionEntity; +import com.oceanbase.odc.metadb.iam.UserPermissionRepository; +import com.oceanbase.odc.metadb.iam.UserRepository; +import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleEntity; +import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; +import com.oceanbase.odc.service.flow.task.BaseODCFlowTaskDelegate; +import com.oceanbase.odc.service.flow.util.FlowTaskUtil; +import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; +import com.oceanbase.odc.service.permission.table.model.ApplyTableParameter; +import com.oceanbase.odc.service.permission.table.model.ApplyTableParameter.ApplyTable; +import com.oceanbase.odc.service.permission.table.model.ApplyTableResult; +import com.oceanbase.odc.service.task.TaskService; + +import lombok.extern.slf4j.Slf4j; + +/** + * + * @Author: fenghao + * @Create 2024/3/14 17:25 + * @Version 1.0 + */ +@Slf4j +public class ApplyTableFlowableTask extends BaseODCFlowTaskDelegate { + + @Autowired + private TransactionTemplate transactionTemplate; + + @Autowired + private UserRepository userRepository; + + @Autowired + private PermissionRepository permissionRepository; + + @Autowired + private UserPermissionRepository userPermissionRepository; + + @Autowired + private ProjectRepository projectRepository; + + @Autowired + private DatabaseRepository databaseRepository; + + @Autowired + private UserResourceRoleRepository userResourceRoleRepository; + + @Autowired + private DBObjectRepository dbObjectRepository; + + private volatile boolean success = false; + private volatile boolean failure = false; + private Long creatorId; + + @Override + protected ApplyTableResult start(Long taskId, TaskService taskService, DelegateExecution execution) + throws Exception { + this.creatorId = FlowTaskUtil.getTaskCreator(execution).getId(); + TaskContextHolder.trace(creatorId, taskId); + ApplyTableResult result = new ApplyTableResult(); + log.info("Apply table task starts, taskId={}, activityId={}", taskId, execution.getCurrentActivityId()); + try { + transactionTemplate.executeWithoutResult(status -> { + try { + taskService.start(taskId); + ApplyTableParameter parameter = FlowTaskUtil.getApplyTableParameter(execution); + result.setParameter(parameter); + checkResourceAndPermission(parameter); + List permissionEntities = new ArrayList<>(); + Long organizationId = FlowTaskUtil.getOrganizationId(execution); + for (ApplyTable table : parameter.getTables()) { + for (DatabasePermissionType permissionType : parameter.getTypes()) { + PermissionEntity permissionEntity = new PermissionEntity(); + permissionEntity.setAction(permissionType.getAction()); + permissionEntity.setResourceIdentifier( + ResourceType.ODC_DATABASE.name() + ":" + table.getDatabaseId() + "/" + + ResourceType.ODC_TABLE.name() + ":" + table.getTableId()); + permissionEntity.setType(PermissionType.PUBLIC_RESOURCE); + permissionEntity.setCreatorId(this.creatorId); + permissionEntity.setOrganizationId(organizationId); + permissionEntity.setBuiltIn(false); + permissionEntity.setExpireTime(parameter.getExpireTime()); + permissionEntity.setAuthorizationType(AuthorizationType.TICKET_APPLICATION); + permissionEntity.setTicketId(FlowTaskUtil.getFlowInstanceId(execution)); + permissionEntity.setResourceType(ResourceType.ODC_TABLE); + permissionEntity.setResourceId(table.getTableId()); + permissionEntities.add(permissionEntity); + } + } + List saved = permissionRepository.batchCreate(permissionEntities); + List userPermissionEntities = new ArrayList<>(); + for (PermissionEntity permissionEntity : saved) { + UserPermissionEntity userPermissionEntity = new UserPermissionEntity(); + userPermissionEntity.setUserId(this.creatorId); + userPermissionEntity.setPermissionId(permissionEntity.getId()); + userPermissionEntity.setCreatorId(this.creatorId); + userPermissionEntity.setOrganizationId(organizationId); + userPermissionEntities.add(userPermissionEntity); + } + userPermissionRepository.batchCreate(userPermissionEntities); + success = true; + } catch (Exception e) { + failure = true; + log.warn("Error occurs while apply table task executing", e); + status.setRollbackOnly(); + } + }); + result.setSuccess(success); + if (result.isSuccess()) { + taskService.succeed(taskId, result); + log.info("Apply table task success"); + } else { + taskService.fail(taskId, 0, result); + log.info("Apply table task failed"); + } + } catch (Exception e) { + failure = true; + result.setSuccess(false); + taskService.fail(taskId, 0, result); + log.warn("Apply table task failed, error={}", e.getMessage()); + } finally { + TaskContextHolder.clear(); + } + return result; + } + + @Override + protected boolean isSuccessful() { + return this.success; + } + + @Override + protected boolean isFailure() { + return this.failure; + } + + @Override + protected void onFailure(Long taskId, TaskService taskService) { + TaskContextHolder.trace(getTaskCreatorId(taskId, taskService), taskId); + log.warn("Apply table task failed, taskId={}", taskId); + TaskContextHolder.clear(); + super.onFailure(taskId, taskService); + } + + @Override + protected void onSuccessful(Long taskId, TaskService taskService) { + TaskContextHolder.trace(getTaskCreatorId(taskId, taskService), taskId); + log.info("Apply table task succeed, taskId={}", taskId); + TaskContextHolder.clear(); + super.onSuccessful(taskId, taskService); + updateFlowInstanceStatus(FlowStatus.EXECUTION_SUCCEEDED); + } + + @Override + protected void onTimeout(Long taskId, TaskService taskService) { + TaskContextHolder.trace(getTaskCreatorId(taskId, taskService), taskId); + log.warn("Apply table permission task timeout, taskId={}", taskId); + TaskContextHolder.clear(); + super.onTimeout(taskId, taskService); + } + + @Override + protected void onProgressUpdate(Long taskId, TaskService taskService) { + + } + + @Override + protected boolean cancel(boolean mayInterruptIfRunning, Long taskId, TaskService taskService) { + throw new UnsupportedException(ErrorCodes.RunningTaskNotTerminable, null, + "The task is not terminable during execution"); + } + + @Override + public boolean isCancelled() { + return false; + } + + private long getTaskCreatorId(Long taskId, TaskService taskService) { + if (this.creatorId != null) { + return this.creatorId; + } + return taskService.detail(taskId).getCreatorId(); + } + + private void checkResourceAndPermission(ApplyTableParameter parameter) { + // Check project still exists + Long projectId = parameter.getProject().getId(); + projectRepository.findById(projectId).orElseThrow( + () -> { + log.warn("Project not found, id={}", projectId); + return new NotFoundException(ResourceType.ODC_PROJECT, "id", projectId); + }); + // Check user still exists and is member of the project + userRepository.findById(this.creatorId).orElseThrow( + () -> { + log.warn("User not found, id={}", this.creatorId); + return new NotFoundException(ResourceType.ODC_USER, "id", this.creatorId); + }); + Set projectMemberIds = userResourceRoleRepository.findByResourceId(projectId).stream() + .map(UserResourceRoleEntity::getUserId).collect(Collectors.toSet()); + if (!projectMemberIds.contains(this.creatorId)) { + log.warn("User not member of project, userId={}, projectId={}", this.creatorId, projectId); + throw new IllegalStateException("User not member of project"); + } + // Check databases still exists and belong to the project + List tableIds = parameter.getTables().stream().map(ApplyTable::getTableId).collect(Collectors.toList()); + List databaseIds = + parameter.getTables().stream().map(ApplyTable::getDatabaseId).collect(Collectors.toList()); + Map id2tableEntity = dbObjectRepository.findByIdIn(tableIds).stream() + .collect(Collectors.toMap(DBObjectEntity::getId, t -> t, (t1, t2) -> t1)); + Map id2databaseEntity = databaseRepository.findByIdIn(databaseIds).stream() + .collect(Collectors.toMap(DatabaseEntity::getId, d -> d, (d1, d2) -> d1)); + for (Long tableId : tableIds) { + if (!id2tableEntity.containsKey(tableId)) { + log.warn("Table not found, id={}", tableId); + throw new NotFoundException(ResourceType.ODC_TABLE, "id", tableId); + } + Long databaseId = id2tableEntity.get(tableId).getDatabaseId(); + DatabaseEntity databaseEntity = id2databaseEntity.get(databaseId); + if (databaseEntity == null) { + log.warn("Database not found, id={}", databaseId); + throw new NotFoundException(ResourceType.ODC_DATABASE, "id", databaseId); + } + if (databaseEntity.getProjectId() == null || !databaseEntity.getProjectId().equals(projectId)) { + log.warn("Database not belong to project, databaseId={}, projectId={}", databaseId, projectId); + throw new IllegalStateException("Database not belong to project"); + } + } + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTablePermissionPreprocessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTablePermissionPreprocessor.java new file mode 100644 index 0000000000..bcfd50ed05 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTablePermissionPreprocessor.java @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.permission.table; + +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.i18n.LocaleContextHolder; + +import com.oceanbase.odc.common.i18n.I18n; +import com.oceanbase.odc.common.util.TimeUtils; +import com.oceanbase.odc.core.shared.Verify; +import com.oceanbase.odc.core.shared.constant.ResourceRoleName; +import com.oceanbase.odc.core.shared.constant.ResourceType; +import com.oceanbase.odc.core.shared.constant.TaskType; +import com.oceanbase.odc.core.shared.exception.AccessDeniedException; +import com.oceanbase.odc.core.shared.exception.NotFoundException; +import com.oceanbase.odc.metadb.collaboration.ProjectEntity; +import com.oceanbase.odc.metadb.dbobject.DBObjectEntity; +import com.oceanbase.odc.metadb.dbobject.DBObjectRepository; +import com.oceanbase.odc.service.collaboration.project.ProjectService; +import com.oceanbase.odc.service.connection.ConnectionService; +import com.oceanbase.odc.service.connection.database.DatabaseService; +import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.flow.model.CreateFlowInstanceReq; +import com.oceanbase.odc.service.flow.processor.FlowTaskPreprocessor; +import com.oceanbase.odc.service.flow.processor.Preprocessor; +import com.oceanbase.odc.service.iam.ProjectPermissionValidator; +import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; +import com.oceanbase.odc.service.permission.table.model.ApplyTableParameter; +import com.oceanbase.odc.service.permission.table.model.ApplyTableParameter.ApplyTable; + +/** + * + * @Author: fenghao + * @Create 2024/3/18 17:38 + * @Version 1.0 + */ +@FlowTaskPreprocessor(type = TaskType.APPLY_TABLE_PERMISSION) +public class ApplyTablePermissionPreprocessor implements Preprocessor { + + @Autowired + private ProjectService projectService; + + @Autowired + private ProjectPermissionValidator projectPermissionValidator; + + @Autowired + private DatabaseService databaseService; + + @Autowired + private ConnectionService connectionService; + + @Autowired + private DBObjectRepository dbObjectRepository; + + @Override + public void process(CreateFlowInstanceReq req) { + ApplyTableParameter parameter = (ApplyTableParameter) req.getParameters(); + Verify.notNull(parameter.getProject(), "project"); + Verify.notEmpty(parameter.getTables(), "tables"); + Verify.notEmpty(parameter.getTypes(), "types"); + Verify.notEmpty(parameter.getApplyReason(), "applyReason"); + // Check project permission and fill in project name + Long projectId = parameter.getProject().getId(); + ProjectEntity projectEntity = projectService.nullSafeGet(projectId); + Verify.verify(Boolean.FALSE.equals(projectEntity.getArchived()), "Project is archived"); + projectPermissionValidator.checkProjectRole(projectId, ResourceRoleName.all()); + parameter.getProject().setName(projectEntity.getName()); + // Check table permission and fill in table related information + List tables = parameter.getTables(); + Set tableIds = tables.stream().map(ApplyTable::getTableId).collect(Collectors.toSet()); + Map id2TableEntity = dbObjectRepository.findByIdIn(tableIds).stream() + .collect(Collectors.toMap(DBObjectEntity::getId, t -> t, (t1, t2) -> t1)); + Set databaseIds = + id2TableEntity.values().stream().map(DBObjectEntity::getDatabaseId).collect(Collectors.toSet()); + Map id2Database = databaseService.listDatabasesByIds(databaseIds).stream() + .collect(Collectors.toMap(Database::getId, d -> d, (d1, d2) -> d1)); + Set dataSourceIds = + id2Database.values().stream().map(d -> d.getDataSource().getId()).collect(Collectors.toSet()); + Map id2DataSource = connectionService.innerListByIds(dataSourceIds).stream() + .collect(Collectors.toMap(ConnectionConfig::getId, d -> d, (d1, d2) -> d1)); + for (ApplyTable table : tables) { + DBObjectEntity tableEntity = id2TableEntity.get(table.getTableId()); + if (tableEntity == null) { + throw new NotFoundException(ResourceType.ODC_TABLE, "id", table.getTableId()); + } + table.setTableName(tableEntity.getName()); + Database database = id2Database.get(tableEntity.getDatabaseId()); + if (database == null) { + throw new NotFoundException(ResourceType.ODC_DATABASE, "id", tableEntity.getDatabaseId()); + } + if (database.getProject() == null || !Objects.equals(database.getProject().getId(), projectId)) { + throw new AccessDeniedException(); + } + table.setDatabaseId(database.getId()); + table.setDatabaseName(database.getName()); + ConnectionConfig dataSource = id2DataSource.get(database.getDataSource().getId()); + if (dataSource == null) { + throw new NotFoundException(ResourceType.ODC_CONNECTION, "id", database.getDataSource().getId()); + } + table.setDataSourceId(dataSource.getId()); + table.setDataSourceName(dataSource.getName()); + } + parameter.setExpireTime(parameter.getExpireTime() == null ? TimeUtils.getMySQLMaxDatetime() + : TimeUtils.getEndOfDay(parameter.getExpireTime())); + // Fill in other parameters + req.setProjectId(projectId); + req.setProjectName(projectEntity.getName()); + Locale locale = LocaleContextHolder.getLocale(); + String i18nKey = "com.oceanbase.odc.builtin-resource.permission-apply.table.description"; + req.setDescription(I18n.translate( + i18nKey, + new Object[] {parameter.getTypes().stream().map(DatabasePermissionType::getLocalizedMessage) + .collect(Collectors.joining(","))}, + locale)); + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/TablePermissionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/TablePermissionService.java new file mode 100644 index 0000000000..e4e3baf5d6 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/TablePermissionService.java @@ -0,0 +1,234 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.permission.table; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.validation.Valid; +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; + +import org.apache.commons.lang.time.DateUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.domain.Specification; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.validation.annotation.Validated; + +import com.oceanbase.odc.common.util.TimeUtils; +import com.oceanbase.odc.core.authority.util.Authenticated; +import com.oceanbase.odc.core.authority.util.PreAuthenticate; +import com.oceanbase.odc.core.authority.util.SkipAuthorize; +import com.oceanbase.odc.core.shared.constant.AuthorizationType; +import com.oceanbase.odc.core.shared.constant.PermissionType; +import com.oceanbase.odc.core.shared.constant.ResourceRoleName; +import com.oceanbase.odc.core.shared.constant.ResourceType; +import com.oceanbase.odc.core.shared.exception.AccessDeniedException; +import com.oceanbase.odc.core.shared.exception.NotFoundException; +import com.oceanbase.odc.metadb.dbobject.DBObjectEntity; +import com.oceanbase.odc.metadb.dbobject.DBObjectRepository; +import com.oceanbase.odc.metadb.iam.PermissionEntity; +import com.oceanbase.odc.metadb.iam.PermissionRepository; +import com.oceanbase.odc.metadb.iam.UserPermissionEntity; +import com.oceanbase.odc.metadb.iam.UserPermissionRepository; +import com.oceanbase.odc.metadb.iam.UserTablePermissionEntity; +import com.oceanbase.odc.metadb.iam.UserTablePermissionRepository; +import com.oceanbase.odc.metadb.iam.UserTablePermissionSpec; +import com.oceanbase.odc.service.collaboration.project.ProjectService; +import com.oceanbase.odc.service.collaboration.project.model.Project; +import com.oceanbase.odc.service.connection.database.DatabaseService; +import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.odc.service.iam.PermissionService; +import com.oceanbase.odc.service.iam.ProjectPermissionValidator; +import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; +import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; +import com.oceanbase.odc.service.permission.database.model.ExpirationStatusFilter; +import com.oceanbase.odc.service.permission.table.model.CreateTablePermissionReq; +import com.oceanbase.odc.service.permission.table.model.QueryTablePermissionParams; +import com.oceanbase.odc.service.permission.table.model.UserTablePermission; + +import lombok.extern.slf4j.Slf4j; + +/** + * + * @Author: fenghao + * @Create 2024/3/11 20:49 + * @Version 1.0 + */ +@Service +@Validated +@Authenticated +@Slf4j +public class TablePermissionService { + + @Autowired + private ProjectService projectService; + + @Autowired + private ProjectPermissionValidator projectPermissionValidator; + + @Autowired + @Lazy + private DatabaseService databaseService; + + @Autowired + private AuthenticationFacade authenticationFacade; + + @Autowired + private PermissionRepository permissionRepository; + + @Autowired + private UserPermissionRepository userPermissionRepository; + + @Autowired + private UserTablePermissionRepository userTablePermissionRepository; + + @Autowired + private PermissionService permissionService; + + @Autowired + private DBObjectRepository dbObjectRepository; + + @Value("${odc.iam.permission.expired-retention-time-seconds:7776000}") + private long expiredRetentionTimeSeconds; + + private static final UserTablePermissionMapper mapper = UserTablePermissionMapper.INSTANCE; + private static final int EXPIRING_DAYS = 7; + + @Transactional(rollbackFor = Exception.class) + @SkipAuthorize("permission check inside") + public Page list(@NotNull Long projectId, @NotNull QueryTablePermissionParams params, + Pageable pageable) { + if (params.getUserId() == null || params.getUserId() != authenticationFacade.currentUserId()) { + projectPermissionValidator.checkProjectRole(projectId, + Arrays.asList(ResourceRoleName.OWNER, ResourceRoleName.DBA)); + } else { + projectPermissionValidator.checkProjectRole(projectId, ResourceRoleName.all()); + } + Date expiredTime = new Date(System.currentTimeMillis() - expiredRetentionTimeSeconds * 1000); + permissionService.deleteExpiredPermission(expiredTime); + Date expireTimeThreshold = TimeUtils.getStartOfDay(new Date()); + Specification spec = Specification + .where(UserTablePermissionSpec.projectIdEqual(projectId)) + .and(UserTablePermissionSpec.organizationIdEqual(authenticationFacade.currentOrganizationId())) + .and(UserTablePermissionSpec.userIdEqual(params.getUserId())) + .and(UserTablePermissionSpec.ticketIdEqual(params.getTicketId())) + .and(UserTablePermissionSpec.tableNameLike(params.getFuzzyTableName())) + .and(UserTablePermissionSpec.databaseNameLike(params.getFuzzyDatabaseName())) + .and(UserTablePermissionSpec.dataSourceNameLike(params.getFuzzyDataSourceName())) + .and(UserTablePermissionSpec.typeIn(params.getTypes())) + .and(UserTablePermissionSpec.authorizationTypeEqual(params.getAuthorizationType())) + .and(UserTablePermissionSpec.filterByExpirationStatus(params.getStatuses(), expireTimeThreshold)); + return userTablePermissionRepository.findAll(spec, pageable).map( + e -> { + UserTablePermission permission = mapper.entityToModel(e); + Date expireTime = permission.getExpireTime(); + if (expireTime.before(expireTimeThreshold)) { + permission.setStatus(ExpirationStatusFilter.EXPIRED); + } else if (expireTime.before(DateUtils.addDays(expireTimeThreshold, EXPIRING_DAYS))) { + permission.setStatus(ExpirationStatusFilter.EXPIRING); + } else { + permission.setStatus(ExpirationStatusFilter.NOT_EXPIRED); + } + return permission; + }); + } + + @Transactional(rollbackFor = Exception.class) + @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + public List batchCreate(@NotNull Long projectId, + @NotNull @Valid CreateTablePermissionReq req) { + Set projectIds = projectService.getMemberProjectIds(req.getUserId()); + if (!projectIds.contains(projectId)) { + throw new AccessDeniedException(); + } + List tables = dbObjectRepository.findByIdIn(req.getTableIds()); + Map id2TableEntity = tables.stream() + .collect(Collectors.toMap(DBObjectEntity::getId, t -> t, (t1, t2) -> t1)); + Map id2Database = databaseService + .listDatabasesByIds(tables.stream().map(DBObjectEntity::getDatabaseId).collect(Collectors.toList())) + .stream().collect(Collectors.toMap(Database::getId, d -> d, (d1, d2) -> d1)); + for (Long tableId : req.getTableIds()) { + DBObjectEntity table = id2TableEntity.get(tableId); + if (table == null) { + throw new NotFoundException(ResourceType.ODC_TABLE, "id", tableId); + } + Project project = id2Database.get(table.getDatabaseId()).getProject(); + if (project == null || !Objects.equals(project.getId(), projectId)) { + throw new AccessDeniedException(); + } + } + List permissionEntities = new ArrayList<>(); + Long organizationId = authenticationFacade.currentOrganizationId(); + Long creatorId = authenticationFacade.currentUserId(); + Date expireTime = req.getExpireTime() == null ? TimeUtils.getMySQLMaxDatetime() + : TimeUtils.getEndOfDay(req.getExpireTime()); + for (Long tableId : req.getTableIds()) { + DBObjectEntity table = id2TableEntity.get(tableId); + for (DatabasePermissionType permissionType : req.getTypes()) { + PermissionEntity entity = new PermissionEntity(); + entity.setAction(permissionType.getAction()); + entity.setResourceIdentifier(ResourceType.ODC_DATABASE.name() + ":" + table.getDatabaseId() + "/" + + ResourceType.ODC_TABLE.name() + ":" + table.getId()); + entity.setType(PermissionType.PUBLIC_RESOURCE); + entity.setCreatorId(creatorId); + entity.setOrganizationId(organizationId); + entity.setBuiltIn(false); + entity.setExpireTime(expireTime); + entity.setAuthorizationType(AuthorizationType.USER_AUTHORIZATION); + entity.setResourceType(ResourceType.ODC_TABLE); + entity.setResourceId(table.getId()); + permissionEntities.add(entity); + } + } + List saved = permissionRepository.batchCreate(permissionEntities); + List userPermissionEntities = new ArrayList<>(); + for (PermissionEntity permissionEntity : saved) { + UserPermissionEntity userPermissionEntity = new UserPermissionEntity(); + userPermissionEntity.setUserId(req.getUserId()); + userPermissionEntity.setPermissionId(permissionEntity.getId()); + userPermissionEntity.setCreatorId(creatorId); + userPermissionEntity.setOrganizationId(organizationId); + userPermissionEntities.add(userPermissionEntity); + } + userPermissionRepository.batchCreate(userPermissionEntities); + return saved.stream().map(PermissionEntity::getId).map(UserTablePermission::from).collect(Collectors.toList()); + } + + @Transactional(rollbackFor = Exception.class) + @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + public List batchRevoke(@NotNull Long projectId, @NotEmpty List ids) { + List entities = + userTablePermissionRepository.findByProjectIdAndIdIn(projectId, ids); + List permissionIds = + entities.stream().map(UserTablePermissionEntity::getId).collect(Collectors.toList()); + permissionRepository.deleteByIds(permissionIds); + userPermissionRepository.deleteByPermissionIds(permissionIds); + return permissionIds.stream().map(UserTablePermission::from).collect(Collectors.toList()); + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/UserTablePermissionMapper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/UserTablePermissionMapper.java new file mode 100644 index 0000000000..6c033dec52 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/UserTablePermissionMapper.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.permission.table; + +import org.mapstruct.Mapper; +import org.mapstruct.Mapping; +import org.mapstruct.Named; +import org.mapstruct.factory.Mappers; + +import com.oceanbase.odc.metadb.iam.UserTablePermissionEntity; +import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; +import com.oceanbase.odc.service.permission.table.model.UserTablePermission; + +/** + * ClassName: UserTablePermissionMapper.java Package: com.oceanbase.odc.service.permission.table + * Description: + * + * @Author: fenghao + * @Create 2024/3/11 20:26 + * @Version 1.0 + */ +@Mapper +public interface UserTablePermissionMapper { + + UserTablePermissionMapper INSTANCE = Mappers.getMapper(UserTablePermissionMapper.class); + + @Mapping(source = "action", target = "type", qualifiedByName = "actionToType") + UserTablePermission entityToModel(UserTablePermissionEntity entity); + + @Named("actionToType") + static DatabasePermissionType actionToType(String action) { + return DatabasePermissionType.from(action); + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/model/ApplyTableParameter.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/model/ApplyTableParameter.java new file mode 100644 index 0000000000..5aef351d89 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/model/ApplyTableParameter.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.permission.table.model; + +import java.io.Serializable; +import java.util.Date; +import java.util.List; + +import com.oceanbase.odc.core.flow.model.TaskParameters; +import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; +import com.oceanbase.odc.service.permission.project.ApplyProjectParameter.ApplyProject; +import com.oceanbase.odc.service.permission.table.ApplyTablePermissionPreprocessor; + +import lombok.Data; + +/** + * + * @Author: fenghao + * @Create 2024/3/14 16:30 + * @Version 1.0 + */ +@Data +public class ApplyTableParameter implements Serializable, TaskParameters { + + private static final long serialVersionUID = -2482302525012272875L; + + /** + * Project to be applied for, required + */ + private ApplyProject project; + /** + * Tables to be applied for, required + */ + private List tables; + /** + * Permission types to be applied for, required + */ + private List types; + /** + * Expiration time, null means no expiration, optional + */ + private Date expireTime; + /** + * Reason for application, required + */ + private String applyReason; + + @Data + public static class ApplyTable implements Serializable { + + private static final long serialVersionUID = -8433967513537417701L; + + /** + * ID of the table to be applied for, required + */ + private Long tableId; + /** + * Following fields are all filled in by the {@link ApplyTablePermissionPreprocessor} + */ + private String tableName; + private Long databaseId; + private String databaseName; + private Long dataSourceId; + private String dataSourceName; + + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/model/ApplyTableResult.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/model/ApplyTableResult.java new file mode 100644 index 0000000000..954ccfb13e --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/model/ApplyTableResult.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.permission.table.model; + +import com.oceanbase.odc.core.flow.model.FlowTaskResult; + +import lombok.Data; + +/** + * + * @Author: fenghao + * @Create 2024/3/18 16:51 + * @Version 1.0 + */ +@Data +public class ApplyTableResult implements FlowTaskResult { + + private boolean success; + private ApplyTableParameter parameter; + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/model/CreateTablePermissionReq.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/model/CreateTablePermissionReq.java new file mode 100644 index 0000000000..49becb61ce --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/model/CreateTablePermissionReq.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.permission.table.model; + +import java.util.Date; +import java.util.List; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; + +import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; + +import lombok.Data; + +/** + * + * @Author: fenghao + * @Create 2024/3/11 20:26 + * @Version 1.0 + */ +@Data +public class CreateTablePermissionReq { + + @NotEmpty + private List tableIds; + + @NotEmpty + private List types; + + private Date expireTime; + + @NotNull + private Long userId; + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/model/QueryTablePermissionParams.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/model/QueryTablePermissionParams.java new file mode 100644 index 0000000000..1ddb4af84d --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/model/QueryTablePermissionParams.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.permission.table.model; + +import com.oceanbase.odc.service.permission.database.model.QueryDatabasePermissionParams; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.SuperBuilder; + +/** + * + * @Author: fenghao + * @Create 2024/3/11 20:46 + * @Version 1.0 + */ +@Data +@SuperBuilder +@EqualsAndHashCode(callSuper = true) +public class QueryTablePermissionParams extends QueryDatabasePermissionParams { + + private String fuzzyTableName; + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/model/UserTablePermission.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/model/UserTablePermission.java new file mode 100644 index 0000000000..2022195e18 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/model/UserTablePermission.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.permission.table.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty.Access; +import com.oceanbase.odc.service.permission.database.model.UserDatabasePermission; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * ClassName: UserTablePermission Package: com.oceanbase.odc.service.permission.table.model.table + * Description: + * + * @Author: fenghao + * @Create 2024/3/11 20:39 + * @Version 1.0 + */ +@Data +@EqualsAndHashCode(callSuper = true) +public class UserTablePermission extends UserDatabasePermission { + + @JsonProperty(access = Access.READ_ONLY) + private String tableId; + + @JsonProperty(access = Access.READ_ONLY) + private String tableName; + + public static UserTablePermission from(Long permissionId) { + UserTablePermission permission = new UserTablePermission(); + permission.setId(permissionId); + return permission; + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/regulation/risklevel/RiskLevelService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/regulation/risklevel/RiskLevelService.java index 0e68621e0d..be73d22575 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/regulation/risklevel/RiskLevelService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/regulation/risklevel/RiskLevelService.java @@ -117,6 +117,7 @@ public Optional findById(@NonNull Long id) { return riskLevelRepository.findById(id).map(this::entityToModel); } + @SkipAuthorize("internal usage") public boolean exists(@NonNull Long organizationId, @NonNull Long id) { RiskLevelEntity example = new RiskLevelEntity(); @@ -151,5 +152,4 @@ private RiskLevel entityToModel(RiskLevelEntity entity) { model.setApprovalFlowConfig(approvalFlowConfigService.findById(entity.getApprovalFlowConfigId())); return model; } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java index 37ab674b7b..2132998e0d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java @@ -82,6 +82,7 @@ import com.oceanbase.odc.service.common.util.WebResponseUtils; import com.oceanbase.odc.service.config.UserConfigFacade; import com.oceanbase.odc.service.connection.ConnectionService; +import com.oceanbase.odc.service.connection.database.model.UnauthorizedDBResource; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.db.browser.DBSchemaAccessors; import com.oceanbase.odc.service.db.session.DefaultDBSessionManage; @@ -91,7 +92,6 @@ import com.oceanbase.odc.service.feature.AllFeatures; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; -import com.oceanbase.odc.service.permission.database.model.UnauthorizedDatabase; import com.oceanbase.odc.service.session.interceptor.SqlCheckInterceptor; import com.oceanbase.odc.service.session.interceptor.SqlConsoleInterceptor; import com.oceanbase.odc.service.session.interceptor.SqlExecuteInterceptorService; @@ -192,13 +192,13 @@ public SqlExecuteResult queryTableOrViewData(@NotNull String sessionId, asyncExecuteReq.setFullLinkTraceEnabled(false); SqlAsyncExecuteResp resp = execute(sessionId, asyncExecuteReq, false); - List unauthorizedDatabases = resp.getUnauthorizedDatabases(); - if (CollectionUtils.isNotEmpty(unauthorizedDatabases)) { - UnauthorizedDatabase unauthorizedDatabase = unauthorizedDatabases.get(0); + List unauthorizedDBResources = resp.getUnauthorizedDBResources(); + if (CollectionUtils.isNotEmpty(unauthorizedDBResources)) { + UnauthorizedDBResource unauthorizedDBResource = unauthorizedDBResources.get(0); throw new BadRequestException(ErrorCodes.DatabaseAccessDenied, - new Object[] {unauthorizedDatabase.getUnauthorizedPermissionTypes().stream() + new Object[] {unauthorizedDBResource.getUnauthorizedPermissionTypes().stream() .map(DatabasePermissionType::getLocalizedMessage).collect(Collectors.joining(","))}, - "Lack permission for the database with id " + unauthorizedDatabase.getId()); + "Lack permission for the database with id " + unauthorizedDBResource.getDatabaseId()); } String requestId = resp.getRequestId(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectSessionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectSessionService.java index 42b35b3605..c5e2889c47 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectSessionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectSessionService.java @@ -87,7 +87,7 @@ import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.iam.auth.AuthorizationFacade; import com.oceanbase.odc.service.lab.model.LabProperties; -import com.oceanbase.odc.service.permission.database.DatabasePermissionHelper; +import com.oceanbase.odc.service.permission.DBResourcePermissionHelper; import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; import com.oceanbase.odc.service.session.factory.DefaultConnectSessionIdGenerator; @@ -142,7 +142,7 @@ public class ConnectSessionService { @Autowired private SecurityManager securityManager; @Autowired - private DatabasePermissionHelper databasePermissionHelper; + private DBResourcePermissionHelper permissionHelper; @Autowired private CloudMetadataClient cloudMetadataClient; @Autowired @@ -234,7 +234,7 @@ public ConnectionSession create(@NotNull CreateSessionReq req) { throw new AccessDeniedException(); } Map> id2PermissionTypes = - databasePermissionHelper.getPermissions(Collections.singleton(req.getDbId())); + permissionHelper.getDBPermissions(Collections.singleton(req.getDbId())); if (!id2PermissionTypes.containsKey(req.getDbId()) || id2PermissionTypes.get(req.getDbId()).isEmpty()) { throw new AccessDeniedException(); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/DatabasePermissionInterceptor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/DBResourcePermissionInterceptor.java similarity index 70% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/DatabasePermissionInterceptor.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/DBResourcePermissionInterceptor.java index ba1696c133..fc65984fd9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/DatabasePermissionInterceptor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/DBResourcePermissionInterceptor.java @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.oceanbase.odc.service.session.interceptor; import java.util.HashMap; @@ -33,36 +32,42 @@ import com.oceanbase.odc.core.shared.constant.OrganizationType; import com.oceanbase.odc.core.sql.execute.SqlExecuteStages; import com.oceanbase.odc.service.connection.database.DatabaseService; +import com.oceanbase.odc.service.connection.database.model.DBResource; +import com.oceanbase.odc.service.connection.database.model.UnauthorizedDBResource; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; +import com.oceanbase.odc.service.permission.DBResourcePermissionHelper; import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; -import com.oceanbase.odc.service.permission.database.model.UnauthorizedDatabase; import com.oceanbase.odc.service.session.model.AsyncExecuteContext; import com.oceanbase.odc.service.session.model.SqlAsyncExecuteReq; import com.oceanbase.odc.service.session.model.SqlAsyncExecuteResp; import com.oceanbase.odc.service.session.model.SqlExecuteResult; import com.oceanbase.odc.service.session.model.SqlTuplesWithViolation; -import com.oceanbase.odc.service.session.util.SchemaExtractor; +import com.oceanbase.odc.service.session.util.DBSchemaExtractor; +import com.oceanbase.odc.service.session.util.DBSchemaExtractor.DBSchemaIdentity; import com.oceanbase.tools.dbbrowser.parser.constant.SqlType; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; /** - * @Author: Lebie - * @Date: 2023/8/8 22:23 - * @Description: [] + * + * @Author: fenghao + * @Create 2024/3/20 14:44 + * @Version 1.0 */ @Slf4j @Component -public class DatabasePermissionInterceptor extends BaseTimeConsumingInterceptor { - +public class DBResourcePermissionInterceptor extends BaseTimeConsumingInterceptor { @Autowired private DatabaseService databaseService; @Autowired private AuthenticationFacade authenticationFacade; + @Autowired + private DBResourcePermissionHelper dbResourcePermissionHelper; + @Override public int getOrder() { return 0; @@ -76,24 +81,27 @@ public boolean doPreHandle(@NonNull SqlAsyncExecuteReq request, @NonNull SqlAsyn } ConnectionConfig connectionConfig = (ConnectionConfig) ConnectionSessionUtil.getConnectionConfig(session); String currentSchema = ConnectionSessionUtil.getCurrentSchema(session); - Map> schemaName2SqlTypes = SchemaExtractor.listSchemaName2SqlTypes( + Map> identity2Types = DBSchemaExtractor.listDBSchemasWithSqlTypes( response.getSqls().stream().map(SqlTuplesWithViolation::getSqlTuple).collect(Collectors.toList()), - currentSchema, session.getDialectType()); - Map> schemaName2PermissionTypes = new HashMap<>(); - for (Entry> entry : schemaName2SqlTypes.entrySet()) { + session.getDialectType(), currentSchema); + Map> resource2PermissionTypes = new HashMap<>(); + for (Entry> entry : identity2Types.entrySet()) { + DBSchemaIdentity identity = entry.getKey(); Set sqlTypes = entry.getValue(); if (CollectionUtils.isNotEmpty(sqlTypes)) { Set permissionTypes = sqlTypes.stream().map(DatabasePermissionType::from) .filter(Objects::nonNull).collect(Collectors.toSet()); if (CollectionUtils.isNotEmpty(permissionTypes)) { - schemaName2PermissionTypes.put(entry.getKey(), permissionTypes); + resource2PermissionTypes.put( + DBResource.from(connectionConfig, identity.getSchema(), identity.getTable()), + permissionTypes); } } } - List unauthorizedDatabases = - databaseService.filterUnauthorizedDatabases(schemaName2PermissionTypes, connectionConfig.getId(), true); - if (CollectionUtils.isNotEmpty(unauthorizedDatabases)) { - response.setUnauthorizedDatabases(unauthorizedDatabases); + List unauthorizedDBResource = dbResourcePermissionHelper + .filterUnauthorizedDBResources(resource2PermissionTypes, false); + if (CollectionUtils.isNotEmpty(unauthorizedDBResource)) { + response.setUnauthorizedDBResources(unauthorizedDBResource); return false; } return true; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/SwitchDatabaseInterceptor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/SwitchDatabaseInterceptor.java index 9d41f99e8e..31f01d83f8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/SwitchDatabaseInterceptor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/SwitchDatabaseInterceptor.java @@ -26,7 +26,7 @@ import com.oceanbase.odc.core.sql.execute.model.SqlExecuteStatus; import com.oceanbase.odc.service.session.model.AsyncExecuteContext; import com.oceanbase.odc.service.session.model.SqlExecuteResult; -import com.oceanbase.odc.service.session.util.SchemaExtractor; +import com.oceanbase.odc.service.session.util.DBSchemaExtractor; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -46,7 +46,7 @@ public void afterCompletion(@NonNull SqlExecuteResult response, @NonNull Connect return; } String currentSchema = ConnectionSessionUtil.getCurrentSchema(session); - Optional switchSchema = SchemaExtractor + Optional switchSchema = DBSchemaExtractor .extractSwitchedSchemaName(Collections.singletonList(response.getSqlTuple()), session.getDialectType()); if (switchSchema.isPresent() && !Objects.equals(switchSchema.get(), currentSchema)) { ConnectionSessionUtil.setCurrentSchema(session, switchSchema.get()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlAsyncExecuteResp.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlAsyncExecuteResp.java index c54eb7d91e..5203536931 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlAsyncExecuteResp.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlAsyncExecuteResp.java @@ -20,7 +20,7 @@ import java.util.stream.Collectors; import com.oceanbase.odc.core.sql.execute.model.SqlTuple; -import com.oceanbase.odc.service.permission.database.model.UnauthorizedDatabase; +import com.oceanbase.odc.service.connection.database.model.UnauthorizedDBResource; import com.oceanbase.odc.service.regulation.ruleset.model.Rule; import lombok.Data; @@ -37,7 +37,7 @@ public class SqlAsyncExecuteResp { private String requestId; private List violatedRules; private List sqls; - private List unauthorizedDatabases; + private List unauthorizedDBResources; public SqlAsyncExecuteResp(String requestId, List sqls) { this.requestId = requestId; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/SchemaExtractor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/DBSchemaExtractor.java similarity index 70% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/SchemaExtractor.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/DBSchemaExtractor.java index 3c18261317..82508fbd74 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/SchemaExtractor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/DBSchemaExtractor.java @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.oceanbase.odc.service.session.util; import java.util.ArrayList; @@ -26,7 +25,6 @@ import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.commons.collections4.CollectionUtils; @@ -35,7 +33,6 @@ import com.oceanbase.odc.core.sql.execute.model.SqlTuple; import com.oceanbase.odc.core.sql.parser.AbstractSyntaxTree; import com.oceanbase.odc.core.sql.parser.AbstractSyntaxTreeFactories; -import com.oceanbase.odc.core.sql.parser.AbstractSyntaxTreeFactory; import com.oceanbase.tools.dbbrowser.parser.constant.SqlType; import com.oceanbase.tools.dbbrowser.parser.result.BasicResult; import com.oceanbase.tools.dbbrowser.parser.result.ParseMysqlPLResult; @@ -69,14 +66,17 @@ import com.oceanbase.tools.sqlparser.statement.expression.FunctionCall; import com.oceanbase.tools.sqlparser.statement.expression.RelationReference; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.NoArgsConstructor; /** - * @Author: Lebie - * @Date: 2023/8/10 13:59 - * @Description: [] + * @author gaoda.xy + * @date 2024/5/7 11:53 */ -public class SchemaExtractor { +public class DBSchemaExtractor { public static Optional extractSwitchedSchemaName(List sqlTuples, DialectType dialectType) { Optional schemaName = Optional.empty(); @@ -107,9 +107,9 @@ public static Optional extractSwitchedSchemaName(List sqlTuple return schemaName; } - public static Map> listSchemaName2SqlTypes(List sqlTuples, String defaultSchema, - DialectType dialectType) { - Map> schemaName2SqlTypes = new HashMap<>(); + public static Map> listDBSchemasWithSqlTypes(List sqlTuples, + DialectType dialectType, String defaultSchema) { + Map> res = new HashMap<>(); for (SqlTuple sqlTuple : sqlTuples) { try { AbstractSyntaxTree ast = sqlTuple.getAst(); @@ -117,108 +117,105 @@ public static Map> listSchemaName2SqlTypes(List s sqlTuple.initAst(AbstractSyntaxTreeFactories.getAstFactory(dialectType, 0)); ast = sqlTuple.getAst(); } - Set schemaNames = listSchemaNames(ast, defaultSchema, dialectType); + Set identities = listDBSchemas(ast, dialectType, defaultSchema); SqlType sqlType = SqlType.OTHERS; BasicResult basicResult = ast.getParseResult(); if (Objects.nonNull(basicResult) && Objects.nonNull(basicResult.getSqlType()) && basicResult.getSqlType() != SqlType.UNKNOWN) { sqlType = basicResult.getSqlType(); } - for (String schemaName : schemaNames) { - Set sqlTypes = schemaName2SqlTypes.computeIfAbsent(schemaName, k -> new HashSet<>()); - sqlTypes.add(sqlType); + for (DBSchemaIdentity identity : identities) { + res.computeIfAbsent(identity, k -> new HashSet<>()).add(sqlType); } } catch (Exception e) { // just eat exception due to parse failed } } - return schemaName2SqlTypes; + return res; } - public static Set listSchemaNames(List sqls, DialectType dialectType, String defaultSchema) { - AbstractSyntaxTreeFactory factory = AbstractSyntaxTreeFactories.getAstFactory(dialectType, 0); - if (factory == null) { - return new HashSet<>(); - } - return sqls.stream().flatMap(sql -> { - try { - return listSchemaNames(factory.buildAst(sql), defaultSchema, dialectType).stream(); - } catch (Exception e) { - // just eat exception due to parse failed - return Stream.empty(); - } - }).collect(Collectors.toSet()); - } - - private static Set listSchemaNames(AbstractSyntaxTree ast, String defaultSchema, DialectType dialectType) { - List relationFactorList; + private static Set listDBSchemas(AbstractSyntaxTree ast, DialectType dialectType, + String defaultSchema) { + Set identities = new HashSet<>(); BasicResult basicResult = ast.getParseResult(); if (dialectType.isMysql() || dialectType.isDoris()) { if (basicResult.isPlDdl() || basicResult instanceof ParseMysqlPLResult) { OBMySQLPLRelationFactorVisitor visitor = new OBMySQLPLRelationFactorVisitor(); visitor.visit(ast.getRoot()); - relationFactorList = visitor.getRelationFactorList(); + identities = visitor.getIdentities(); } else { OBMySQLRelationFactorVisitor visitor = new OBMySQLRelationFactorVisitor(); visitor.visit(ast.getRoot()); - relationFactorList = visitor.getRelationFactorList(); + identities = visitor.getIdentities(); } - return relationFactorList.stream() - .filter(r -> StringUtils.isBlank(r.getUserVariable())) - .map(r -> { - String schema = StringUtils.isNotBlank(r.getSchema()) ? r.getSchema() : defaultSchema; - return StringUtils.unquoteMySqlIdentifier(schema); - }) - .filter(Objects::nonNull).collect(Collectors.toSet()); + identities = identities.stream().map(e -> { + DBSchemaIdentity i = new DBSchemaIdentity(e.getSchema(), e.getTable()); + String schema = StringUtils.isNotBlank(i.getSchema()) ? i.getSchema() : defaultSchema; + i.setSchema(StringUtils.unquoteMySqlIdentifier(schema)); + i.setTable(StringUtils.unquoteMySqlIdentifier(i.getTable())); + return i; + }).collect(Collectors.toSet()); } else if (dialectType.isOracle()) { if (basicResult.isPlDdl() || basicResult instanceof ParseOraclePLResult) { OBOraclePLRelationFactorVisitor visitor = new OBOraclePLRelationFactorVisitor(); visitor.visit(ast.getRoot()); - relationFactorList = visitor.getRelationFactorList(); + identities = visitor.getIdentities(); } else { OBOracleRelationFactorVisitor visitor = new OBOracleRelationFactorVisitor(); visitor.visit(ast.getRoot()); - relationFactorList = visitor.getRelationFactorList(); + identities = visitor.getIdentities(); } - return relationFactorList.stream() - .filter(r -> StringUtils.isBlank(r.getUserVariable())) - .map(r -> { - String schema = StringUtils.isNotBlank(r.getSchema()) ? r.getSchema() : defaultSchema; - if (StringUtils.startsWith(schema, "\"") && StringUtils.endsWith(schema, "\"")) { - return StringUtils.unquoteOracleIdentifier(schema); - } - return StringUtils.upperCase(schema); - }).filter(Objects::nonNull).collect(Collectors.toSet()); + identities = identities.stream().map(e -> { + DBSchemaIdentity i = new DBSchemaIdentity(e.getSchema(), e.getTable()); + String schema = StringUtils.isNotBlank(i.getSchema()) ? i.getSchema() : defaultSchema; + if (StringUtils.startsWith(schema, "\"") && StringUtils.endsWith(schema, "\"")) { + schema = StringUtils.unquoteOracleIdentifier(schema); + } else { + schema = StringUtils.upperCase(schema); + } + String table = StringUtils.isNotBlank(i.getTable()) ? i.getTable() : null; + if (StringUtils.startsWith(table, "\"") && StringUtils.endsWith(table, "\"")) { + table = StringUtils.unquoteOracleIdentifier(table); + } else { + table = StringUtils.upperCase(table); + } + i.setSchema(schema); + i.setTable(table); + return i; + }).collect(Collectors.toSet()); } - return new HashSet<>(); + return identities; } + @Getter private static class OBMySQLRelationFactorVisitor extends OBParserBaseVisitor { - private final List relationFactorList = new ArrayList<>(); + private final Set identities = new HashSet<>(); @Override public RelationFactor visitRelation_factor(Relation_factorContext ctx) { - relationFactorList.add(MySQLFromReferenceFactory.getRelationFactor(ctx)); + addRelationFactor(MySQLFromReferenceFactory.getRelationFactor(ctx)); return null; } @Override public RelationFactor visitNormal_relation_factor(Normal_relation_factorContext ctx) { - relationFactorList.add(MySQLFromReferenceFactory.getRelationFactor(ctx)); + addRelationFactor(MySQLFromReferenceFactory.getRelationFactor(ctx)); return null; } @Override public RelationFactor visitDot_relation_factor(Dot_relation_factorContext ctx) { - relationFactorList.add(new RelationFactor(ctx, ctx.relation_name().getText())); + addRelationFactor(new RelationFactor(ctx, ctx.relation_name().getText())); return null; } @Override public RelationFactor visitRelation_factor_with_star(Relation_factor_with_starContext ctx) { - ctx.relation_name().stream().forEach(r -> relationFactorList.add(new RelationFactor(ctx, r.getText()))); + ctx.relation_name().forEach(r -> { + addRelationFactor(new RelationFactor(ctx, r.getText())); + }); return null; } @@ -226,7 +223,7 @@ public RelationFactor visitRelation_factor_with_star(Relation_factor_with_starCo public RelationFactor visitDatabase_factor(Database_factorContext ctx) { RelationFactor relationFactor = new RelationFactor(ctx, ""); relationFactor.setSchema(ctx.relation_name().getText()); - relationFactorList.add(relationFactor); + identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); return null; } @@ -246,34 +243,96 @@ public RelationFactor visitSimple_expr(Simple_exprContext ctx) { String[] names = relationName.split("\\."); relationFactor.setSchema(names[0]); } - relationFactorList.add(relationFactor); + identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); } return null; } + private void addRelationFactor(RelationFactor rf) { + if (StringUtils.isBlank(rf.getUserVariable())) { + identities.add(new DBSchemaIdentity(rf.getSchema(), rf.getRelation())); + } + } + } + @Getter - private static class OBOracleRelationFactorVisitor extends - com.oceanbase.tools.sqlparser.oboracle.OBParserBaseVisitor { + private static class OBMySQLPLRelationFactorVisitor extends PLParserBaseVisitor { + private final Set identities = new HashSet<>(); private final List relationFactorList = new ArrayList<>(); + @Override + public RelationFactor visitSp_name(Sp_nameContext ctx) { + List idents = ctx.ident(); + RelationFactor rf; + if (idents.size() == 1) { + rf = new RelationFactor(idents.get(0).getText()); + } else { + rf = new RelationFactor(idents.get(idents.size() - 1).getText()); + rf.setSchema(idents.get(0).getText()); + } + identities.add(new DBSchemaIdentity(rf.getSchema(), null)); + return null; + } + + @Override + public RelationFactor visitSp_call_name(Sp_call_nameContext ctx) { + List idents = ctx.ident(); + RelationFactor rf; + if (idents.size() == 1) { + rf = new RelationFactor(idents.get(0).getText()); + } else { + // If there exists two idents, we can not determine weather it is a schema or a package (because OB + // MySQL holds system package such as `dbms_stats`) name. The provisional program is ignoring the + // package name and always treat it as schema name. + rf = new RelationFactor(idents.get(idents.size() - 1).getText()); + rf.setSchema(idents.get(0).getText()); + } + identities.add(new DBSchemaIdentity(rf.getSchema(), null)); + return null; + } + + } + + + @Getter + private static class OBMySQLUseDatabaseStmtVisitor extends OBParserBaseVisitor { + + private final Set schemaSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + + @Override + public Void visitUse_database_stmt(Use_database_stmtContext ctx) { + Relation_nameContext relationName = ctx.database_factor().relation_name(); + schemaSet.add(StringUtils.unquoteMySqlIdentifier(relationName.getText())); + return null; + } + + } + + + @Getter + private static class OBOracleRelationFactorVisitor + extends com.oceanbase.tools.sqlparser.oboracle.OBParserBaseVisitor { + + private final Set identities = new HashSet<>(); + @Override public RelationFactor visitRelation_factor(OBParser.Relation_factorContext ctx) { - relationFactorList.add(OracleFromReferenceFactory.getRelationFactor(ctx)); + addRelationFactor(OracleFromReferenceFactory.getRelationFactor(ctx)); return null; } @Override public RelationFactor visitNormal_relation_factor(OBParser.Normal_relation_factorContext ctx) { - relationFactorList.add(OracleFromReferenceFactory.getRelationFactor(ctx)); + addRelationFactor(OracleFromReferenceFactory.getRelationFactor(ctx)); return null; } @Override public RelationFactor visitDot_relation_factor(OBParser.Dot_relation_factorContext ctx) { - relationFactorList.add(new RelationFactor(ctx, ctx.relation_name().getText())); + addRelationFactor(new RelationFactor(ctx, ctx.relation_name().getText())); return null; } @@ -281,7 +340,7 @@ public RelationFactor visitDot_relation_factor(OBParser.Dot_relation_factorConte public RelationFactor visitDatabase_factor(OBParser.Database_factorContext ctx) { RelationFactor relationFactor = new RelationFactor(ctx, ""); relationFactor.setSchema(ctx.relation_name().getText()); - relationFactorList.add(relationFactor); + identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); return null; } @@ -295,7 +354,7 @@ public RelationFactor visitRoutine_access_name(OBParser.Routine_access_nameConte // The provisional program is ignoring the package name and always treat it as schema name. relationFactor.setSchema(varNames.get(0).getText()); } - relationFactorList.add(relationFactor); + identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); return null; } @@ -304,7 +363,7 @@ public RelationFactor visitCurrent_schema(OBParser.Current_schemaContext ctx) { OBParser.Relation_nameContext relationName = ctx.relation_name(); RelationFactor relationFactor = new RelationFactor(ctx, ""); relationFactor.setSchema(relationName.getText()); - relationFactorList.add(relationFactor); + identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); return null; } @@ -320,89 +379,49 @@ public RelationFactor visitObj_access_ref(OBParser.Obj_access_refContext ctx) { if (e instanceof FunctionCall) { RelationFactor relationFactor = new RelationFactor(((FunctionCall) e).getFunctionName()); relationFactor.setSchema(((RelationReference) expr).getRelationName()); - relationFactorList.add(relationFactor); + identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); } } else if (expr instanceof FunctionCall) { RelationFactor relationFactor = new RelationFactor(((FunctionCall) expr).getFunctionName()); - relationFactorList.add(relationFactor); + identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); } return null; } - } - - @Getter - private static class OBMySQLPLRelationFactorVisitor extends PLParserBaseVisitor { - - private final List relationFactorList = new ArrayList<>(); - - @Override - public RelationFactor visitSp_name(Sp_nameContext ctx) { - List idents = ctx.ident(); - if (idents.size() == 1) { - relationFactorList.add(new RelationFactor(idents.get(0).getText())); - } else { - RelationFactor relationFactor = new RelationFactor(idents.get(idents.size() - 1).getText()); - relationFactor.setSchema(idents.get(0).getText()); - relationFactorList.add(relationFactor); + private void addRelationFactor(RelationFactor rf) { + if (StringUtils.isBlank(rf.getUserVariable())) { + identities.add(new DBSchemaIdentity(rf.getSchema(), rf.getRelation())); } - return null; - } - - @Override - public RelationFactor visitSp_call_name(Sp_call_nameContext ctx) { - List idents = ctx.ident(); - if (idents.size() == 1) { - relationFactorList.add(new RelationFactor(idents.get(0).getText())); - } else { - // If there exists two idents, we can not determine weather it is a schema or a package (because OB - // MySQL holds system package such as `dbms_stats`) name. The provisional program is ignoring the - // package name and always treat it as schema name. - RelationFactor relationFactor = new RelationFactor(idents.get(idents.size() - 1).getText()); - relationFactor.setSchema(idents.get(0).getText()); - relationFactorList.add(relationFactor); - } - return null; } } + @Getter private static class OBOraclePLRelationFactorVisitor extends com.oceanbase.tools.sqlparser.oboracle.PLParserBaseVisitor { private final List relationFactorList = new ArrayList<>(); + private final Set identities = new HashSet<>(); @Override public RelationFactor visitPl_schema_name(Pl_schema_nameContext ctx) { List identifiers = ctx.identifier(); + RelationFactor rf; if (identifiers.size() == 1) { - relationFactorList.add(new RelationFactor(identifiers.get(0).getText())); + rf = new RelationFactor(identifiers.get(0).getText()); } else { // If there exists two identifiers, we can not determine weather it is a schema or a package name. // The provisional program is ignoring the package name and always treat it as schema name. - RelationFactor relationFactor = new RelationFactor(identifiers.get(identifiers.size() - 1).getText()); - relationFactor.setSchema(identifiers.get(0).getText()); - relationFactorList.add(relationFactor); + rf = new RelationFactor(identifiers.get(identifiers.size() - 1).getText()); + rf.setSchema(identifiers.get(0).getText()); } + identities.add(new DBSchemaIdentity(rf.getSchema(), null)); return null; } } - @Getter - private static class OBMySQLUseDatabaseStmtVisitor extends OBParserBaseVisitor { - - private final Set schemaSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - - @Override - public Void visitUse_database_stmt(Use_database_stmtContext ctx) { - Relation_nameContext relationName = ctx.database_factor().relation_name(); - schemaSet.add(StringUtils.unquoteMySqlIdentifier(relationName.getText())); - return null; - } - - } @Getter private static class OBOracleCurrentSchemaVisitor @@ -419,4 +438,16 @@ public Void visitCurrent_schema(Current_schemaContext ctx) { } + + @Data + @NoArgsConstructor + @AllArgsConstructor + @EqualsAndHashCode + public static class DBSchemaIdentity { + + private String schema; + private String table; + + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskService.java index 04c693c3d2..3751289bc1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskService.java @@ -94,6 +94,7 @@ public class TaskService { private static final String APPLY_DATABASE_LOG_PATH_PATTERN = "%s/apply-database/%d/%s/apply-database-task.%s"; private static final String STRUCTURE_COMPARISON_LOG_PATH_PATTERN = "%s/structure-comparison/%d/%s/structure-comparison.%s"; + private static final String APPLY_TABLE_LOG_PATH_PATTERN = "%s/apply-table/%d/%s/apply-table-task.%s"; @Autowired public TaskService(@Value("${odc.log.directory:./log}") String baseTaskLogDir) { @@ -251,6 +252,10 @@ public File getLogFile(Long userId, String taskId, TaskType type, OdcTaskLogLeve filePath = String.format(STRUCTURE_COMPARISON_LOG_PATH_PATTERN, logFilePrefix, userId, taskId, logLevel.name().toLowerCase()); break; + case APPLY_TABLE_PERMISSION: + filePath = String.format(APPLY_TABLE_LOG_PATH_PATTERN, logFilePrefix, userId, taskId, + logLevel.name().toLowerCase()); + break; default: throw new UnsupportedException(ErrorCodes.Unsupported, new Object[] {ResourceType.ODC_TASK}, "Unsupported task type: " + type); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/PreCheckTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/PreCheckTask.java index 1e0991dc6a..a1dd3b2f87 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/PreCheckTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/PreCheckTask.java @@ -23,10 +23,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Objects; -import java.util.Set; -import java.util.TreeMap; import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; @@ -39,11 +36,10 @@ import com.oceanbase.odc.core.flow.model.FlowTaskResult; import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.constant.TaskType; -import com.oceanbase.odc.core.sql.execute.model.SqlTuple; import com.oceanbase.odc.core.sql.split.OffsetString; import com.oceanbase.odc.core.sql.split.SqlStatementIterator; import com.oceanbase.odc.service.common.util.SqlUtils; -import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.odc.service.connection.database.model.UnauthorizedDBResource; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.flow.model.PreCheckTaskResult; import com.oceanbase.odc.service.flow.task.model.DatabaseChangeParameters; @@ -52,8 +48,6 @@ import com.oceanbase.odc.service.objectstorage.model.ObjectMetadata; import com.oceanbase.odc.service.objectstorage.util.ObjectStorageUtils; import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeParameters; -import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; -import com.oceanbase.odc.service.permission.database.model.UnauthorizedDatabase; import com.oceanbase.odc.service.regulation.ruleset.model.Rule; import com.oceanbase.odc.service.regulation.ruleset.model.Rule.RuleViolation; import com.oceanbase.odc.service.regulation.ruleset.model.RuleMetadata; @@ -62,7 +56,6 @@ import com.oceanbase.odc.service.schedule.flowtask.AlterScheduleParameters; import com.oceanbase.odc.service.schedule.model.JobType; import com.oceanbase.odc.service.session.factory.OBConsoleDataSourceFactory; -import com.oceanbase.odc.service.session.util.SchemaExtractor; import com.oceanbase.odc.service.sqlcheck.DefaultSqlChecker; import com.oceanbase.odc.service.sqlcheck.SqlCheckContext; import com.oceanbase.odc.service.sqlcheck.SqlCheckRule; @@ -72,9 +65,7 @@ import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.executor.task.BaseTask; -import com.oceanbase.odc.service.task.runtime.PreCheckTaskParameters.AuthorizedDatabase; import com.oceanbase.odc.service.task.util.JobUtils; -import com.oceanbase.tools.dbbrowser.parser.constant.SqlType; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -113,15 +104,15 @@ protected boolean doStart(JobContext context) throws Exception { try { List sqls = new ArrayList<>(); this.overLimit = getSqlContentUntilOverLimit(sqls, this.parameters.getMaxReadContentBytes()); - List unauthorizedDatabases = new ArrayList<>(); + List unauthorizedDBResources = new ArrayList<>(); List violations = new ArrayList<>(); if (CollectionUtils.isNotEmpty(sqls)) { violations.addAll(checkViolations(sqls)); log.info("SQL check successfully, taskId={}", taskId); - unauthorizedDatabases.addAll(filterUnAuthorizedDatabase(sqls)); + unauthorizedDBResources.addAll(filterUnAuthorizedDatabase(sqls)); log.info("Database permission check successfully, taskId={}", taskId); } - this.permissionCheckResult = new DatabasePermissionCheckResult(unauthorizedDatabases); + this.permissionCheckResult = new DatabasePermissionCheckResult(unauthorizedDBResources); this.sqlCheckResult = SqlCheckTaskResult.success(violations); this.success = true; log.info("Pre-check task end up running, task id: {}", taskId); @@ -303,71 +294,11 @@ private void fullFillRiskLevel(List rules, @NonNull List v }); } - private List filterUnAuthorizedDatabase(List sqls) { - // Get needed permission types for accessing the schemas - Map> neededSchemaName2PermissionTypes = new HashMap<>(); - Map> schemaName2SqlTypes = SchemaExtractor.listSchemaName2SqlTypes( - sqls.stream().map(e -> SqlTuple.newTuple(e.getStr())).collect(Collectors.toList()), - this.parameters.getDefaultSchema(), this.parameters.getConnectionConfig().getDialectType()); - for (Entry> entry : schemaName2SqlTypes.entrySet()) { - Set sqlTypes = entry.getValue(); - if (CollectionUtils.isNotEmpty(sqlTypes)) { - Set permissionTypes = sqlTypes.stream().map(DatabasePermissionType::from) - .filter(Objects::nonNull).collect(Collectors.toSet()); - permissionTypes.addAll(DatabasePermissionType.from(this.parameters.getTaskType())); - if (CollectionUtils.isNotEmpty(permissionTypes)) { - neededSchemaName2PermissionTypes.put(entry.getKey(), permissionTypes); - } - } - } - if (neededSchemaName2PermissionTypes.isEmpty()) { - return Collections.emptyList(); - } - // Get authorized permission types for all schemas in current datasource - Map> authorizedSchema2PermissionTypes = new HashMap<>(); - Map schemaName2Database = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - List authorizedDatabases = this.parameters.getAuthorizedDatabase(); - if (CollectionUtils.isNotEmpty(authorizedDatabases)) { - for (AuthorizedDatabase authorizedDatabase : authorizedDatabases) { - authorizedSchema2PermissionTypes.put(authorizedDatabase.getName(), - authorizedDatabase.getPermissionTypes()); - Database database = new Database(); - database.setId(authorizedDatabase.getId()); - database.setName(authorizedDatabase.getName()); - database.setDataSource(this.parameters.getConnectionConfig()); - schemaName2Database.put(authorizedDatabase.getName(), database); - } - } - List ret = new ArrayList<>(); - for (Map.Entry> entry : neededSchemaName2PermissionTypes.entrySet()) { - String schemaName = entry.getKey(); - Set needs = entry.getValue(); - if (CollectionUtils.isEmpty(needs)) { - continue; - } - if (schemaName2Database.containsKey(schemaName)) { - Database database = schemaName2Database.get(schemaName); - Set authorized = authorizedSchema2PermissionTypes.get(schemaName); - if (CollectionUtils.isEmpty(authorized)) { - ret.add(UnauthorizedDatabase.from(database, needs, false)); - } else { - Set unauthorized = - needs.stream().filter(p -> !authorized.contains(p)).collect(Collectors.toSet()); - if (CollectionUtils.isNotEmpty(unauthorized)) { - ret.add(UnauthorizedDatabase.from(database, unauthorized, false)); - } - } - } else { - Database unknownDatabase = new Database(); - unknownDatabase.setName(schemaName); - unknownDatabase.setDataSource(this.parameters.getConnectionConfig()); - ret.add(UnauthorizedDatabase.from(unknownDatabase, needs, false)); - } - } - return ret; + // TODO: Implement this method for database and table permission check + private List filterUnAuthorizedDatabase(List sqls) { + return Collections.emptyList(); } - private void tryCloseInputStream() { if (Objects.nonNull(this.uploadFileInputStream)) { try { diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/session/DBSchemaExtractorTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/session/DBSchemaExtractorTest.java new file mode 100644 index 0000000000..ca2b1a31c9 --- /dev/null +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/session/DBSchemaExtractorTest.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.session; + +import static org.junit.Assert.*; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.junit.Assert; +import org.junit.Test; + +import com.oceanbase.odc.common.util.YamlUtils; +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.core.sql.execute.model.SqlTuple; +import com.oceanbase.odc.service.session.util.DBSchemaExtractor; +import com.oceanbase.odc.service.session.util.DBSchemaExtractor.DBSchemaIdentity; +import com.oceanbase.tools.dbbrowser.parser.constant.SqlType; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author gaoda.xy + * @date 2024/5/7 15:08 + */ +public class DBSchemaExtractorTest { + + private static final String TEST_RESOURCE_FILE_PATH = "session/test_db_schema_extractor.yaml"; + + @Test + public void test() { + List testCases = + YamlUtils.fromYamlList(TEST_RESOURCE_FILE_PATH, DBSchemaExtractorTestCase.class); + for (DBSchemaExtractorTestCase testCase : testCases) { + Map> actual = DBSchemaExtractor.listDBSchemasWithSqlTypes( + testCase.getSqls().stream().map(SqlTuple::newTuple).collect(Collectors.toList()), + testCase.getDialectType(), testCase.getDefaultSchema()); + Assert.assertEquals(testCase.getExpected().size(), actual.size()); + for (DBSchemaIdentity expected : testCase.getExpected()) { + Assert.assertTrue(actual.containsKey(expected)); + } + System.out.println("Test case: " + testCase.getId() + " passed."); + } + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + static class DBSchemaExtractorTestCase { + private Integer id; + private DialectType dialectType; + private String defaultSchema; + private List sqls; + private Set expected; + } + +} diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/session/SchemaExtractorTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/session/SchemaExtractorTest.java deleted file mode 100644 index f1711bdc88..0000000000 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/session/SchemaExtractorTest.java +++ /dev/null @@ -1,252 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oceanbase.odc.service.session; - -import java.util.Arrays; -import java.util.Collections; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - -import org.junit.Assert; -import org.junit.Test; - -import com.oceanbase.odc.core.shared.constant.DialectType; -import com.oceanbase.odc.core.sql.execute.model.SqlTuple; -import com.oceanbase.odc.service.session.util.SchemaExtractor; -import com.oceanbase.tools.dbbrowser.parser.constant.SqlType; - -/** - * @Author: Lebie - * @Date: 2023/12/8 15:56 - * @Description: [] - */ -public class SchemaExtractorTest { - - @Test - public void testMySQL_listSchemaName2SqlTypes() { - String sql1 = "select * from db1.table1;"; - String sql2 = "truncate table db1.table1;"; - Map> actual = SchemaExtractor.listSchemaName2SqlTypes( - Arrays.asList(SqlTuple.newTuple(sql1), SqlTuple.newTuple(sql2)), "default", DialectType.OB_MYSQL); - Assert.assertTrue(actual.keySet().size() == 1 && actual.containsKey("db1")); - Assert.assertTrue(actual.get("db1").size() == 2 && actual.get("db1").contains(SqlType.SELECT) - && actual.get("db1").contains(SqlType.TRUNCATE)); - } - - @Test - public void testMySQL_ListSchemaNames() { - String sql = "select * from db1.table1"; - Set actual = SchemaExtractor.listSchemaNames(Arrays.asList(sql), DialectType.OB_MYSQL, "default"); - Set expect = Collections.singleton("db1"); - Assert.assertEquals(expect, actual); - } - - @Test - public void testMySQL_ListSchemaNames_FunctionDdl() { - String sql = "create function `schema_name`.`func` (\n" - + "\t`str1` varchar ( 45 ),\n" - + "\t`str2` varchar ( 45 )) returns varchar ( 128 ) begin\n" - + "return ( select concat( str1, str2 ) from dual );\n" - + "end;"; - Set actual = SchemaExtractor.listSchemaNames(Arrays.asList(sql), DialectType.OB_MYSQL, "default"); - Set expect = Collections.singleton("schema_name"); - Assert.assertEquals(expect, actual); - } - - @Test - public void testMySQL_ListSchemaNames_FunctionDdl_UseDefaultSchema() { - String sql = "CREATE FUNCTION `func` (\n" - + "\t`str1` VARCHAR ( 45 ),\n" - + "\t`str2` VARCHAR ( 45 )) RETURNS VARCHAR ( 128 ) BEGIN\n" - + "RETURN ( SELECT concat( str1, str2 ) FROM DUAL );\n" - + "END;"; - Set actual = SchemaExtractor.listSchemaNames(Arrays.asList(sql), DialectType.OB_MYSQL, "default"); - Set expect = Collections.singleton("default"); - Assert.assertEquals(expect, actual); - } - - @Test - public void testMySQL_ListSchemaNames_SelectFunction() { - String sql = "select user_schema.user_function(4) from dual;"; - Set actual = SchemaExtractor.listSchemaNames(Arrays.asList(sql), DialectType.OB_MYSQL, "default"); - Set expect = Collections.singleton("user_schema"); - Assert.assertEquals(expect, actual); - } - - @Test - public void testMySQL_ListSchemaNames_SelectFunction_UseDefaultSchema() { - String sql = "select user_function(4) from dual;"; - Set actual = SchemaExtractor.listSchemaNames(Arrays.asList(sql), DialectType.OB_MYSQL, "default"); - Set expect = Collections.singleton("default"); - Assert.assertEquals(expect, actual); - } - - @Test - public void testMySQL_ListSchemaNames_CallProcedure() { - String sql = "call `schema_name`.`user_procedure`();"; - Set actual = SchemaExtractor.listSchemaNames(Arrays.asList(sql), DialectType.OB_MYSQL, "default"); - Set expect = Collections.singleton("schema_name"); - Assert.assertEquals(expect, actual); - } - - @Test - public void testMySQL_ListSchemaNames_CallProcedure_UseDefaultSchema() { - String sql = "call user_procedure();"; - Set actual = SchemaExtractor.listSchemaNames(Arrays.asList(sql), DialectType.OB_MYSQL, "default"); - Set expect = Collections.singleton("default"); - Assert.assertEquals(expect, actual); - } - - @Test - public void testMySQL_ListSchemaNames_SwitchSchema() { - String sql = "use `schema_name`;"; - Set actual = SchemaExtractor.listSchemaNames(Arrays.asList(sql), DialectType.OB_MYSQL, "default"); - Set expect = Collections.singleton("schema_name"); - Assert.assertEquals(expect, actual); - } - - @Test - public void testMySQL_ExtractSwitchedSchemaName() { - String sql = "use `schema_name`;"; - Optional actual = - SchemaExtractor.extractSwitchedSchemaName(Arrays.asList(SqlTuple.newTuple(sql)), DialectType.OB_MYSQL); - Assert.assertEquals("schema_name", actual.get()); - } - - @Test - public void testMySQL_ExtractSwitchedSchemaName_NotExist() { - String sql = "select 1 from dual;"; - Optional actual = - SchemaExtractor.extractSwitchedSchemaName(Arrays.asList(SqlTuple.newTuple(sql)), DialectType.OB_MYSQL); - Assert.assertFalse(actual.isPresent()); - } - - - @Test - public void testOracle_listSchemaName2SqlTypes() { - String sql = "SELECT * FROM DB1.TABLE1"; - Map> actual = SchemaExtractor - .listSchemaName2SqlTypes(Arrays.asList(SqlTuple.newTuple(sql)), "default", DialectType.OB_ORACLE); - Assert.assertTrue(actual.keySet().size() == 1 && actual.containsKey("DB1")); - Assert.assertTrue(actual.get("DB1").size() == 1 && actual.get("DB1").contains(SqlType.SELECT)); - } - - @Test - public void testOracle_listSchemaName2SqlTypes_WithDBLink() { - String sql = "SELECT * FROM DB1.TABLE1@FAKE_DBLINK;"; - Map> actual = SchemaExtractor - .listSchemaName2SqlTypes(Arrays.asList(SqlTuple.newTuple(sql)), "default", DialectType.OB_ORACLE); - Assert.assertTrue(actual.isEmpty()); - } - - @Test - public void testOracle_ListSchemaNames() { - String sql = "SELECT * FROM DB1.TABLE1"; - Set actual = SchemaExtractor.listSchemaNames(Arrays.asList(sql), DialectType.OB_ORACLE, "default"); - Set expect = Collections.singleton("DB1"); - Assert.assertEquals(expect, actual); - } - - @Test - public void testOracle_ListSchemaNames_WithDBLink() { - String sql = "SELECT * FROM DB1.TABLE1@FAKE_DBLINK;"; - Set actual = SchemaExtractor.listSchemaNames(Arrays.asList(sql), DialectType.OB_ORACLE, "default"); - Assert.assertTrue(actual.isEmpty()); - } - - @Test - public void testOracle_ListSchemaNames_FunctionDdl() { - String sql = "CREATE OR REPLACE FUNCTION SCHEMA_NAME.INCREMENT_BY_ONE (INPUT_NUMBER IN NUMBER)\n" - + "RETURN NUMBER IS\n" - + "BEGIN\n" - + " RETURN INPUT_NUMBER + 1;\n" - + "END INCREMENT_BY_ONE;"; - Set actual = SchemaExtractor.listSchemaNames(Arrays.asList(sql), DialectType.OB_ORACLE, "DEFAULT"); - Set expect = Collections.singleton("SCHEMA_NAME"); - Assert.assertEquals(expect, actual); - } - - @Test - public void testOracle_ListSchemaNames_FunctionDdl_UseDefaultSchema() { - String sql = "CREATE OR REPLACE FUNCTION INCREMENT_BY_ONE (INPUT_NUMBER IN NUMBER)\n" - + "RETURN NUMBER IS\n" - + "BEGIN\n" - + " RETURN INPUT_NUMBER + 1;\n" - + "END INCREMENT_BY_ONE;"; - Set actual = SchemaExtractor.listSchemaNames(Arrays.asList(sql), DialectType.OB_ORACLE, "DEFAULT"); - Set expect = Collections.singleton("DEFAULT"); - Assert.assertEquals(expect, actual); - } - - @Test - public void testOracle_ListSchemaNames_SelectFunction() { - String sql = "SELECT USER_SCHEMA.USER_FUNCTION(4) FROM DUAL;"; - Set actual = SchemaExtractor.listSchemaNames(Arrays.asList(sql), DialectType.OB_ORACLE, "DEFAULT"); - Set expect = Collections.singleton("USER_SCHEMA"); - Assert.assertEquals(expect, actual); - } - - @Test - public void testOracle_ListSchemaNames_SelectFunction_UseDefaultSchema() { - String sql = "SELECT USER_FUNCTION(4) FROM DUAL;"; - Set actual = SchemaExtractor.listSchemaNames(Arrays.asList(sql), DialectType.OB_ORACLE, "DEFAULT"); - Set expect = Collections.singleton("DEFAULT"); - Assert.assertEquals(expect, actual); - } - - @Test - public void testOracle_ListSchemaNames_CallProcedure() { - String sql = "CALL USER_SCHEMA_NAME.USER_PROCEDURE()"; - Set actual = SchemaExtractor.listSchemaNames(Arrays.asList(sql), DialectType.OB_ORACLE, "DEFAULT"); - Set expect = Collections.singleton("USER_SCHEMA_NAME"); - Assert.assertEquals(expect, actual); - } - - @Test - public void testOracle_ListSchemaNames_CallProcedure_UseDefaultSchema() { - String sql = "CALL USER_PROCEDURE()"; - Set actual = SchemaExtractor.listSchemaNames(Arrays.asList(sql), DialectType.OB_ORACLE, "DEFAULT"); - Set expect = Collections.singleton("DEFAULT"); - Assert.assertEquals(expect, actual); - } - - @Test - public void testOracle_ListSchemaNames_SwitchSchema() { - String sql = "ALTER SESSION SET CURRENT_SCHEMA = OTHER_SCHEMA"; - Set actual = SchemaExtractor.listSchemaNames(Arrays.asList(sql), DialectType.OB_ORACLE, "DEFAULT"); - Set expect = Collections.singleton("OTHER_SCHEMA"); - Assert.assertEquals(expect, actual); - } - - @Test - public void testOracle_ExtractSwitchedSchemaName() { - String sql = "ALTER SESSION SET CURRENT_SCHEMA = OTHER_SCHEMA"; - Optional actual = - SchemaExtractor.extractSwitchedSchemaName(Arrays.asList(SqlTuple.newTuple(sql)), DialectType.OB_ORACLE); - Assert.assertEquals("OTHER_SCHEMA", actual.get()); - } - - @Test - public void testOracle_ExtractSwitchedSchemaName_NotExist() { - String sql = "SELECT 1 FROM DUAL;"; - Optional actual = - SchemaExtractor.extractSwitchedSchemaName(Arrays.asList(SqlTuple.newTuple(sql)), DialectType.OB_ORACLE); - Assert.assertFalse(actual.isPresent()); - } - -} diff --git a/server/odc-service/src/test/resources/session/test_db_schema_extractor.yaml b/server/odc-service/src/test/resources/session/test_db_schema_extractor.yaml new file mode 100644 index 0000000000..b76d73a976 --- /dev/null +++ b/server/odc-service/src/test/resources/session/test_db_schema_extractor.yaml @@ -0,0 +1,194 @@ +## Test cases for MySQL +# access database and table +- id: 1 + dialect_type: OB_MYSQL + default_schema: default_schema + sqls: + - select 1 from dual; + expected: [ ] +- id: 2 + dialect_type: OB_MYSQL + default_schema: default_schema + sqls: + - select * from db1.table1; + expected: + - schema: db1 + table: table1 +- id: 3 + dialect_type: OB_MYSQL + default_schema: default_schema + sqls: + - select * from table1; + expected: + - schema: default_schema + table: table1 +- id: 4 + dialect_type: OB_MYSQL + default_schema: default_schema + sqls: + - select * from table1; + - truncate table db1.table1; + expected: + - schema: default_schema + table: table1 + - schema: db1 + table: table1 +- id: 5 + dialect_type: OB_MYSQL + default_schema: default_schema + sqls: + - use db1; + expected: + - schema: db1 + table: ~ +- id: 6 + dialect_type: OB_MYSQL + default_schema: default_schema + sqls: + - create table db1.table1 (id int); + expected: + - schema: db1 + table: table1 +- id: 7 + dialect_type: OB_MYSQL + default_schema: default_schema + sqls: + - alter table db1.table1 add column name varchar(10); + expected: + - schema: db1 + table: table1 + +# access function or procedure +- id: 8 + dialect_type: OB_MYSQL + default_schema: default_schema + sqls: + - select func(); + expected: + - schema: default_schema + table: ~ +- id: 9 + dialect_type: OB_MYSQL + default_schema: default_schema + sqls: + - CREATE FUNCTION `func` ( + `str1` VARCHAR ( 45 ), + `str2` VARCHAR ( 45 )) RETURNS VARCHAR ( 128 ) BEGIN + RETURN ( SELECT concat( str1, str2 ) FROM DUAL ); + END; + expected: + - schema: default_schema + table: ~ +- id: 10 + dialect_type: OB_MYSQL + default_schema: default_schema + sqls: + - call `schema_name`.`user_procedure`(); + expected: + - schema: schema_name + table: ~ +# access DBLink +- id: 11 + dialect_type: OB_MYSQL + default_schema: default_schema + sqls: + - "select * from db1.table1@fake_dblink;" + expected: [ ] + + +## Test cases for Oracle +# access database and table +- id: 12 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - SELECT 1 FROM DUAL; + expected: [ ] +- id: 13 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - SELECT * FROM DB1.TABLE1; + expected: + - schema: DB1 + table: TABLE1 +- id: 14 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - SELECT * FROM TABLE1; + expected: + - schema: DEFAULT_SCHEMA + table: TABLE1 +- id: 15 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - SELECT * FROM TABLE1; + - TRUNCATE TABLE DB1.TABLE1; + expected: + - schema: DEFAULT_SCHEMA + table: TABLE1 + - schema: DB1 + table: TABLE1 +- id: 16 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - ALTER SESSION SET CURRENT_SCHEMA = OTHER_SCHEMA; + expected: + - schema: OTHER_SCHEMA + table: ~ +- id: 17 + dialect_type: OB_ORACLE + default_schema: default_schema + sqls: + - CREATE TABLE DB1.TABLE1 (ID INT); + expected: + - schema: DB1 + table: TABLE1 +- id: 18 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - ALTER TABLE "DB1"."TABLE1" ADD "NAME" VARCHAR(120) DEFAULT NULL; + expected: + - schema: DB1 + table: TABLE1 + +# access function or procedure +- id: 19 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - SELECT FUNC() FROM DUAL; + expected: + - schema: DEFAULT_SCHEMA + table: ~ +- id: 20 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - CREATE OR REPLACE FUNCTION INCREMENT_BY_ONE (INPUT_NUMBER IN NUMBER) + RETURN NUMBER IS + BEGIN + RETURN INPUT_NUMBER + 1; + END INCREMENT_BY_ONE; + expected: + - schema: DEFAULT_SCHEMA + table: ~ +- id: 21 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - CALL SCHEMA_NAME.USER_PROCEDURE(); + expected: + - schema: SCHEMA_NAME + table: ~ +# access DBLink +- id: 22 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "SELECT * FROM DB1.TABLE1@FAKE_DBLINK;" + expected: [ ] \ No newline at end of file From b02a56ea903ab4bc18200494538b04d79852e8d1 Mon Sep 17 00:00:00 2001 From: yizhou Date: Mon, 20 May 2024 19:28:51 +0800 Subject: [PATCH 03/64] build: fix build_jar.sh remove old version jar not works (#2435) * build: fix build_jar.sh remove old version jar not works * fix odc-server jar copy --- script/build_jar.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/build_jar.sh b/script/build_jar.sh index 7a195539f9..0cd01878cf 100755 --- a/script/build_jar.sh +++ b/script/build_jar.sh @@ -15,17 +15,17 @@ fi echo "maven build jar success, copy executable jar to ${ODC_DIR}/lib for use script/start-odc.sh locally." mkdir -p "${ODC_DIR}/"{lib,conf,plugins,starters} -[[ -f "${ODC_DIR}"/lib/*.jar ]] && rm -fv "${ODC_DIR}"/lib/*.jar +find "${ODC_DIR}/lib" -type f -name '*.jar' -exec rm -fv {} + cp -fv "${ODC_DIR}"/server/odc-server/target/odc-*-executable.jar "${ODC_DIR}"/lib/ cp -fv "${ODC_DIR}"/server/odc-server/target/classes/log4j2.xml "${ODC_DIR}"/conf/ cp -fv "${ODC_DIR}"/server/odc-server/target/classes/log4j2-task.xml "${ODC_DIR}"/conf/ echo "copy plugin jars to ${ODC_DIR}/plugins ." -[[ -f "${ODC_DIR}"/plugins/*.jar ]] && rm -fv "${ODC_DIR}"/plugins/*.jar +find "${ODC_DIR}/plugins" -type f -name '*.jar' -exec rm -fv {} + cp -fv "${ODC_DIR}"/distribution/plugins/*.jar "${ODC_DIR}"/plugins/ echo "copy starter jars to ${ODC_DIR}/starters ." -[[ -f "${ODC_DIR}"/starters/*.jar ]] && rm -fv "${ODC_DIR}"/starters/*.jar +find "${ODC_DIR}/starters" -type f -name '*.jar' -exec rm -fv {} + cp -fv "${ODC_DIR}"/distribution/starters/*.jar "${ODC_DIR}"/starters/ exit $? From 0a206bfa29b7852dc0c80fbbf19284dc71941763 Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Mon, 20 May 2024 19:49:54 +0800 Subject: [PATCH 04/64] fix(deserialization): failed to deserialize the page object (#2434) --- .../oceanbase/odc/service/common/response/PaginatedData.java | 2 ++ .../odc/service/common/response/PaginatedResponse.java | 3 +++ 2 files changed, 5 insertions(+) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/common/response/PaginatedData.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/common/response/PaginatedData.java index beb5381b99..9a51f01a0a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/common/response/PaginatedData.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/common/response/PaginatedData.java @@ -21,6 +21,7 @@ import com.oceanbase.odc.service.common.model.Stats; import lombok.Data; +import lombok.NoArgsConstructor; /** * 分页数据，包含 列表 和 分页信息
@@ -29,6 +30,7 @@ * @param */ @Data +@NoArgsConstructor public class PaginatedData { private CustomPage page; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/common/response/PaginatedResponse.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/common/response/PaginatedResponse.java index 04ed577bf6..4d5f2c34ed 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/common/response/PaginatedResponse.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/common/response/PaginatedResponse.java @@ -15,6 +15,8 @@ */ package com.oceanbase.odc.service.common.response; +import lombok.NoArgsConstructor; + /** * 分页 response * @@ -26,6 +28,7 @@ * @param * */ +@NoArgsConstructor public class PaginatedResponse extends SuccessResponse> { public PaginatedResponse(PaginatedData data) { From 6957528416f079a9e517a8163757abb4e3f0bc74 Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Wed, 22 May 2024 19:58:02 +0800 Subject: [PATCH 05/64] fix(metadb): change systemConfigDao to systemConfigRepository. (#2467) * change configSystemRepository * formatter --- ...t.java => SystemConfigRepositoryTest.java} | 43 ++++++++++--------- .../odc/metadb/config/ConfigEntity.java | 14 ++++++ .../odc/metadb/config/SystemConfigEntity.java | 5 +++ ...igDAO.java => SystemConfigRepository.java} | 31 +++++++------ .../service/config/SystemConfigService.java | 12 +++--- .../systemconfig/DbConfigChangeMatcher.java | 6 +-- 6 files changed, 65 insertions(+), 46 deletions(-) rename server/integration-test/src/test/java/com/oceanbase/odc/metadb/config/{SystemConfigDAOTest.java => SystemConfigRepositoryTest.java} (57%) rename server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/{SystemConfigDAO.java => SystemConfigRepository.java} (67%) diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/config/SystemConfigDAOTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/config/SystemConfigRepositoryTest.java similarity index 57% rename from server/integration-test/src/test/java/com/oceanbase/odc/metadb/config/SystemConfigDAOTest.java rename to server/integration-test/src/test/java/com/oceanbase/odc/metadb/config/SystemConfigRepositoryTest.java index 420296cde2..633f8f80f6 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/config/SystemConfigDAOTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/config/SystemConfigRepositoryTest.java @@ -21,7 +21,6 @@ import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.test.jdbc.JdbcTestUtils; import com.oceanbase.odc.ServiceTestEnv; @@ -30,69 +29,71 @@ * @author liuyizhuo.lyz * @date 2024/3/21 */ -public class SystemConfigDAOTest extends ServiceTestEnv { +public class SystemConfigRepositoryTest extends ServiceTestEnv { @Autowired - private SystemConfigDAO systemConfigDAO; - @Autowired - private JdbcTemplate jdbcTemplate; + private SystemConfigRepository systemConfigRepository; @Before public void setUp() { - jdbcTemplate.batchUpdate("delete from config_system_configuration"); + systemConfigRepository.getJdbcTemplate().batchUpdate("delete from config_system_configuration"); } @Test public void test_Insert_NotExists() { - systemConfigDAO.insert(getConfigEntity()); - Assert.assertEquals(1, JdbcTestUtils.countRowsInTable(jdbcTemplate, "config_system_configuration")); + systemConfigRepository.getJdbcTemplate().batchUpdate("delete from config_system_configuration"); + systemConfigRepository.insert(getConfigEntity()); + Assert.assertEquals(1, JdbcTestUtils.countRowsInTable(systemConfigRepository.getJdbcTemplate(), + "config_system_configuration")); } @Test public void test_Insert_Exists() { SystemConfigEntity config = getConfigEntity(); - systemConfigDAO.insert(config); + systemConfigRepository.insert(config); config.setValue("value1"); - systemConfigDAO.insert(config); + systemConfigRepository.insert(config); - SystemConfigEntity entity = systemConfigDAO.queryByKey("dummy.key"); + SystemConfigEntity entity = systemConfigRepository.queryByKey("dummy.key"); Assert.assertEquals("value", entity.getValue()); } @Test public void test_Upsert_NotExists() { - systemConfigDAO.upsert(getConfigEntity()); - Assert.assertEquals(1, JdbcTestUtils.countRowsInTable(jdbcTemplate, "config_system_configuration")); + systemConfigRepository.getJdbcTemplate().batchUpdate("delete from config_system_configuration"); + systemConfigRepository.upsert(getConfigEntity()); + Assert.assertEquals(1, JdbcTestUtils.countRowsInTable(systemConfigRepository.getJdbcTemplate(), + "config_system_configuration")); } @Test public void test_Upsert_Exists() { SystemConfigEntity config = getConfigEntity(); - systemConfigDAO.upsert(config); + systemConfigRepository.upsert(config); config.setValue("value1"); - systemConfigDAO.upsert(config); + systemConfigRepository.upsert(config); - SystemConfigEntity entity = systemConfigDAO.queryByKey("dummy.key"); + SystemConfigEntity entity = systemConfigRepository.queryByKey("dummy.key"); Assert.assertEquals("value1", entity.getValue()); } @Test public void test_QueryByKeyPrefix() { SystemConfigEntity entity = getConfigEntity(); - systemConfigDAO.upsert(entity); + systemConfigRepository.upsert(entity); entity.setKey("dummy.key1"); - systemConfigDAO.upsert(entity); + systemConfigRepository.upsert(entity); - List entities = systemConfigDAO.queryByKeyPrefix("dummy"); + List entities = systemConfigRepository.queryByKeyPrefix("dummy"); Assert.assertEquals(2, entities.size()); } @Test public void test_QueryByKey() { - systemConfigDAO.upsert(getConfigEntity()); - SystemConfigEntity entity = systemConfigDAO.queryByKey("dummy.key"); + systemConfigRepository.upsert(getConfigEntity()); + SystemConfigEntity entity = systemConfigRepository.queryByKey("dummy.key"); Assert.assertEquals("value", entity.getValue()); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/ConfigEntity.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/ConfigEntity.java index aff5413229..fce4125322 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/ConfigEntity.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/ConfigEntity.java @@ -17,6 +17,11 @@ import java.util.Date; +import javax.persistence.Column; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.MappedSuperclass; import javax.validation.constraints.NotBlank; import lombok.Data; @@ -27,26 +32,35 @@ * @Description: [] */ @Data +@MappedSuperclass public class ConfigEntity { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; /** * configuration key */ @NotBlank + @Column(name = "`key`") private String key; /** * configuration value */ + @Column(name = "`value`") private String value; /** * Created time */ + @Column(name = "create_time") private Date createTime; /** * Latest update time */ + @Column(name = "update_time") private Date updateTime; /** diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigEntity.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigEntity.java index 86684b7f7e..0659166890 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigEntity.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigEntity.java @@ -15,6 +15,9 @@ */ package com.oceanbase.odc.metadb.config; +import javax.persistence.Entity; +import javax.persistence.Table; + import lombok.Data; import lombok.EqualsAndHashCode; @@ -25,6 +28,8 @@ */ @Data @EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = "config_system_configuration") public class SystemConfigEntity extends ConfigEntity { /** diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigDAO.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigRepository.java similarity index 67% rename from server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigDAO.java rename to server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigRepository.java index 7935b9ae9c..a27721a0af 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigDAO.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigRepository.java @@ -17,46 +17,45 @@ import java.util.List; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.BeanPropertyRowMapper; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.stereotype.Component; +import com.oceanbase.odc.config.jpa.OdcJpaRepository; import com.oceanbase.odc.core.shared.PreConditions; -@Component -public class SystemConfigDAO { +public interface SystemConfigRepository extends OdcJpaRepository { - @Autowired - private JdbcTemplate jdbcTemplate; - - public List queryByKeyPrefix(String keyPrefix) { + default List queryByKeyPrefix(String keyPrefix) { PreConditions.notNull(keyPrefix, "keyPrefix"); String sql = "SELECT `application`, `profile`, `key`, `value`, `create_time`, `update_time`, `description` " + "FROM `config_system_configuration` " + "WHERE `application`='odc' AND `profile`='default' AND `label`='master' AND `key` LIKE ?"; - return jdbcTemplate.query(sql, new BeanPropertyRowMapper<>(SystemConfigEntity.class), keyPrefix + "%"); + return getJdbcTemplate().query(sql, new BeanPropertyRowMapper<>(SystemConfigEntity.class), keyPrefix + "%"); } - public SystemConfigEntity queryByKey(String key) { + default SystemConfigEntity queryByKey(String key) { PreConditions.notEmpty(key, "key"); String sql = "SELECT `application`, `profile`, `key`, `value`, `create_time`, `update_time`, `description` " + "FROM `config_system_configuration` " + "WHERE `application`='odc' AND `profile`='default' AND `label`='master' AND `key` = ?"; - return jdbcTemplate.queryForObject(sql, new BeanPropertyRowMapper<>(SystemConfigEntity.class), key); + return getJdbcTemplate().queryForObject(sql, new BeanPropertyRowMapper<>(SystemConfigEntity.class), key); } - public int insert(SystemConfigEntity entity) { + default int insert(SystemConfigEntity entity) { PreConditions.notNull(entity, "systemConfigEntity"); String sql = "INSERT INTO `config_system_configuration`(`key`, `value`, `description`)" + " VALUES(?, ?, ?) ON DUPLICATE KEY UPDATE `id`=`id`"; - return jdbcTemplate.update(sql, entity.getKey(), entity.getValue(), entity.getDescription()); + return getJdbcTemplate().update(sql, entity.getKey(), entity.getValue(), entity.getDescription()); } - public int upsert(SystemConfigEntity entity) { + default int upsert(SystemConfigEntity entity) { PreConditions.notNull(entity, "systemConfigEntity"); String sql = "INSERT INTO `config_system_configuration`(`key`, `value`, `description`)" + " VALUES(?, ?, ?) ON DUPLICATE KEY UPDATE `value`=?"; - return jdbcTemplate.update(sql, entity.getKey(), entity.getValue(), entity.getDescription(), entity.getValue()); + return getJdbcTemplate().update(sql, entity.getKey(), entity.getValue(), entity.getDescription(), + entity.getValue()); } + + SystemConfigEntity findByKeyLike(String key); + + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/config/SystemConfigService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/config/SystemConfigService.java index 48ddd841ae..9e673984c5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/config/SystemConfigService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/config/SystemConfigService.java @@ -29,8 +29,8 @@ import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.core.authority.util.SkipAuthorize; -import com.oceanbase.odc.metadb.config.SystemConfigDAO; import com.oceanbase.odc.metadb.config.SystemConfigEntity; +import com.oceanbase.odc.metadb.config.SystemConfigRepository; import com.oceanbase.odc.service.config.model.Configuration; import com.oceanbase.odc.service.config.util.ConfigurationUtils; import com.oceanbase.odc.service.systemconfig.SystemConfigRefreshMatcher; @@ -49,7 +49,7 @@ public class SystemConfigService { private static final String SENSITIVE_MASK_VALUE = "******"; @Autowired - private SystemConfigDAO systemConfigDAO; + private SystemConfigRepository systemConfigRepository; @Autowired private ContextRefresher contextRefresher; @@ -111,7 +111,7 @@ private boolean isSensitive(String key) { @SkipAuthorize("odc internal usage") public List queryByKeyPrefix(String keyPrefix) { - List configEntities = systemConfigDAO.queryByKeyPrefix(keyPrefix); + List configEntities = systemConfigRepository.queryByKeyPrefix(keyPrefix); return ConfigurationUtils.fromEntity(configEntities).stream().peek(config -> { for (Consumer consumer : getConfigurationConsumer()) { consumer.accept(config); @@ -121,7 +121,7 @@ public List queryByKeyPrefix(String keyPrefix) { @SkipAuthorize("odc internal usage") public Configuration queryByKey(String key) { - SystemConfigEntity systemConfigEntity = systemConfigDAO.queryByKey(key); + SystemConfigEntity systemConfigEntity = systemConfigRepository.queryByKey(key); Configuration config = ConfigurationUtils.fromEntity(systemConfigEntity); for (Consumer consumer : getConfigurationConsumer()) { consumer.accept(config); @@ -142,7 +142,7 @@ public synchronized void refresh() { @SkipAuthorize("public readonly resource") @Transactional(rollbackFor = Exception.class) public void insert(@NotNull List entities) { - entities.forEach(entity -> systemConfigDAO.insert(entity)); + entities.forEach(entity -> systemConfigRepository.insert(entity)); } private boolean needRefresh() { @@ -158,7 +158,7 @@ private boolean needRefresh() { @SkipAuthorize("odc internal usage") @Transactional(rollbackFor = Exception.class) public void upsert(@NotNull List entities) { - entities.forEach(entity -> systemConfigDAO.upsert(entity)); + entities.forEach(entity -> systemConfigRepository.upsert(entity)); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/systemconfig/DbConfigChangeMatcher.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/systemconfig/DbConfigChangeMatcher.java index c9a170d80c..9838d2f161 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/systemconfig/DbConfigChangeMatcher.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/systemconfig/DbConfigChangeMatcher.java @@ -32,8 +32,8 @@ import com.google.common.collect.Sets.SetView; import com.oceanbase.odc.core.alarm.AlarmUtils; import com.oceanbase.odc.core.authority.util.SkipAuthorize; -import com.oceanbase.odc.metadb.config.SystemConfigDAO; import com.oceanbase.odc.metadb.config.SystemConfigEntity; +import com.oceanbase.odc.metadb.config.SystemConfigRepository; import com.oceanbase.odc.service.config.model.Configuration; import com.oceanbase.odc.service.config.util.ConfigurationUtils; @@ -45,7 +45,7 @@ public class DbConfigChangeMatcher extends SystemConfigRefreshMatcher implements InitializingBean { @Autowired - private SystemConfigDAO systemConfigDAO; + private SystemConfigRepository systemConfigRepository; private List lastVersion = new ArrayList<>(); @@ -58,7 +58,7 @@ public List listAll() { @SkipAuthorize("odc internal usage") public List queryByKeyPrefix(String keyPrefix) { - List configEntities = systemConfigDAO.queryByKeyPrefix(keyPrefix); + List configEntities = systemConfigRepository.queryByKeyPrefix(keyPrefix); return ConfigurationUtils.fromEntity(configEntities); } From 91b1583c200bf88690c8ac61e8162d4a5adf1b2d Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Fri, 24 May 2024 11:56:33 +0800 Subject: [PATCH 06/64] feat(config):add creator_id to config entity (#2485) * add creator_id and last_modifier_id to config entity * add creator_id and last_modifier_id to config entity * add creator_id and last_modifier_id to config entity --- .../oceanbase/odc/metadb/config/SystemConfigEntity.java | 7 +++++++ .../odc/metadb/config/SystemConfigRepository.java | 4 +++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigEntity.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigEntity.java index 0659166890..182d29d26c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigEntity.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigEntity.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.metadb.config; +import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Table; @@ -32,6 +33,12 @@ @Table(name = "config_system_configuration") public class SystemConfigEntity extends ConfigEntity { + @Column(name = "creator_id") + private Long creatorId; + + @Column(name = "last_modifier_id") + private Long lastModifierId; + /** * application name */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigRepository.java index a27721a0af..4efa15f901 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigRepository.java @@ -55,7 +55,9 @@ default int upsert(SystemConfigEntity entity) { entity.getValue()); } - SystemConfigEntity findByKeyLike(String key); + List findByKeyLike(String key); + + SystemConfigEntity findByKey(String key); } From 8b9da887d05c2a9b0bb3fe6ca3ecdccd4fb42cbd Mon Sep 17 00:00:00 2001 From: pynzzZ <39962741+MarkPotato777@users.noreply.github.com> Date: Fri, 24 May 2024 15:43:42 +0800 Subject: [PATCH 07/64] feat(logical-database): logical database metadata management (#2358) --- .../tools/dbbrowser/model/DBObjectType.java | 1 + .../connection/DatabaseServiceTest.java | 2 + .../LogicalDatabaseServiceTest.java | 163 ++++++++++++ .../LogicalTableFinderTest.java | 5 +- .../LogicalTableServiceTest.java | 3 +- .../core/shared/constant/ResourceType.java | 1 + .../i18n/BusinessMessages.properties | 1 + .../i18n/BusinessMessages_zh_CN.properties | 1 + .../i18n/BusinessMessages_zh_TW.properties | 1 + .../V_4_3_1_4__add_logical_database.sql | 31 +++ .../web/controller/v2/DataBaseController.java | 5 + .../v2/LogicalDatabaseController.java | 80 ++++++ .../odc/config/ScheduleConfiguration.java | 18 ++ .../odc/metadb/connection/DatabaseEntity.java | 21 +- .../odc/metadb/connection/DatabaseSpecs.java | 7 + .../DatabaseMappingEntity.java | 62 +++++ .../DatabaseMappingRepository.java | 46 ++++ .../logicaldatabase/TableMappingEntity.java | 74 ++++++ .../TableMappingRepository.java | 64 +++++ .../connection/database/DatabaseService.java | 1 + .../connection/database/model/Database.java | 4 + .../database/model/DatabaseSyncStatus.java | 2 +- .../database/model/DatabaseType.java | 21 ++ .../database/model/QueryDatabaseParams.java | 4 + .../LogicalDatabaseService.java | 236 ++++++++++++++++++ .../LogicalDatabaseSyncManager.java | 84 +++++++ .../LogicalTableCheckConsistencyTask.java | 100 ++++++++ .../LogicalTableExtractTask.java | 126 ++++++++++ .../logicaldatabase/LogicalTableService.java | 186 ++++++++++++++ .../{ => core}/LogicalTableFinder.java | 46 ++-- .../LogicalTableRecognitionUtils.java | 6 +- .../{ => core}/model/DataNode.java | 26 +- .../{ => core}/model/LogicalDatabase.java | 2 +- .../{ => core}/model/LogicalTable.java | 2 +- .../BadLogicalTableExpressionException.java} | 8 +- .../parser/BaseLogicalTableExpression.java | 4 +- .../parser/BaseRangeExpression.java | 4 +- .../parser/ConsecutiveSliceRange.java | 4 +- .../DefaultLogicalTableExpressionParser.java | 2 +- .../{ => core}/parser/EnumRange.java | 6 +- .../parser/LogicalTableExpression.java | 6 +- .../LogicalTableExpressionParseUtils.java | 11 +- .../parser/LogicalTableExpressionVisitor.java | 2 +- .../parser/LogicalTableExpressions.java | 4 +- .../{ => core}/parser/SchemaExpression.java | 4 +- .../{ => core}/parser/SteppedRange.java | 4 +- .../{ => core}/parser/TableExpression.java | 2 +- .../model/CreateLogicalDatabaseReq.java | 43 ++++ .../model/DetailLogicalDatabaseResp.java | 52 ++++ .../model/DetailLogicalTableResp.java | 45 ++++ .../parser/LogicalTableService.java | 56 ----- .../LogicalTableRecognitionUtilsTest.java | 6 +- 52 files changed, 1570 insertions(+), 125 deletions(-) create mode 100644 server/integration-test/src/test/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalDatabaseServiceTest.java create mode 100644 server/odc-migrate/src/main/resources/migrate/common/V_4_3_1_4__add_logical_database.sql create mode 100644 server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/LogicalDatabaseController.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/logicaldatabase/DatabaseMappingEntity.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/logicaldatabase/DatabaseMappingRepository.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/logicaldatabase/TableMappingEntity.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/logicaldatabase/TableMappingRepository.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/DatabaseType.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalDatabaseService.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalDatabaseSyncManager.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableCheckConsistencyTask.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableExtractTask.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableService.java rename server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/{ => core}/LogicalTableFinder.java (82%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/{ => core}/LogicalTableRecognitionUtils.java (99%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/{ => core}/model/DataNode.java (87%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/{ => core}/model/LogicalDatabase.java (93%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/{ => core}/model/LogicalTable.java (97%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/{parser/BadExpressionException.java => core/parser/BadLogicalTableExpressionException.java} (87%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/{ => core}/parser/BaseLogicalTableExpression.java (85%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/{ => core}/parser/BaseRangeExpression.java (84%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/{ => core}/parser/ConsecutiveSliceRange.java (88%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/{ => core}/parser/DefaultLogicalTableExpressionParser.java (96%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/{ => core}/parser/EnumRange.java (84%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/{ => core}/parser/LogicalTableExpression.java (93%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/{ => core}/parser/LogicalTableExpressionParseUtils.java (83%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/{ => core}/parser/LogicalTableExpressionVisitor.java (99%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/{ => core}/parser/LogicalTableExpressions.java (89%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/{ => core}/parser/SchemaExpression.java (94%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/{ => core}/parser/SteppedRange.java (89%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/{ => core}/parser/TableExpression.java (93%) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/CreateLogicalDatabaseReq.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/DetailLogicalDatabaseResp.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/DetailLogicalTableResp.java delete mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/LogicalTableService.java rename server/odc-service/src/test/java/com/oceanbase/odc/service/connection/logicaldatabase/{ => core}/LogicalTableRecognitionUtilsTest.java (94%) diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBObjectType.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBObjectType.java index 8de2507c1b..7b2de36182 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBObjectType.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBObjectType.java @@ -24,6 +24,7 @@ public enum DBObjectType { */ SCHEMA("SCHEMA"), TABLE("TABLE"), + LOGICAL_TABLE("LOGICAL TABLE"), COLUMN("COLUMN"), INDEX("INDEX"), CONSTRAINT("CONSTRAINT"), diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/connection/DatabaseServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/connection/DatabaseServiceTest.java index b4282ec290..9f6a4d2c0d 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/connection/DatabaseServiceTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/connection/DatabaseServiceTest.java @@ -42,6 +42,7 @@ import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.connection.database.model.DatabaseSyncStatus; +import com.oceanbase.odc.service.connection.database.model.DatabaseType; import com.oceanbase.odc.service.connection.database.model.QueryDatabaseParams; import com.oceanbase.odc.service.connection.database.model.TransferDatabasesReq; import com.oceanbase.odc.service.connection.model.ConnectionConfig; @@ -170,6 +171,7 @@ public void testTransfer_Success() { private DatabaseEntity getEntity() { DatabaseEntity entity = new DatabaseEntity(); + entity.setType(DatabaseType.PHYSICAL); entity.setName("fake_db"); entity.setDatabaseId("fake_id"); entity.setExisted(true); diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalDatabaseServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalDatabaseServiceTest.java new file mode 100644 index 0000000000..88d5a5d460 --- /dev/null +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalDatabaseServiceTest.java @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection.logicaldatabase; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; + +import com.oceanbase.odc.ServiceTestEnv; +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.metadb.connection.ConnectionConfigRepository; +import com.oceanbase.odc.metadb.connection.ConnectionEntity; +import com.oceanbase.odc.metadb.connection.DatabaseEntity; +import com.oceanbase.odc.metadb.connection.DatabaseRepository; +import com.oceanbase.odc.metadb.connection.logicaldatabase.DatabaseMappingEntity; +import com.oceanbase.odc.metadb.connection.logicaldatabase.DatabaseMappingRepository; +import com.oceanbase.odc.service.collaboration.environment.EnvironmentService; +import com.oceanbase.odc.service.collaboration.environment.model.Environment; +import com.oceanbase.odc.service.connection.database.DatabaseMapper; +import com.oceanbase.odc.service.connection.database.model.DatabaseType; +import com.oceanbase.odc.service.connection.logicaldatabase.model.CreateLogicalDatabaseReq; +import com.oceanbase.odc.service.iam.ProjectPermissionValidator; +import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; +import com.oceanbase.odc.service.permission.DBResourcePermissionHelper; +import com.oceanbase.odc.test.tool.TestRandom; + +/** + * @Author: Lebie + * @Date: 2024/5/21 16:00 + * @Description: [] + */ +public class LogicalDatabaseServiceTest extends ServiceTestEnv { + private static final Long ORGANIZATION_ID = 1L; + private static final Long USER_ID = 1L; + private static final Long CONNECTION_ID = 1L; + private static final Long ENVIRONMENT_ID = 1L; + private static final Long PROJECT_ID = 1L; + private static final Long PHYSICAL_DATABASE_ID = 1L; + private static final Long LOGICAL_DATABASE_ID = 2L; + private static final String LOGICAL_DATABASE_NAME = "lebie_test"; + private static final String LOGICAL_DATABASE_ALIAS = "lebie_test_alias"; + private static final DatabaseMapper databaseMapper = DatabaseMapper.INSTANCE; + + @Autowired + private LogicalDatabaseService logicalDatabaseService; + @MockBean + private DatabaseRepository databaseRepository; + @MockBean + private AuthenticationFacade authenticationFacade; + @MockBean + private ProjectPermissionValidator projectPermissionValidator; + @MockBean + private ConnectionConfigRepository connectionRepository; + @MockBean + private EnvironmentService environmentService; + @MockBean + private DBResourcePermissionHelper permissionHelper; + @MockBean + private DatabaseMappingRepository databaseMappingRepository; + + @Before + public void setUp() throws Exception { + when(authenticationFacade.currentOrganizationId()).thenReturn(ORGANIZATION_ID); + when(connectionRepository.findById(anyLong())).thenReturn(Optional.of(getConnectionEntity())); + when(environmentService.detailSkipPermissionCheck(anyLong())) + .thenReturn(TestRandom.nextObject(Environment.class)); + when(databaseRepository.findById(PHYSICAL_DATABASE_ID)).thenReturn(Optional.of(getPhysicalDatabase())); + when(databaseRepository.saveAndFlush(any(DatabaseEntity.class))).thenReturn(getLogicalDatabase()); + when(databaseRepository.findById(LOGICAL_DATABASE_ID)).thenReturn(Optional.of(getLogicalDatabase())); + when(databaseRepository.findByIdIn(anyCollection())).thenReturn(Arrays.asList(getPhysicalDatabase())); + doNothing().when(projectPermissionValidator).checkProjectRole(anyLong(), anyList()); + doNothing().when(permissionHelper).checkDBPermissions(anyCollection(), anyCollection()); + } + + @Test + public void testCreate() { + CreateLogicalDatabaseReq req = new CreateLogicalDatabaseReq(); + req.setProjectId(PROJECT_ID); + req.setAlias(LOGICAL_DATABASE_ALIAS); + req.setName(LOGICAL_DATABASE_NAME); + Set databaseIds = new HashSet<>(); + databaseIds.addAll(Arrays.asList(PHYSICAL_DATABASE_ID)); + req.setPhysicalDatabaseIds(databaseIds); + Assert.assertTrue(logicalDatabaseService.create(req)); + } + + @Test + public void testDetail() { + when(databaseMappingRepository.findByLogicalDatabaseId(anyLong())).thenReturn(listDatabaseMappings(1)); + Assert.assertNotNull(logicalDatabaseService.detail(LOGICAL_DATABASE_ID)); + } + + private ConnectionEntity getConnectionEntity() { + ConnectionEntity config = new ConnectionEntity(); + config.setId(CONNECTION_ID); + config.setCreatorId(USER_ID); + config.setDialectType(DialectType.OB_MYSQL); + return config; + } + + private DatabaseEntity getPhysicalDatabase() { + DatabaseEntity entity = TestRandom.nextObject(DatabaseEntity.class); + entity.setId(PHYSICAL_DATABASE_ID); + entity.setType(DatabaseType.PHYSICAL); + entity.setOrganizationId(ORGANIZATION_ID); + entity.setProjectId(PROJECT_ID); + entity.setEnvironmentId(ENVIRONMENT_ID); + return entity; + } + + private List listDatabaseMappings(int quantity) { + List mappings = new ArrayList<>(); + for (int i = 0; i < quantity; i++) { + DatabaseMappingEntity mapping = TestRandom.nextObject(DatabaseMappingEntity.class); + mapping.setOrganizationId(ORGANIZATION_ID); + mapping.setLogicalDatabaseId(LOGICAL_DATABASE_ID); + mapping.setPhysicalDatabaseId((long) i + 1); + mapping.setLogicalDatabaseId(PROJECT_ID); + mappings.add(mapping); + } + return mappings; + } + + private DatabaseEntity getLogicalDatabase() { + DatabaseEntity entity = TestRandom.nextObject(DatabaseEntity.class); + entity.setId(LOGICAL_DATABASE_ID); + entity.setType(DatabaseType.LOGICAL); + entity.setOrganizationId(ORGANIZATION_ID); + entity.setProjectId(PROJECT_ID); + entity.setEnvironmentId(ENVIRONMENT_ID); + return entity; + } + +} diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableFinderTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableFinderTest.java index ff27db22c7..52201c7423 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableFinderTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableFinderTest.java @@ -35,8 +35,9 @@ import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.service.common.util.SqlUtils; import com.oceanbase.odc.service.connection.database.model.Database; -import com.oceanbase.odc.service.connection.logicaldatabase.model.DataNode; -import com.oceanbase.odc.service.connection.logicaldatabase.model.LogicalTable; +import com.oceanbase.odc.service.connection.logicaldatabase.core.LogicalTableFinder; +import com.oceanbase.odc.service.connection.logicaldatabase.core.model.DataNode; +import com.oceanbase.odc.service.connection.logicaldatabase.core.model.LogicalTable; import com.oceanbase.odc.test.database.TestDBConfigurations; import lombok.SneakyThrows; diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableServiceTest.java index fffe277829..083b1ec10a 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableServiceTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableServiceTest.java @@ -24,8 +24,7 @@ import com.oceanbase.odc.ServiceTestEnv; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.common.util.YamlUtils; -import com.oceanbase.odc.service.connection.logicaldatabase.model.DataNode; -import com.oceanbase.odc.service.connection.logicaldatabase.parser.LogicalTableService; +import com.oceanbase.odc.service.connection.logicaldatabase.core.model.DataNode; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ResourceType.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ResourceType.java index 36e1075d11..7369120b55 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ResourceType.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ResourceType.java @@ -77,6 +77,7 @@ public enum ResourceType implements Translatable { ODC_PROJECT, ODC_ENVIRONMENT, ODC_DATABASE, + ODC_LOGICAL_TABLE, ODC_RULESET, ODC_RULE, ODC_SENSITIVE_COLUMN, diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages.properties index bf0e4ad8c1..06b225796d 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages.properties @@ -64,6 +64,7 @@ com.oceanbase.odc.ResourceType.ODC_PROJECT=Project com.oceanbase.odc.ResourceType.ODC_ENVIRONMENT=Environment com.oceanbase.odc.ResourceType.ODC_DATASOURCE=DataSource com.oceanbase.odc.ResourceType.ODC_DATABASE=Database +com.oceanbase.odc.ResourceType.ODC_LOGICAL_TABLE=Logical Table com.oceanbase.odc.ResourceType.ODC_TABLE=Table com.oceanbase.odc.ResourceType.ODC_RULESET=Ruleset com.oceanbase.odc.ResourceType.ODC_RULE=Rule diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties index 90d9a5fbbc..f8dbc8bd96 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties @@ -64,6 +64,7 @@ com.oceanbase.odc.ResourceType.ODC_PROJECT=项目 com.oceanbase.odc.ResourceType.ODC_ENVIRONMENT=环境 com.oceanbase.odc.ResourceType.ODC_DATASOURCE=数据源 com.oceanbase.odc.ResourceType.ODC_DATABASE=数据库 +com.oceanbase.odc.ResourceType.ODC_LOGICAL_TABLE=逻辑表 com.oceanbase.odc.ResourceType.ODC_TABLE=数据表 com.oceanbase.odc.ResourceType.ODC_RULESET=规则集 com.oceanbase.odc.ResourceType.ODC_RULE=规则 diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties index f379475be9..53154c2b8d 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties @@ -64,6 +64,7 @@ com.oceanbase.odc.ResourceType.ODC_PROJECT=項目 com.oceanbase.odc.ResourceType.ODC_ENVIRONMENT=環境 com.oceanbase.odc.ResourceType.ODC_DATASOURCE=數據源 com.oceanbase.odc.ResourceType.ODC_DATABASE=數據庫 +com.oceanbase.odc.ResourceType.ODC_LOGICAL_TABLE=邏輯表 com.oceanbase.odc.ResourceType.ODC_TABLE=數據表 com.oceanbase.odc.ResourceType.ODC_RULESET=規則集 com.oceanbase.odc.ResourceType.ODC_RULE=規則 diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_4_3_1_4__add_logical_database.sql b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_1_4__add_logical_database.sql new file mode 100644 index 0000000000..6b3f79263e --- /dev/null +++ b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_1_4__add_logical_database.sql @@ -0,0 +1,31 @@ +alter table connect_database modify column connection_id bigint(20) DEFAULT NULL COMMENT 'refernce to connect_connection.id'; +alter table connect_database add column `type` varchar(32) NOT NULL DEFAULT 'PHYSICAL' COMMENT 'optional value: PHYSICAL, LOGICAL'; +alter table connect_database add column `alias` varchar(256) DEFAULT NULL COMMENT 'alias name for database'; +alter table `connect_database` modify `last_sync_time` datetime default null comment 'last synchronizing time'; +alter table `connect_database` add column `dialect_type` varchar(64) DEFAULT NULL COMMENT 'database dialect type'; + +create table if not exists `connect_database_mapping`( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `logical_database_id` bigint(20) NOT NULL COMMENT 'reference to connect_database.id whose type is LOGICAL', + `physical_database_id` bigint(20) NOT NULL COMMENT 'reference to connect_database.id whose type is PHYSICAL', + `organization_id` bigint(20) NOT NULL COMMENT 'reference to iam_organization.id', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_connect_database_mapping_pdi` (`physical_database_id`) +); + +create table if not exists `database_table_mapping`( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `logical_table_id` bigint(20) NOT NULL COMMENT 'reference to database_schema_object.id', + `physical_database_id` bigint(20) NOT NULL COMMENT 'reference to database_schema_object.database_id', + `physical_database_name` varchar(256) NOT NULL COMMENT 'reference to connect_database.name', + `physical_table_name` varchar(128) NOT NULL COMMENT 'reference to database_schema_object.name', + `expression` varchar(1024) NOT NULL COMMENT 'logical table expression, e.g., db_[0-3].tb_[0-3]', + `is_consistent` tinyint(1) NOT NULL DEFAULT 1 COMMENT '0: inconsistent, 1: consistent', + `organization_id` bigint(20) NOT NULL COMMENT 'reference to iam_organization.id', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_database_table_mapping_pdi_ptn` (`physical_database_id`, `physical_table_name`) +); \ No newline at end of file diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DataBaseController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DataBaseController.java index fbb8d7af86..f5612b1b07 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DataBaseController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DataBaseController.java @@ -15,6 +15,8 @@ */ package com.oceanbase.odc.server.web.controller.v2; +import java.util.List; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort.Direction; @@ -33,6 +35,7 @@ import com.oceanbase.odc.service.connection.database.DatabaseSyncManager; import com.oceanbase.odc.service.connection.database.model.CreateDatabaseReq; import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.odc.service.connection.database.model.DatabaseType; import com.oceanbase.odc.service.connection.database.model.DeleteDatabasesReq; import com.oceanbase.odc.service.connection.database.model.ModifyDatabaseOwnerReq; import com.oceanbase.odc.service.connection.database.model.QueryDatabaseParams; @@ -65,6 +68,7 @@ public SuccessResponse getDatabase(@PathVariable Long id) { @RequestMapping(value = "/databases", method = RequestMethod.GET) public PaginatedResponse listDatabases( @RequestParam(required = false, name = "name") String name, + @RequestParam(required = false, name = "type") List types, @RequestParam(required = false, name = "existed") Boolean existed, @RequestParam(required = false, name = "dataSourceName") String dataSourceName, @RequestParam(required = false, name = "dataSourceId") Long dataSourceId, @@ -78,6 +82,7 @@ public PaginatedResponse listDatabases( @PageableDefault(size = Integer.MAX_VALUE, sort = {"id"}, direction = Direction.DESC) Pageable pageable) { QueryDatabaseParams params = QueryDatabaseParams.builder() .dataSourceId(dataSourceId) + .types(types) .existed(existed) .environmentId(environmentId) .schemaName(name) diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/LogicalDatabaseController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/LogicalDatabaseController.java new file mode 100644 index 0000000000..a641b4d291 --- /dev/null +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/LogicalDatabaseController.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.server.web.controller.v2; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import com.oceanbase.odc.service.common.response.Responses; +import com.oceanbase.odc.service.common.response.SuccessResponse; +import com.oceanbase.odc.service.connection.logicaldatabase.LogicalDatabaseService; +import com.oceanbase.odc.service.connection.logicaldatabase.LogicalTableService; +import com.oceanbase.odc.service.connection.logicaldatabase.model.CreateLogicalDatabaseReq; +import com.oceanbase.odc.service.connection.logicaldatabase.model.DetailLogicalDatabaseResp; + +/** + * @Author: Lebie + * @Date: 2024/5/7 15:07 + * @Description: [] + */ +@RestController +@RequestMapping("/api/v2/connect/logicaldatabase") +public class LogicalDatabaseController { + @Autowired + private LogicalDatabaseService databaseService; + + @Autowired + private LogicalTableService tableService; + + @RequestMapping(value = "/logicaldatabases", method = RequestMethod.POST) + public SuccessResponse create(@RequestBody CreateLogicalDatabaseReq req) { + return Responses.success(databaseService.create(req)); + } + + @RequestMapping(value = "/logicaldatabases/{id:[\\d]+}", method = RequestMethod.GET) + public SuccessResponse detail(@PathVariable Long id) { + return Responses.success(databaseService.detail(id)); + } + + @RequestMapping(value = "/logicaldatabases/{id:[\\d]+}", method = RequestMethod.DELETE) + public SuccessResponse delete(@PathVariable Long id) { + return Responses.success(databaseService.delete(id)); + } + + @RequestMapping(value = "/logicaldatabases/{logicalDatabaseId:[\\d]+}/logicaltables/{logicalTableId:[\\d]+}", + method = RequestMethod.DELETE) + public SuccessResponse deleteLogicalTable(@PathVariable Long logicalDatabaseId, + @PathVariable Long logicalTableId) { + return Responses.success(tableService.delete(logicalDatabaseId, logicalTableId)); + } + + @RequestMapping(value = "/logicaldatabases/{id:[\\d]+}/logicaltables/extract", method = RequestMethod.POST) + public SuccessResponse extractLogicalTables(@PathVariable Long id) { + return Responses.success(databaseService.extractLogicalTables(id)); + } + + @RequestMapping( + value = "/logicaldatabases/{logicalDatabaseId:[\\d]+}/logicaltables/{logicalTableId:[\\d]+}/checkStructureConsistency", + method = RequestMethod.POST) + public SuccessResponse checkLogicalTable(@PathVariable Long logicalDatabaseId, + @PathVariable Long logicalTableId) { + return Responses.success(tableService.checkStructureConsistency(logicalDatabaseId, logicalTableId)); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/config/ScheduleConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/config/ScheduleConfiguration.java index 16ff918e9a..2dda376db5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/config/ScheduleConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/config/ScheduleConfiguration.java @@ -266,6 +266,24 @@ public ThreadPoolTaskExecutor taskFrameworkMonitorExecutor() { return executor; } + @Bean(name = "logicalTableExtractTaskExecutor") + public ThreadPoolTaskExecutor logicalTableExtractTaskExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + int poolSize = Math.max(SystemUtils.availableProcessors() * 8, 64); + executor.setCorePoolSize(poolSize); + executor.setMaxPoolSize(poolSize); + executor.setQueueCapacity(0); + executor.setThreadNamePrefix("logicaltable-extract-"); + executor.setWaitForTasksToCompleteOnShutdown(true); + executor.setAwaitTerminationSeconds(5); + executor.setTaskDecorator(new TraceDecorator<>()); + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy()); + executor.initialize(); + log.info("logicalTableExtractTaskExecutor initialized"); + return executor; + } + + @Scheduled(fixedDelay = REFRESH_CONFIG_RATE_MILLIS) public void refreshSysConfig() { systemConfigService.refresh(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseEntity.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseEntity.java index 2294db3d47..f2aacf1702 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseEntity.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseEntity.java @@ -29,7 +29,9 @@ import org.hibernate.annotations.Generated; import org.hibernate.annotations.GenerationTime; +import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.service.connection.database.model.DatabaseSyncStatus; +import com.oceanbase.odc.service.connection.database.model.DatabaseType; import com.oceanbase.odc.service.db.schema.model.DBObjectSyncStatus; import lombok.Data; @@ -68,10 +70,21 @@ public class DatabaseEntity { @Column(name = "name", updatable = false, nullable = false) private String name; + @Column(name = "alias") + private String alias; + + @Enumerated(value = EnumType.STRING) + @Column(name = "type", updatable = false, nullable = false) + private DatabaseType type; + + @Enumerated(value = EnumType.STRING) + @Column(name = "dialect_type", updatable = false) + private DialectType dialectType; + @Column(name = "project_id") private Long projectId; - @Column(name = "connection_id", updatable = false, nullable = false) + @Column(name = "connection_id", updatable = false) private Long connectionId; @Column(name = "environment_id", nullable = false) @@ -81,13 +94,13 @@ public class DatabaseEntity { @Column(name = "sync_status", nullable = false) private DatabaseSyncStatus syncStatus; - @Column(name = "last_sync_time", nullable = false) + @Column(name = "last_sync_time") private Date lastSyncTime; - @Column(name = "charset_name", nullable = false) + @Column(name = "charset_name") private String charsetName; - @Column(name = "collation_name", nullable = false) + @Column(name = "collation_name") private String collationName; @Column(name = "table_count") diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseSpecs.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseSpecs.java index 4410362f3e..fd6b669962 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseSpecs.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseSpecs.java @@ -16,13 +16,16 @@ package com.oceanbase.odc.metadb.connection; import java.util.Collection; +import java.util.List; import java.util.Objects; import java.util.Set; import org.apache.commons.collections4.CollectionUtils; import org.springframework.data.jpa.domain.Specification; +import com.oceanbase.odc.common.jpa.SpecificationUtil; import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.service.connection.database.model.DatabaseType; /** * @Author: Lebie @@ -74,4 +77,8 @@ public static Specification projectIdIsNull() { return (root, query, builder) -> builder.isNull(root.get("projectId")); } + public static Specification typeIn(List types) { + return SpecificationUtil.columnIn(DatabaseEntity_.TYPE, types); + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/logicaldatabase/DatabaseMappingEntity.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/logicaldatabase/DatabaseMappingEntity.java new file mode 100644 index 0000000000..23a684bfca --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/logicaldatabase/DatabaseMappingEntity.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.metadb.connection.logicaldatabase; + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.hibernate.annotations.Generated; +import org.hibernate.annotations.GenerationTime; + +import lombok.Data; + +/** + * @Author: Lebie + * @Date: 2024/5/7 14:51 + * @Description: [] + */ +@Data +@Entity +@Table(name = "connect_database_mapping") +public class DatabaseMappingEntity { + @Id + @Column(name = "id", nullable = false) + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Generated(GenerationTime.ALWAYS) + @Column(name = "create_time", insertable = false, updatable = false) + private Date createTime; + + @Generated(GenerationTime.ALWAYS) + @Column(name = "update_time", insertable = false, updatable = false) + private Date updateTime; + + @Column(name = "organization_id", updatable = false, nullable = false) + private Long organizationId; + + @Column(name = "logical_database_id", updatable = false, nullable = false) + private Long logicalDatabaseId; + + @Column(name = "physical_database_id", updatable = false, nullable = false) + private Long physicalDatabaseId; +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/logicaldatabase/DatabaseMappingRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/logicaldatabase/DatabaseMappingRepository.java new file mode 100644 index 0000000000..0508fd0d79 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/logicaldatabase/DatabaseMappingRepository.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.metadb.connection.logicaldatabase; + +import java.util.List; +import java.util.function.Function; + +import org.springframework.transaction.annotation.Transactional; + +import com.oceanbase.odc.common.jpa.InsertSqlTemplateBuilder; +import com.oceanbase.odc.config.jpa.OdcJpaRepository; + +public interface DatabaseMappingRepository extends OdcJpaRepository { + + List findByLogicalDatabaseId(Long logicalDatabaseId); + + @Transactional + int deleteByLogicalDatabaseId(Long logicalDatabaseId); + + default List batchCreate(List entities) { + String sql = InsertSqlTemplateBuilder.from("connect_database_mapping") + .field(DatabaseMappingEntity_.logicalDatabaseId) + .field(DatabaseMappingEntity_.physicalDatabaseId) + .field(DatabaseMappingEntity_.organizationId) + .build(); + List> getter = valueGetterBuilder() + .add(DatabaseMappingEntity::getLogicalDatabaseId) + .add(DatabaseMappingEntity::getPhysicalDatabaseId) + .add(DatabaseMappingEntity::getOrganizationId) + .build(); + return batchCreate(entities, sql, getter, DatabaseMappingEntity::setId, 200); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/logicaldatabase/TableMappingEntity.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/logicaldatabase/TableMappingEntity.java new file mode 100644 index 0000000000..b461071dd5 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/logicaldatabase/TableMappingEntity.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.metadb.connection.logicaldatabase; + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.hibernate.annotations.Generated; +import org.hibernate.annotations.GenerationTime; + +import lombok.Data; + +/** + * @Author: Lebie + * @Date: 2024/5/8 16:26 + * @Description: [] + */ +@Data +@Entity +@Table(name = "database_table_mapping") +public class TableMappingEntity { + @Id + @Column(name = "id", nullable = false) + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Generated(GenerationTime.ALWAYS) + @Column(name = "create_time", insertable = false, updatable = false) + private Date createTime; + + @Generated(GenerationTime.ALWAYS) + @Column(name = "update_time", insertable = false, updatable = false) + private Date updateTime; + + @Column(name = "organization_id", updatable = false, nullable = false) + private Long organizationId; + + @Column(name = "logical_table_id", updatable = false, nullable = false) + private Long logicalTableId; + + @Column(name = "physical_database_id", updatable = false, nullable = false) + private Long physicalDatabaseId; + + @Column(name = "physical_database_name", updatable = false, nullable = false) + private String physicalDatabaseName; + + @Column(name = "physical_table_name", updatable = false, nullable = false) + private String physicalTableName; + + @Column(name = "expression", updatable = false, nullable = false) + private String expression; + + @Column(name = "is_consistent", nullable = false) + private Boolean consistent; +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/logicaldatabase/TableMappingRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/logicaldatabase/TableMappingRepository.java new file mode 100644 index 0000000000..8ef979132d --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/logicaldatabase/TableMappingRepository.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.metadb.connection.logicaldatabase; + +import java.util.Collection; +import java.util.List; +import java.util.function.Function; + +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; + +import com.oceanbase.odc.common.jpa.InsertSqlTemplateBuilder; +import com.oceanbase.odc.config.jpa.OdcJpaRepository; + +public interface TableMappingRepository extends OdcJpaRepository { + List findByLogicalTableIdIn(Collection logicalTableId); + + List findByLogicalTableId(Long logicalTableId); + + @Transactional + int deleteByLogicalTableId(Long logicalTableId); + + @Modifying + @Transactional + @Query(value = "delete from database_table_mapping t where t.physical_database_id in (:physicalDatabaseIds)", + nativeQuery = true) + int deleteByPhysicalDatabaseIds(@Param("physicalDatabaseIds") Collection physicalDatabaseIds); + + default List batchCreate(List entities) { + String sql = InsertSqlTemplateBuilder.from("database_table_mapping") + .field(TableMappingEntity_.logicalTableId) + .field(TableMappingEntity_.physicalDatabaseId) + .field(TableMappingEntity_.physicalTableName) + .field(TableMappingEntity_.expression) + .field(TableMappingEntity_.consistent) + .field(TableMappingEntity_.organizationId) + .build(); + List> getter = valueGetterBuilder() + .add(TableMappingEntity::getLogicalTableId) + .add(TableMappingEntity::getPhysicalDatabaseId) + .add(TableMappingEntity::getPhysicalTableName) + .add(TableMappingEntity::getExpression) + .add(TableMappingEntity::getConsistent) + .add(TableMappingEntity::getOrganizationId) + .build(); + return batchCreate(entities, sql, getter, TableMappingEntity::setId, 200); + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java index d82ce1f217..e9c45b3aaa 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java @@ -268,6 +268,7 @@ public Page list(@NonNull QueryDatabaseParams params, @NotNull Pageabl Specification specs = DatabaseSpecs .environmentIdEquals(params.getEnvironmentId()) .and(DatabaseSpecs.nameLike(params.getSchemaName())) + .and(DatabaseSpecs.typeIn(params.getTypes())) .and(DatabaseSpecs.existedEquals(params.getExisted())) .and(DatabaseSpecs.organizationIdEquals(authenticationFacade.currentOrganizationId())); Set joinedProjectIds = diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/Database.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/Database.java index 29d342f5aa..28fd4b2f18 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/Database.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/Database.java @@ -21,6 +21,7 @@ import java.util.Set; import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty.Access; @@ -57,6 +58,9 @@ public class Database implements SecurityResource, OrganizationIsolated, Seriali @NotBlank private String name; + @NotNull + private DatabaseType type; + private Project project; private ConnectionConfig dataSource; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/DatabaseSyncStatus.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/DatabaseSyncStatus.java index f7f851ab3c..6279152081 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/DatabaseSyncStatus.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/DatabaseSyncStatus.java @@ -20,5 +20,5 @@ public enum DatabaseSyncStatus { SUCCEEDED, - PENDING, + INITIALIZED, } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/DatabaseType.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/DatabaseType.java new file mode 100644 index 0000000000..20b5706a8d --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/DatabaseType.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection.database.model; + +public enum DatabaseType { + LOGICAL, + PHYSICAL, +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/QueryDatabaseParams.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/QueryDatabaseParams.java index 1ceccf03dd..3155d763c7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/QueryDatabaseParams.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/QueryDatabaseParams.java @@ -15,6 +15,8 @@ */ package com.oceanbase.odc.service.connection.database.model; +import java.util.List; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -32,6 +34,8 @@ public class QueryDatabaseParams { private String schemaName; + private List types; + private Long projectId; private Long dataSourceId; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalDatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalDatabaseService.java new file mode 100644 index 0000000000..e4669a32ff --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalDatabaseService.java @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection.logicaldatabase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.validation.Valid; +import javax.validation.constraints.NotNull; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.task.TaskRejectedException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.validation.annotation.Validated; + +import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.core.authority.util.SkipAuthorize; +import com.oceanbase.odc.core.shared.Verify; +import com.oceanbase.odc.core.shared.constant.ResourceRoleName; +import com.oceanbase.odc.core.shared.constant.ResourceType; +import com.oceanbase.odc.core.shared.exception.BadRequestException; +import com.oceanbase.odc.core.shared.exception.NotFoundException; +import com.oceanbase.odc.metadb.connection.ConnectionConfigRepository; +import com.oceanbase.odc.metadb.connection.ConnectionEntity; +import com.oceanbase.odc.metadb.connection.DatabaseEntity; +import com.oceanbase.odc.metadb.connection.DatabaseRepository; +import com.oceanbase.odc.metadb.connection.logicaldatabase.DatabaseMappingEntity; +import com.oceanbase.odc.metadb.connection.logicaldatabase.DatabaseMappingRepository; +import com.oceanbase.odc.metadb.connection.logicaldatabase.TableMappingRepository; +import com.oceanbase.odc.metadb.dbobject.DBObjectRepository; +import com.oceanbase.odc.service.collaboration.environment.EnvironmentService; +import com.oceanbase.odc.service.collaboration.environment.model.Environment; +import com.oceanbase.odc.service.connection.database.DatabaseService; +import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.odc.service.connection.database.model.DatabaseSyncStatus; +import com.oceanbase.odc.service.connection.database.model.DatabaseType; +import com.oceanbase.odc.service.connection.logicaldatabase.model.CreateLogicalDatabaseReq; +import com.oceanbase.odc.service.connection.logicaldatabase.model.DetailLogicalDatabaseResp; +import com.oceanbase.odc.service.db.schema.model.DBObjectSyncStatus; +import com.oceanbase.odc.service.iam.ProjectPermissionValidator; +import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; +import com.oceanbase.odc.service.permission.DBResourcePermissionHelper; + +import lombok.extern.slf4j.Slf4j; + +/** + * @Author: Lebie + * @Date: 2024/5/8 17:33 + * @Description: [] + */ +@Service +@SkipAuthorize +@Slf4j +@Validated +public class LogicalDatabaseService { + + @Autowired + private ProjectPermissionValidator projectPermissionValidator; + + @Autowired + private DatabaseRepository databaseRepository; + + @Autowired + private DatabaseMappingRepository databaseMappingRepository; + + @Autowired + private DBObjectRepository dbObjectRepository; + + @Autowired + private TableMappingRepository tableMappingRepository; + + @Autowired + private ConnectionConfigRepository connectionRepository; + + @Autowired + private AuthenticationFacade authenticationFacade; + + @Autowired + private EnvironmentService environmentService; + + @Autowired + private DatabaseService databaseService; + + @Autowired + private LogicalTableService tableService; + + @Autowired + private LogicalDatabaseSyncManager syncManager; + + @Autowired + private DBResourcePermissionHelper permissionHelper; + + + @Transactional(rollbackFor = Exception.class) + public Boolean create(@Valid CreateLogicalDatabaseReq req) { + preCheck(req); + + Long organizationId = authenticationFacade.currentOrganizationId(); + DatabaseEntity basePhysicalDatabase = databaseRepository + .findById(req.getPhysicalDatabaseIds().iterator().next()).orElseThrow(() -> new NotFoundException( + ResourceType.ODC_DATABASE, "id", req.getPhysicalDatabaseIds().iterator().next())); + ConnectionEntity baseConnection = connectionRepository.findById(basePhysicalDatabase.getConnectionId()) + .orElseThrow(() -> new NotFoundException( + ResourceType.ODC_CONNECTION, "id", basePhysicalDatabase.getConnectionId())); + DatabaseEntity logicalDatabase = new DatabaseEntity(); + logicalDatabase.setProjectId(req.getProjectId()); + logicalDatabase.setName(req.getName()); + logicalDatabase.setAlias(req.getAlias()); + logicalDatabase.setEnvironmentId(basePhysicalDatabase.getEnvironmentId()); + logicalDatabase.setDatabaseId(StringUtils.uuid()); + logicalDatabase.setType(DatabaseType.LOGICAL); + logicalDatabase.setDialectType(baseConnection.getDialectType()); + logicalDatabase.setSyncStatus(DatabaseSyncStatus.INITIALIZED); + logicalDatabase.setObjectSyncStatus(DBObjectSyncStatus.INITIALIZED); + logicalDatabase.setExisted(true); + logicalDatabase.setOrganizationId(organizationId); + DatabaseEntity savedLogicalDatabase = databaseRepository.saveAndFlush(logicalDatabase); + + List mappings = new ArrayList<>(); + req.getPhysicalDatabaseIds().stream().forEach(physicalDatabaseId -> { + DatabaseMappingEntity relation = new DatabaseMappingEntity(); + relation.setLogicalDatabaseId(savedLogicalDatabase.getId()); + relation.setPhysicalDatabaseId(physicalDatabaseId); + relation.setOrganizationId(organizationId); + mappings.add(relation); + }); + databaseMappingRepository.batchCreate(mappings); + + return true; + } + + public DetailLogicalDatabaseResp detail(@NotNull Long id) { + DatabaseEntity logicalDatabase = databaseRepository.findById(id).orElseThrow(() -> new NotFoundException( + ResourceType.ODC_DATABASE, "id", id)); + Verify.equals(DatabaseType.LOGICAL, logicalDatabase.getType(), "database type"); + projectPermissionValidator.checkProjectRole(logicalDatabase.getProjectId(), ResourceRoleName.all()); + + Environment environment = environmentService.detailSkipPermissionCheck(logicalDatabase.getEnvironmentId()); + + Set physicalDBIds = + databaseMappingRepository.findByLogicalDatabaseId(logicalDatabase.getId()).stream() + .map(DatabaseMappingEntity::getPhysicalDatabaseId).collect(Collectors.toSet()); + List physicalDatabases = databaseService.listDatabasesByIds(physicalDBIds); + + DetailLogicalDatabaseResp resp = new DetailLogicalDatabaseResp(); + resp.setId(logicalDatabase.getId()); + resp.setName(logicalDatabase.getName()); + resp.setAlias(logicalDatabase.getAlias()); + resp.setDialectType(logicalDatabase.getDialectType()); + resp.setEnvironment(environment); + resp.setPhysicalDatabases(physicalDatabases); + resp.setLogicalTables(tableService.list(logicalDatabase.getId())); + + return resp; + } + + @Transactional(rollbackFor = Exception.class) + public Boolean delete(@NotNull Long id) { + DatabaseEntity logicalDatabase = databaseRepository.findById(id).orElseThrow(() -> new NotFoundException( + ResourceType.ODC_DATABASE, "id", id)); + Verify.equals(DatabaseType.LOGICAL, logicalDatabase.getType(), "database type"); + projectPermissionValidator.checkProjectRole(logicalDatabase.getProjectId(), + Arrays.asList(ResourceRoleName.DBA, ResourceRoleName.OWNER)); + + databaseRepository.deleteById(id); + Set physicalDBIds = databaseMappingRepository.findByLogicalDatabaseId(id).stream() + .map(DatabaseMappingEntity::getId).collect( + Collectors.toSet()); + databaseMappingRepository.deleteByLogicalDatabaseId(id); + dbObjectRepository.deleteByDatabaseIdIn(Collections.singleton(id)); + tableMappingRepository.deleteByPhysicalDatabaseIds(physicalDBIds); + return true; + } + + public boolean extractLogicalTables(@NotNull Long logicalDatabaseId) { + Database logicalDatabase = + databaseService.getBasicSkipPermissionCheck(logicalDatabaseId); + Verify.equals(logicalDatabase.getType(), DatabaseType.LOGICAL, "database type"); + projectPermissionValidator.checkProjectRole(logicalDatabase.getProject().getId(), + Arrays.asList(ResourceRoleName.DBA, ResourceRoleName.OWNER)); + try { + syncManager.submitExtractLogicalTablesTask(logicalDatabase); + } catch (TaskRejectedException ex) { + log.warn("submit extract logical tables task rejected, logical database id={}", logicalDatabaseId); + return false; + } + return true; + } + + protected void preCheck(CreateLogicalDatabaseReq req) { + projectPermissionValidator.checkProjectRole(req.getProjectId(), + Arrays.asList(ResourceRoleName.DBA, ResourceRoleName.OWNER)); + List databases = databaseRepository.findByIdIn(req.getPhysicalDatabaseIds()); + Verify.equals(databases.size(), req.getPhysicalDatabaseIds().size(), "physical database"); + + if (!databases.stream().allMatch(database -> Objects.equals(req.getProjectId(), database.getProjectId()))) { + throw new BadRequestException( + "physical databases not all in the same project, project id=" + req.getProjectId()); + } + + if (databases.stream().map(DatabaseEntity::getEnvironmentId).collect(Collectors.toSet()).size() > 1) { + throw new BadRequestException("physical databases are not all the same environment"); + } + + List connections = connectionRepository + .findByIdIn(databases.stream().map(DatabaseEntity::getConnectionId).collect(Collectors.toSet())); + if (connections.stream().map(ConnectionEntity::getDialectType).collect(Collectors.toSet()).size() > 1) { + throw new BadRequestException("physical databases are not all the same dialect type"); + } + + Set aliasNames = databaseRepository.findByProjectId(req.getProjectId()).stream() + .map(DatabaseEntity::getAlias).collect(Collectors.toSet()); + if (aliasNames.contains(req.getAlias())) { + throw new BadRequestException("alias name already exists, alias=" + req.getAlias()); + } + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalDatabaseSyncManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalDatabaseSyncManager.java new file mode 100644 index 0000000000..63eb861d51 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalDatabaseSyncManager.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection.logicaldatabase; + +import java.util.concurrent.Future; +import java.util.function.Supplier; + +import javax.validation.constraints.NotNull; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.integration.jdbc.lock.JdbcLockRegistry; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.stereotype.Service; + +import com.oceanbase.odc.core.shared.exception.BadRequestException; +import com.oceanbase.odc.metadb.connection.DatabaseRepository; +import com.oceanbase.odc.metadb.connection.logicaldatabase.DatabaseMappingRepository; +import com.oceanbase.odc.metadb.connection.logicaldatabase.TableMappingRepository; +import com.oceanbase.odc.metadb.dbobject.DBObjectRepository; +import com.oceanbase.odc.service.connection.database.DatabaseService; +import com.oceanbase.odc.service.connection.database.model.Database; + +import lombok.extern.slf4j.Slf4j; + +/** + * @Author: Lebie + * @Date: 2024/5/10 11:41 + * @Description: [] + */ + +@Service +@Slf4j +public class LogicalDatabaseSyncManager { + @Autowired + @Qualifier("logicalTableExtractTaskExecutor") + private ThreadPoolTaskExecutor executor; + @Autowired + private DatabaseRepository databaseRepository; + @Autowired + private DatabaseService databaseService; + @Autowired + private DatabaseMappingRepository dbRelationRepository; + @Autowired + private TableMappingRepository tableRelationRepository; + @Autowired + private DBObjectRepository dbObjectRepository; + @Autowired + private JdbcLockRegistry jdbcLockRegistry; + + public void submitExtractLogicalTablesTask(@NotNull Database logicalDatabase) { + doExecute(() -> executor + .submit(new LogicalTableExtractTask(logicalDatabase, databaseRepository, dbRelationRepository, + databaseService, dbObjectRepository, tableRelationRepository, jdbcLockRegistry))); + } + + public void submitCheckConsistencyTask(@NotNull Long logicalTableId) { + doExecute(() -> executor + .submit(new LogicalTableCheckConsistencyTask(logicalTableId, tableRelationRepository, + databaseService))); + } + + private Future doExecute(Supplier> supplier) { + try { + return supplier.get(); + } catch (Exception ex) { + throw new BadRequestException("submit logical table extract task failed, ", ex); + } + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableCheckConsistencyTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableCheckConsistencyTask.java new file mode 100644 index 0000000000..4bf9e5aab4 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableCheckConsistencyTask.java @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection.logicaldatabase; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import com.oceanbase.odc.core.shared.exception.UnexpectedException; +import com.oceanbase.odc.metadb.connection.logicaldatabase.TableMappingEntity; +import com.oceanbase.odc.metadb.connection.logicaldatabase.TableMappingRepository; +import com.oceanbase.odc.service.connection.database.DatabaseService; +import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.odc.service.connection.logicaldatabase.core.LogicalTableFinder; +import com.oceanbase.odc.service.connection.logicaldatabase.core.model.DataNode; +import com.oceanbase.tools.dbbrowser.model.DBTable; + +/** + * @Author: Lebie + * @Date: 2024/5/15 19:34 + * @Description: [] + */ +public class LogicalTableCheckConsistencyTask implements Runnable { + private final Long tableId; + private final TableMappingRepository relationRepository; + private final DatabaseService databaseService; + + public LogicalTableCheckConsistencyTask(Long tableId, + TableMappingRepository relationRepository, DatabaseService databaseService) { + this.tableId = tableId; + this.relationRepository = relationRepository; + this.databaseService = databaseService; + } + + @Override + public void run() { + List mappings = relationRepository.findByLogicalTableId(this.tableId); + + Set databaseIds = mappings.stream().map(TableMappingEntity::getPhysicalDatabaseId) + .collect(Collectors.toSet()); + Map id2Databases = databaseService.listDatabasesDetailsByIds(databaseIds).stream() + .collect(Collectors.toMap(Database::getId, database -> database)); + + Map> signature2Tables = new HashMap<>(); + mappings.stream().collect(Collectors.groupingBy(TableMappingEntity::getPhysicalDatabaseId)) + .forEach((databaseId, physicalTables) -> { + Database database = id2Databases.get(databaseId); + if (Objects.isNull(database)) { + throw new UnexpectedException("Database not found, databaseId=" + databaseId); + } + Map tableName2Tables = + LogicalTableFinder.getTableName2Tables(database.getDataSource(), database.getName(), + physicalTables.stream().map(TableMappingEntity::getPhysicalTableName) + .collect(Collectors.toList())); + physicalTables.forEach(physicalTable -> { + DBTable table = tableName2Tables.get(physicalTable.getPhysicalTableName()); + if (Objects.isNull(table)) { + return; + } + signature2Tables + .computeIfAbsent(DataNode.getStructureSignature(table), key -> new ArrayList<>()) + .add(physicalTable); + }); + }); + + Optional>> largestEntryOptional = + signature2Tables.entrySet().stream() + .max(Comparator.comparingInt(entry -> entry.getValue().size())); + + if (largestEntryOptional.isPresent()) { + Entry> largestEntry = largestEntryOptional.get(); + signature2Tables.values().stream() + .flatMap(List::stream) + .forEach(table -> table.setConsistent(false)); + + largestEntry.getValue().forEach(table -> table.setConsistent(true)); + relationRepository.saveAll(mappings); + } + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableExtractTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableExtractTask.java new file mode 100644 index 0000000000..e164d7b0b1 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableExtractTask.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection.logicaldatabase; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.stream.Collectors; + +import org.apache.commons.collections.CollectionUtils; +import org.springframework.integration.jdbc.lock.JdbcLockRegistry; + +import com.oceanbase.odc.core.shared.constant.ErrorCodes; +import com.oceanbase.odc.core.shared.exception.ConflictException; +import com.oceanbase.odc.metadb.connection.DatabaseRepository; +import com.oceanbase.odc.metadb.connection.logicaldatabase.DatabaseMappingEntity; +import com.oceanbase.odc.metadb.connection.logicaldatabase.DatabaseMappingRepository; +import com.oceanbase.odc.metadb.connection.logicaldatabase.TableMappingEntity; +import com.oceanbase.odc.metadb.connection.logicaldatabase.TableMappingRepository; +import com.oceanbase.odc.metadb.dbobject.DBObjectEntity; +import com.oceanbase.odc.metadb.dbobject.DBObjectRepository; +import com.oceanbase.odc.service.connection.database.DatabaseService; +import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.odc.service.connection.logicaldatabase.core.LogicalTableFinder; +import com.oceanbase.odc.service.connection.logicaldatabase.core.model.DataNode; +import com.oceanbase.odc.service.connection.logicaldatabase.core.model.LogicalTable; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; + +import lombok.NonNull; + +/** + * @Author: Lebie + * @Date: 2024/5/10 14:01 + * @Description: [] + */ +public class LogicalTableExtractTask implements Runnable { + private final Database logicalDatabase; + private final DatabaseService databaseService; + private final DatabaseMappingRepository dbRelationRepository; + private final TableMappingRepository tableRelationRepository; + private final DBObjectRepository dbObjectRepository; + private JdbcLockRegistry jdbcLockRegistry; + + public LogicalTableExtractTask(@NonNull Database logicalDatabase, @NonNull DatabaseRepository databaseRepository, + @NonNull DatabaseMappingRepository dbRelationRepository, @NonNull DatabaseService databaseService, + @NonNull DBObjectRepository dbObjectRepository, + @NonNull TableMappingRepository tableRelationRepository, + @NonNull JdbcLockRegistry jdbcLockRegistry) { + this.logicalDatabase = logicalDatabase; + this.dbRelationRepository = dbRelationRepository; + this.databaseService = databaseService; + this.dbObjectRepository = dbObjectRepository; + this.tableRelationRepository = tableRelationRepository; + this.jdbcLockRegistry = jdbcLockRegistry; + } + + @Override + public void run() { + List relations = + dbRelationRepository.findByLogicalDatabaseId(logicalDatabase.getId()); + List physicalDatabases = databaseService.listDatabasesDetailsByIds( + relations.stream().map(DatabaseMappingEntity::getPhysicalDatabaseId).collect(Collectors.toList())); + List logicalTables = new LogicalTableFinder(physicalDatabases).find(); + if (CollectionUtils.isEmpty(logicalTables)) { + return; + } + + Lock lock = jdbcLockRegistry.obtain("logicaltable-extract-database-id-" + logicalDatabase.getId()); + + try { + if (!lock.tryLock(3, TimeUnit.SECONDS)) { + throw new ConflictException(ErrorCodes.ResourceModifying, "Can not acquire jdbc lock"); + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + try { + Set existedTables = dbObjectRepository.findByDatabaseIdAndType(logicalDatabase.getId(), + DBObjectType.LOGICAL_TABLE).stream().map(DBObjectEntity::getName).collect(Collectors.toSet()); + + logicalTables.stream().filter(table -> !existedTables.contains(table.getName())).forEach(table -> { + DBObjectEntity tableEntity = new DBObjectEntity(); + tableEntity.setDatabaseId(logicalDatabase.getId()); + tableEntity.setType(DBObjectType.LOGICAL_TABLE); + tableEntity.setName(table.getName()); + tableEntity.setOrganizationId(logicalDatabase.getOrganizationId()); + DBObjectEntity savedTableEntity = dbObjectRepository.save(tableEntity); + + List dataNodes = table.getActualDataNodes(); + List physicalTableEntities = new ArrayList<>(); + dataNodes.stream().forEach(dataNode -> { + TableMappingEntity physicalTableEntity = new TableMappingEntity(); + physicalTableEntity.setLogicalTableId(savedTableEntity.getId()); + physicalTableEntity.setOrganizationId(logicalDatabase.getOrganizationId()); + physicalTableEntity.setPhysicalDatabaseId(dataNode.getDatabaseId()); + physicalTableEntity.setPhysicalDatabaseName(dataNode.getSchemaName()); + physicalTableEntity.setPhysicalTableName(dataNode.getTableName()); + physicalTableEntity.setExpression(table.getFullNameExpression()); + physicalTableEntity.setConsistent(true); + physicalTableEntities.add(physicalTableEntity); + }); + tableRelationRepository.batchCreate(physicalTableEntities); + }); + } finally { + if (lock != null) { + lock.unlock(); + } + } + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableService.java new file mode 100644 index 0000000000..f6b65e931c --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableService.java @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection.logicaldatabase; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.validation.constraints.NotNull; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.task.TaskRejectedException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.validation.annotation.Validated; + +import com.oceanbase.odc.core.authority.util.SkipAuthorize; +import com.oceanbase.odc.core.shared.PreConditions; +import com.oceanbase.odc.core.shared.Verify; +import com.oceanbase.odc.core.shared.constant.ResourceRoleName; +import com.oceanbase.odc.core.shared.constant.ResourceType; +import com.oceanbase.odc.core.shared.exception.NotFoundException; +import com.oceanbase.odc.core.shared.exception.UnexpectedException; +import com.oceanbase.odc.metadb.connection.DatabaseEntity; +import com.oceanbase.odc.metadb.connection.DatabaseRepository; +import com.oceanbase.odc.metadb.connection.logicaldatabase.TableMappingEntity; +import com.oceanbase.odc.metadb.connection.logicaldatabase.TableMappingRepository; +import com.oceanbase.odc.metadb.dbobject.DBObjectEntity; +import com.oceanbase.odc.metadb.dbobject.DBObjectRepository; +import com.oceanbase.odc.service.connection.database.model.DatabaseType; +import com.oceanbase.odc.service.connection.logicaldatabase.core.model.DataNode; +import com.oceanbase.odc.service.connection.logicaldatabase.core.parser.BadLogicalTableExpressionException; +import com.oceanbase.odc.service.connection.logicaldatabase.core.parser.DefaultLogicalTableExpressionParser; +import com.oceanbase.odc.service.connection.logicaldatabase.core.parser.LogicalTableExpressions; +import com.oceanbase.odc.service.connection.logicaldatabase.model.DetailLogicalTableResp; +import com.oceanbase.odc.service.iam.ProjectPermissionValidator; +import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; +import com.oceanbase.odc.service.permission.DBResourcePermissionHelper; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; +import com.oceanbase.tools.sqlparser.SyntaxErrorException; + +import lombok.extern.slf4j.Slf4j; + +/** + * @Author: Lebie + * @Date: 2024/4/23 11:31 + * @Description: [] + */ +@Service +@SkipAuthorize +@Slf4j +@Validated +public class LogicalTableService { + private final DefaultLogicalTableExpressionParser parser = new DefaultLogicalTableExpressionParser(); + + @Autowired + private DatabaseRepository databaseRepository; + + @Autowired + private DBObjectRepository dbObjectRepository; + + @Autowired + private TableMappingRepository mappingRepository; + + @Autowired + private AuthenticationFacade authenticationFacade; + + @Autowired + private ProjectPermissionValidator projectPermissionValidator; + + @Autowired + private LogicalDatabaseSyncManager syncManager; + + @Autowired + private DBResourcePermissionHelper permissionHelper; + + public List list(@NotNull Long logicalDatabaseId) { + DatabaseEntity logicalDatabase = + databaseRepository.findById(logicalDatabaseId).orElseThrow(() -> new NotFoundException( + ResourceType.ODC_DATABASE, "id", logicalDatabaseId)); + Verify.equals(DatabaseType.LOGICAL, logicalDatabase.getType(), "database type"); + projectPermissionValidator.checkProjectRole(logicalDatabase.getProjectId(), ResourceRoleName.all()); + + List logicalTables = + dbObjectRepository.findByDatabaseIdAndType(logicalDatabaseId, DBObjectType.LOGICAL_TABLE); + + + Set logicalTableIds = logicalTables.stream().map(DBObjectEntity::getId).collect(Collectors.toSet()); + if (CollectionUtils.isEmpty(logicalTableIds)) { + return Collections.emptyList(); + } + Map> logicalTbId2Mappings = + mappingRepository.findByLogicalTableIdIn(logicalTableIds).stream() + .collect(Collectors.groupingBy(TableMappingEntity::getLogicalTableId)); + + + return logicalTables.stream().map(tableEntity -> { + DetailLogicalTableResp resp = new DetailLogicalTableResp(); + List relations = + logicalTbId2Mappings.getOrDefault(tableEntity.getId(), Collections.emptyList()); + resp.setId(tableEntity.getId()); + resp.setName(tableEntity.getName()); + resp.setExpression(relations.isEmpty() ? StringUtils.EMPTY : relations.get(0).getExpression()); + resp.setPhysicalTableCount(relations.size()); + List inconsistentPhysicalTables = new ArrayList<>(); + relations.stream().filter(relation -> !relation.getConsistent()).forEach(relation -> { + DataNode dataNode = new DataNode(); + dataNode.setSchemaName(relation.getPhysicalDatabaseName()); + dataNode.setTableName(relation.getPhysicalTableName()); + inconsistentPhysicalTables.add(dataNode); + }); + resp.setInconsistentPhysicalTables(inconsistentPhysicalTables); + return resp; + }).collect(Collectors.toList()); + } + + @Transactional(rollbackFor = Exception.class) + public Boolean delete(@NotNull Long logicalDatabaseId, @NotNull Long logicalTableId) { + DBObjectEntity logicalTable = dbObjectRepository.findById(logicalTableId) + .orElseThrow(() -> new NotFoundException(ResourceType.ODC_LOGICAL_TABLE, "id", logicalTableId)); + Verify.equals(logicalTable.getDatabaseId(), logicalDatabaseId, "logical database id"); + DatabaseEntity logicalDatabase = + databaseRepository.findById(logicalDatabaseId).orElseThrow(() -> new NotFoundException( + ResourceType.ODC_DATABASE, "id", logicalDatabaseId)); + projectPermissionValidator.checkProjectRole(logicalDatabase.getProjectId(), + Arrays.asList(ResourceRoleName.DBA, ResourceRoleName.OWNER)); + + dbObjectRepository.deleteById(logicalTableId); + mappingRepository.deleteByLogicalTableId(logicalTableId); + return true; + } + + public Boolean checkStructureConsistency(@NotNull Long logicalDatabaseId, @NotNull Long logicalTableId) { + DBObjectEntity table = dbObjectRepository.findById(logicalTableId) + .orElseThrow(() -> new NotFoundException(ResourceType.ODC_LOGICAL_TABLE, "id", logicalTableId)); + Verify.equals(table.getDatabaseId(), logicalDatabaseId, "logical database id"); + try { + syncManager.submitCheckConsistencyTask(logicalTableId); + } catch (TaskRejectedException ex) { + log.warn("submit check logical table structure consistency task rejected, logical table id={}", + logicalTableId); + return false; + } + return true; + } + + + public List resolve(String expression) { + PreConditions.notEmpty(expression, "expression"); + LogicalTableExpressions logicalTableExpression; + try { + logicalTableExpression = (LogicalTableExpressions) parser.parse(new StringReader(expression)); + } catch (SyntaxErrorException e) { + throw new BadLogicalTableExpressionException(e); + } catch (Exception e) { + throw new UnexpectedException("failed to parse logical table expression", e); + } + return logicalTableExpression.evaluate().stream().map(name -> { + String[] parts = name.split("\\."); + if (parts.length != 2) { + throw new UnexpectedException("invalid logical table expression"); + } + return new DataNode(parts[0], parts[1]); + }).collect(Collectors.toList()); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableFinder.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/LogicalTableFinder.java similarity index 82% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableFinder.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/LogicalTableFinder.java index 05486081a8..ce2da1c5a2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableFinder.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/LogicalTableFinder.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.connection.logicaldatabase; +package com.oceanbase.odc.service.connection.logicaldatabase.core; import java.util.ArrayList; import java.util.Collections; @@ -21,15 +21,19 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; +import javax.validation.constraints.NotEmpty; + +import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.session.ConnectionSessionFactory; -import com.oceanbase.odc.core.shared.PreConditions; +import com.oceanbase.odc.core.shared.exception.UnexpectedException; import com.oceanbase.odc.service.connection.database.model.Database; -import com.oceanbase.odc.service.connection.logicaldatabase.model.DataNode; -import com.oceanbase.odc.service.connection.logicaldatabase.model.LogicalTable; +import com.oceanbase.odc.service.connection.logicaldatabase.core.model.DataNode; +import com.oceanbase.odc.service.connection.logicaldatabase.core.model.LogicalTable; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.db.browser.DBSchemaAccessors; import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; @@ -45,35 +49,49 @@ * @Description: [] */ @Slf4j +@SkipAuthorize("internal usage") public class LogicalTableFinder { private List databases; + private Map id2DataSource; + private Map> dataSourceId2Databases; - LogicalTableFinder(List databases) { + public LogicalTableFinder(@NotEmpty List databases) { this.databases = databases; - } - - public List find() { - PreConditions.notEmpty(databases, "LogicalTableFinder#find.databases"); - Map> dataSourceId2Databases = + this.id2DataSource = new HashMap<>(); + this.dataSourceId2Databases = databases.stream().collect(Collectors.groupingBy(database -> database.getDataSource().getId())); - Map id2DataSource = new HashMap<>(); databases.stream() - .forEach(database -> id2DataSource.put(database.getDataSource().getId(), database.getDataSource())); + .forEach( + database -> this.id2DataSource.put(database.getDataSource().getId(), database.getDataSource())); + } + public List transferToDataNodes() { List dataNodes = new ArrayList<>(); Set dataSourceIds = dataSourceId2Databases.keySet(); for (Long dataSourceId : dataSourceIds) { List groupedDatabases = dataSourceId2Databases.get(dataSourceId); + Map name2Database = + groupedDatabases.stream().collect(Collectors.toMap(Database::getName, database -> database)); ConnectionConfig dataSource = id2DataSource.get(dataSourceId); getSchemaName2TableNames(dataSource, groupedDatabases).entrySet().forEach(entry -> { String databaseName = entry.getKey(); List tableNames = entry.getValue(); tableNames.forEach(tableName -> { - DataNode dataNode = new DataNode(dataSource, databaseName, tableName); + Database database = name2Database.get(databaseName); + if (Objects.isNull(database)) { + throw new UnexpectedException("Database not found: " + databaseName); + } + DataNode dataNode = new DataNode(dataSource, database.getId(), databaseName, tableName); dataNodes.add(dataNode); }); }); } + return dataNodes; + } + + public List find() { + List dataNodes = transferToDataNodes(); + List logicalTableCandidates = LogicalTableRecognitionUtils.recognizeLogicalTables(dataNodes); Map> dataSourceId2DataNodes = logicalTableCandidates.stream() @@ -142,7 +160,7 @@ private static Map> getSchemaName2TableNames(ConnectionConf } } - private static Map getTableName2Tables(ConnectionConfig dataSource, String schemaName, + public static Map getTableName2Tables(ConnectionConfig dataSource, String schemaName, List tableNames) { ConnectionSessionFactory connectionSessionFactory = new DefaultConnectSessionFactory(dataSource); ConnectionSession connectionSession = connectionSessionFactory.generateSession(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableRecognitionUtils.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/LogicalTableRecognitionUtils.java similarity index 99% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableRecognitionUtils.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/LogicalTableRecognitionUtils.java index 7dd4b7880a..c329e50f9e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableRecognitionUtils.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/LogicalTableRecognitionUtils.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.connection.logicaldatabase; +package com.oceanbase.odc.service.connection.logicaldatabase.core; import java.util.ArrayList; import java.util.Arrays; @@ -39,8 +39,8 @@ import com.oceanbase.odc.core.shared.PreConditions; import com.oceanbase.odc.core.shared.Verify; import com.oceanbase.odc.core.shared.exception.UnexpectedException; -import com.oceanbase.odc.service.connection.logicaldatabase.model.DataNode; -import com.oceanbase.odc.service.connection.logicaldatabase.model.LogicalTable; +import com.oceanbase.odc.service.connection.logicaldatabase.core.model.DataNode; +import com.oceanbase.odc.service.connection.logicaldatabase.core.model.LogicalTable; /** * @Author: Lebie diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/DataNode.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/model/DataNode.java similarity index 87% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/DataNode.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/model/DataNode.java index 88552c1b0a..35d7900055 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/DataNode.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/model/DataNode.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.connection.logicaldatabase.model; +package com.oceanbase.odc.service.connection.logicaldatabase.core.model; import java.util.Collection; import java.util.Collections; @@ -35,6 +35,7 @@ import lombok.AllArgsConstructor; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; /** @@ -45,12 +46,15 @@ @Data @AllArgsConstructor @NoArgsConstructor +@EqualsAndHashCode(of = {"databaseId", "schemaName", "tableName"}) public class DataNode { private static final String DELIMITER = "."; @JsonIgnore private ConnectionConfig dataSourceConfig; + private Long databaseId; + private String schemaName; private String tableName; @@ -60,13 +64,19 @@ public DataNode(String schemaName, String tableName) { this.tableName = tableName; } + public DataNode(Long databaseId, String schemaName, String tableName) { + this.databaseId = databaseId; + this.schemaName = schemaName; + this.tableName = tableName; + } + public String getFullName() { return schemaName + DELIMITER + tableName; } @JsonIgnore - public String getStructureSignature(DBTable table) { + public static String getStructureSignature(DBTable table) { if (Objects.isNull(table)) { return "[ODC] NULL OBJECT"; } @@ -78,7 +88,7 @@ public String getStructureSignature(DBTable table) { .sha1(String.join("|||", columnSignature, indexSignature, constraintSignature, tableOptionSignature)); } - private String getTableOptionSignature(DBTableOptions tableOptions) { + private static String getTableOptionSignature(DBTableOptions tableOptions) { if (tableOptions == null) { return "[ODC] NULL OBJECT"; } @@ -86,7 +96,7 @@ private String getTableOptionSignature(DBTableOptions tableOptions) { nullSafeGet(tableOptions.getCollationName())); } - private String getColumnsSignature(List columns) { + private static String getColumnsSignature(List columns) { if (CollectionUtils.isEmpty(columns)) { return "[ODC] NULL LIST"; } @@ -107,7 +117,7 @@ private String getColumnsSignature(List columns) { .collect(Collectors.joining("||")); } - private String getIndexesSignature(List indexes) { + private static String getIndexesSignature(List indexes) { if (CollectionUtils.isEmpty(indexes)) { return "[ODC] NULL LIST"; } @@ -127,7 +137,7 @@ private String getIndexesSignature(List indexes) { .collect(Collectors.joining("||")); } - private String getConstraintsSignature(List constraints) { + private static String getConstraintsSignature(List constraints) { if (CollectionUtils.isEmpty(constraints)) { return "[ODC] NULL LIST"; } @@ -145,12 +155,12 @@ private String getConstraintsSignature(List constraints) { .collect(Collectors.joining("||")); } - private String nullSafeGet(Object obj) { + private static String nullSafeGet(Object obj) { if (obj instanceof Collection) { if (CollectionUtils.isEmpty((Collection) obj)) { return "[ODC] NULL LIST"; } - return (String) ((Collection) obj).stream().map(this::nullSafeGet).collect(Collectors.joining(",")); + return (String) ((Collection) obj).stream().map(DataNode::nullSafeGet).collect(Collectors.joining(",")); } return obj == null ? "[ODC] NULL OBJECT" : obj.toString(); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/LogicalDatabase.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/model/LogicalDatabase.java similarity index 93% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/LogicalDatabase.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/model/LogicalDatabase.java index b0cb3a16b9..086882c3d7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/LogicalDatabase.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/model/LogicalDatabase.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.connection.logicaldatabase.model; +package com.oceanbase.odc.service.connection.logicaldatabase.core.model; import java.util.List; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/LogicalTable.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/model/LogicalTable.java similarity index 97% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/LogicalTable.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/model/LogicalTable.java index 66904a7497..16afbb3f16 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/LogicalTable.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/model/LogicalTable.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.connection.logicaldatabase.model; +package com.oceanbase.odc.service.connection.logicaldatabase.core.model; import java.util.Collections; import java.util.List; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/BadExpressionException.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/BadLogicalTableExpressionException.java similarity index 87% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/BadExpressionException.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/BadLogicalTableExpressionException.java index a74aa5f3f8..5ce54ea180 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/BadExpressionException.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/BadLogicalTableExpressionException.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.connection.logicaldatabase.parser; +package com.oceanbase.odc.service.connection.logicaldatabase.core.parser; import org.apache.commons.lang.StringUtils; import org.springframework.http.HttpStatus; @@ -28,13 +28,13 @@ * @Date: 2024/4/23 14:03 * @Description: [] */ -class BadExpressionException extends HttpException { +public class BadLogicalTableExpressionException extends HttpException { - public BadExpressionException(ErrorCode errorCode, Object[] args, String message) { + public BadLogicalTableExpressionException(ErrorCode errorCode, Object[] args, String message) { super(errorCode, args, message); } - public BadExpressionException(SyntaxErrorException ex) { + public BadLogicalTableExpressionException(SyntaxErrorException ex) { super(ErrorCodes.LogicalTableBadExpressionSyntax, new Object[] {buildErrorMessage(ex.getText(), ex.getStart(), ex.getStop())}, ErrorCodes.LogicalTableBadExpressionSyntax.getEnglishMessage( diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/BaseLogicalTableExpression.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/BaseLogicalTableExpression.java similarity index 85% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/BaseLogicalTableExpression.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/BaseLogicalTableExpression.java index 42409207b4..2db8876668 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/BaseLogicalTableExpression.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/BaseLogicalTableExpression.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.connection.logicaldatabase.parser; +package com.oceanbase.odc.service.connection.logicaldatabase.core.parser; import java.util.List; @@ -23,7 +23,7 @@ public abstract class BaseLogicalTableExpression extends BaseStatement { - public abstract List evaluate() throws BadExpressionException; + public abstract List evaluate() throws BadLogicalTableExpressionException; BaseLogicalTableExpression(ParserRuleContext ruleNode) { super(ruleNode); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/BaseRangeExpression.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/BaseRangeExpression.java similarity index 84% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/BaseRangeExpression.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/BaseRangeExpression.java index e4223bb1e8..74bfc67f2a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/BaseRangeExpression.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/BaseRangeExpression.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.connection.logicaldatabase.parser; +package com.oceanbase.odc.service.connection.logicaldatabase.core.parser; import java.util.List; @@ -22,7 +22,7 @@ import com.oceanbase.tools.sqlparser.statement.BaseStatement; public abstract class BaseRangeExpression extends BaseStatement { - public abstract List listRanges() throws BadExpressionException; + public abstract List listRanges() throws BadLogicalTableExpressionException; BaseRangeExpression(ParserRuleContext ruleNode) { super(ruleNode); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/ConsecutiveSliceRange.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/ConsecutiveSliceRange.java similarity index 88% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/ConsecutiveSliceRange.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/ConsecutiveSliceRange.java index db4af4bdf9..c36133c7d3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/ConsecutiveSliceRange.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/ConsecutiveSliceRange.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.connection.logicaldatabase.parser; +package com.oceanbase.odc.service.connection.logicaldatabase.core.parser; import java.util.List; @@ -40,7 +40,7 @@ public class ConsecutiveSliceRange extends BaseRangeExpression { @Override - public List listRanges() throws BadExpressionException { + public List listRanges() throws BadLogicalTableExpressionException { return LogicalTableExpressionParseUtils.listSteppedRanges(rangeStart, rangeEnd, "1", this.getText()); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/DefaultLogicalTableExpressionParser.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/DefaultLogicalTableExpressionParser.java similarity index 96% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/DefaultLogicalTableExpressionParser.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/DefaultLogicalTableExpressionParser.java index 9d35fcdc6a..088352de4f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/DefaultLogicalTableExpressionParser.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/DefaultLogicalTableExpressionParser.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.connection.logicaldatabase.parser; +package com.oceanbase.odc.service.connection.logicaldatabase.core.parser; import java.io.IOException; import java.io.Reader; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/EnumRange.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/EnumRange.java similarity index 84% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/EnumRange.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/EnumRange.java index 1393d2b55f..a7a64a4354 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/EnumRange.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/EnumRange.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.connection.logicaldatabase.parser; +package com.oceanbase.odc.service.connection.logicaldatabase.core.parser; import java.util.List; @@ -38,12 +38,12 @@ public class EnumRange extends BaseRangeExpression { } @Override - public List listRanges() throws BadExpressionException { + public List listRanges() throws BadLogicalTableExpressionException { enumValues.stream().forEach(value -> { try { Integer.parseInt(value); } catch (NumberFormatException e) { - throw new BadExpressionException(ErrorCodes.LogicalTableExpressionNotValidIntegerRange, + throw new BadLogicalTableExpressionException(ErrorCodes.LogicalTableExpressionNotValidIntegerRange, new Object[] {this.getText()}, ErrorCodes.LogicalTableExpressionNotValidIntegerRange .getEnglishMessage(new Object[] {this.getText()})); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/LogicalTableExpression.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/LogicalTableExpression.java similarity index 93% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/LogicalTableExpression.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/LogicalTableExpression.java index 5d8132b1a1..db7d82ac7b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/LogicalTableExpression.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/LogicalTableExpression.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.connection.logicaldatabase.parser; +package com.oceanbase.odc.service.connection.logicaldatabase.core.parser; import static com.oceanbase.odc.core.shared.constant.ErrorCodes.LogicalTableExpressionNotEvenlyDivided; @@ -46,7 +46,7 @@ public class LogicalTableExpression extends BaseLogicalTableExpression { } @Override - public List evaluate() throws BadExpressionException { + public List evaluate() throws BadLogicalTableExpressionException { List schemaNames = schemaExpression.evaluate(); List tableNames = tableExpression.evaluate(); Verify.notEmpty(schemaNames, "schemaNames"); @@ -69,7 +69,7 @@ public List evaluate() throws BadExpressionException { return names; } if (tableNames.size() % schemaNames.size() != 0) { - throw new BadExpressionException( + throw new BadLogicalTableExpressionException( LogicalTableExpressionNotEvenlyDivided, new Object[] {tableNames.size(), schemaNames.size()}, LogicalTableExpressionNotEvenlyDivided .getEnglishMessage(new Object[] {tableNames.size(), schemaNames.size()})); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/LogicalTableExpressionParseUtils.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/LogicalTableExpressionParseUtils.java similarity index 83% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/LogicalTableExpressionParseUtils.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/LogicalTableExpressionParseUtils.java index 4d0751fc6d..e8d2b9515d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/LogicalTableExpressionParseUtils.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/LogicalTableExpressionParseUtils.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.connection.logicaldatabase.parser; +package com.oceanbase.odc.service.connection.logicaldatabase.core.parser; import static com.oceanbase.odc.core.shared.constant.ErrorCodes.LogicalTableExpressionNotPositiveStep; @@ -30,7 +30,7 @@ */ public class LogicalTableExpressionParseUtils { public static List listSteppedRanges(String start, String end, String step, String text) - throws BadExpressionException { + throws BadLogicalTableExpressionException { PreConditions.notEmpty(start, "start"); PreConditions.notEmpty(end, "end"); PreConditions.notEmpty(step, "step"); @@ -41,17 +41,18 @@ public static List listSteppedRanges(String start, String end, String st endInt = Integer.parseInt(end); stepInt = Integer.parseInt(step); } catch (NumberFormatException e) { - throw new BadExpressionException(ErrorCodes.LogicalTableExpressionNotValidIntegerRange, + throw new BadLogicalTableExpressionException(ErrorCodes.LogicalTableExpressionNotValidIntegerRange, new Object[] {text}, ErrorCodes.LogicalTableExpressionNotValidIntegerRange.getEnglishMessage(new Object[] {text})); } if (stepInt <= 0) { - throw new BadExpressionException(LogicalTableExpressionNotPositiveStep, new Object[] {text, stepInt}, + throw new BadLogicalTableExpressionException(LogicalTableExpressionNotPositiveStep, + new Object[] {text, stepInt}, LogicalTableExpressionNotPositiveStep.getEnglishMessage(new Object[] {text, stepInt})); } if (startInt > endInt) { - throw new BadExpressionException(ErrorCodes.LogicalTableExpressionRangeStartGreaterThanEnd, + throw new BadLogicalTableExpressionException(ErrorCodes.LogicalTableExpressionRangeStartGreaterThanEnd, new Object[] {text, startInt, endInt}, ErrorCodes.LogicalTableExpressionRangeStartGreaterThanEnd .getEnglishMessage(new Object[] {text, startInt, endInt})); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/LogicalTableExpressionVisitor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/LogicalTableExpressionVisitor.java similarity index 99% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/LogicalTableExpressionVisitor.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/LogicalTableExpressionVisitor.java index d7f74fdcd7..626872abc3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/LogicalTableExpressionVisitor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/LogicalTableExpressionVisitor.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.connection.logicaldatabase.parser; +package com.oceanbase.odc.service.connection.logicaldatabase.core.parser; import java.util.ArrayList; import java.util.List; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/LogicalTableExpressions.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/LogicalTableExpressions.java similarity index 89% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/LogicalTableExpressions.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/LogicalTableExpressions.java index e6aac73c95..daa4403a81 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/LogicalTableExpressions.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/LogicalTableExpressions.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.connection.logicaldatabase.parser; +package com.oceanbase.odc.service.connection.logicaldatabase.core.parser; import java.util.List; import java.util.stream.Collectors; @@ -40,7 +40,7 @@ public class LogicalTableExpressions extends BaseLogicalTableExpression { } @Override - public List evaluate() throws BadExpressionException { + public List evaluate() throws BadLogicalTableExpressionException { return this.expressions.stream().flatMap(expression -> expression.evaluate().stream()) .collect(Collectors.toList()); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/SchemaExpression.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/SchemaExpression.java similarity index 94% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/SchemaExpression.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/SchemaExpression.java index 2cd087983e..584fdf7177 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/SchemaExpression.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/SchemaExpression.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.connection.logicaldatabase.parser; +package com.oceanbase.odc.service.connection.logicaldatabase.core.parser; import java.util.ArrayList; import java.util.Arrays; @@ -48,7 +48,7 @@ public class SchemaExpression extends BaseLogicalTableExpression { } @Override - public List evaluate() throws BadExpressionException { + public List evaluate() throws BadLogicalTableExpressionException { PreConditions.notEmpty(this.getText(), "expression"); if (CollectionUtils.isEmpty(sliceRanges)) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/SteppedRange.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/SteppedRange.java similarity index 89% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/SteppedRange.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/SteppedRange.java index cab9892ce9..f9b5a197c4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/SteppedRange.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/SteppedRange.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.connection.logicaldatabase.parser; +package com.oceanbase.odc.service.connection.logicaldatabase.core.parser; import java.util.List; @@ -40,7 +40,7 @@ public class SteppedRange extends BaseRangeExpression { } @Override - public List listRanges() throws BadExpressionException { + public List listRanges() throws BadLogicalTableExpressionException { return LogicalTableExpressionParseUtils.listSteppedRanges(rangeStart, rangeEnd, rangeStep, this.getText()); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/TableExpression.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/TableExpression.java similarity index 93% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/TableExpression.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/TableExpression.java index 8ae06ca865..b3df7bf88d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/TableExpression.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/core/parser/TableExpression.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.connection.logicaldatabase.parser; +package com.oceanbase.odc.service.connection.logicaldatabase.core.parser; import org.antlr.v4.runtime.ParserRuleContext; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/CreateLogicalDatabaseReq.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/CreateLogicalDatabaseReq.java new file mode 100644 index 0000000000..0528a8a972 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/CreateLogicalDatabaseReq.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection.logicaldatabase.model; + +import java.util.Set; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; + +import lombok.Data; + +/** + * @Author: Lebie + * @Date: 2024/5/7 15:37 + * @Description: [] + */ +@Data +public class CreateLogicalDatabaseReq { + @NotNull + private Long projectId; + + @NotNull + private String name; + + @NotNull + private String alias; + + @NotEmpty + private Set physicalDatabaseIds; +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/DetailLogicalDatabaseResp.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/DetailLogicalDatabaseResp.java new file mode 100644 index 0000000000..47ac91b860 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/DetailLogicalDatabaseResp.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection.logicaldatabase.model; + +import java.util.List; + +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.service.collaboration.environment.model.Environment; +import com.oceanbase.odc.service.connection.database.model.Database; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @Author: Lebie + * @Date: 2024/5/7 16:44 + * @Description: [] + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class DetailLogicalDatabaseResp { + private Long id; + + private String name; + + private String alias; + + private DialectType dialectType; + + private Environment environment; + + private List physicalDatabases; + + private List logicalTables; +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/DetailLogicalTableResp.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/DetailLogicalTableResp.java new file mode 100644 index 0000000000..1eeb2a903d --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/model/DetailLogicalTableResp.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection.logicaldatabase.model; + +import java.util.Date; +import java.util.List; + +import com.oceanbase.odc.service.connection.logicaldatabase.core.model.DataNode; +import com.oceanbase.tools.dbbrowser.model.DBTable; + +import lombok.Data; + +/** + * @Author: Lebie + * @Date: 2024/5/7 19:39 + * @Description: [] + */ + +@Data +public class DetailLogicalTableResp { + private Long id; + + private String name; + + private String expression; + + private Integer physicalTableCount; + + private List inconsistentPhysicalTables; + + private DBTable basePhysicalTable; +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/LogicalTableService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/LogicalTableService.java deleted file mode 100644 index 092a35444d..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/parser/LogicalTableService.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.connection.logicaldatabase.parser; - -import java.io.StringReader; -import java.util.List; -import java.util.stream.Collectors; - -import org.springframework.stereotype.Service; - -import com.oceanbase.odc.core.shared.PreConditions; -import com.oceanbase.odc.core.shared.exception.UnexpectedException; -import com.oceanbase.odc.service.connection.logicaldatabase.model.DataNode; -import com.oceanbase.tools.sqlparser.SyntaxErrorException; - -/** - * @Author: Lebie - * @Date: 2024/4/23 11:31 - * @Description: [] - */ -@Service -public class LogicalTableService { - private final DefaultLogicalTableExpressionParser parser = new DefaultLogicalTableExpressionParser(); - - public List resolve(String expression) { - PreConditions.notEmpty(expression, "expression"); - LogicalTableExpressions logicalTableExpression; - try { - logicalTableExpression = (LogicalTableExpressions) parser.parse(new StringReader(expression)); - } catch (SyntaxErrorException e) { - throw new BadExpressionException(e); - } catch (Exception e) { - throw new UnexpectedException("failed to parse logical table expression", e); - } - return logicalTableExpression.evaluate().stream().map(name -> { - String[] parts = name.split("\\."); - if (parts.length != 2) { - throw new UnexpectedException("invalid logical table expression"); - } - return new DataNode(parts[0], parts[1]); - }).collect(Collectors.toList()); - } -} diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableRecognitionUtilsTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/connection/logicaldatabase/core/LogicalTableRecognitionUtilsTest.java similarity index 94% rename from server/odc-service/src/test/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableRecognitionUtilsTest.java rename to server/odc-service/src/test/java/com/oceanbase/odc/service/connection/logicaldatabase/core/LogicalTableRecognitionUtilsTest.java index 1a8db99183..8c18d7b78e 100644 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableRecognitionUtilsTest.java +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/connection/logicaldatabase/core/LogicalTableRecognitionUtilsTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.connection.logicaldatabase; +package com.oceanbase.odc.service.connection.logicaldatabase.core; import java.util.List; @@ -21,8 +21,8 @@ import org.junit.Test; import com.oceanbase.odc.common.util.YamlUtils; -import com.oceanbase.odc.service.connection.logicaldatabase.model.DataNode; -import com.oceanbase.odc.service.connection.logicaldatabase.model.LogicalTable; +import com.oceanbase.odc.service.connection.logicaldatabase.core.model.DataNode; +import com.oceanbase.odc.service.connection.logicaldatabase.core.model.LogicalTable; import lombok.AllArgsConstructor; import lombok.Data; From 5b50dbe71ec876bf4fb0280253eacd3f05491dbe Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Mon, 27 May 2024 08:40:18 +0800 Subject: [PATCH 08/64] refactor(config): Add more fields to configEntity #2493 --- .../com/oceanbase/odc/metadb/config/ConfigEntity.java | 4 ++-- .../odc/metadb/config/SystemConfigEntity.java | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/ConfigEntity.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/ConfigEntity.java index fce4125322..cdfde24e10 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/ConfigEntity.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/ConfigEntity.java @@ -54,13 +54,13 @@ public class ConfigEntity { /** * Created time */ - @Column(name = "create_time") + @Column(name = "create_time", insertable = false, updatable = false) private Date createTime; /** * Latest update time */ - @Column(name = "update_time") + @Column(name = "update_time", insertable = false, updatable = false) private Date updateTime; /** diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigEntity.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigEntity.java index 182d29d26c..2602716f6e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigEntity.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/config/SystemConfigEntity.java @@ -39,14 +39,22 @@ public class SystemConfigEntity extends ConfigEntity { @Column(name = "last_modifier_id") private Long lastModifierId; + + @Column(name = "`value_deprecated`") + private String valueDeprecated; + /** * application name */ + @Column(updatable = false) private String application; - /** * profile for Spring Cloud Config */ + @Column(updatable = false) private String profile; + @Column(updatable = false) + private String label; + } From 350689cb05d6886c28bb6dfc3749f40dd19bb960 Mon Sep 17 00:00:00 2001 From: krihy <51393259+krihy@users.noreply.github.com> Date: Wed, 5 Jun 2024 19:01:37 +0800 Subject: [PATCH 09/64] feat(osc): add rate limiter for osc (#2402) * add rate limiter * add log * fix terminate osc job * add database id * reset schedule service * add project id * fix throttle rps * add rate-limit.enabled * fix rate limiter * rm unused code * add . * fix start oms project * update task parameters * revert datasql * add system config * modify for pr comment * format code * modify comment --- .../web/controller/v2/OscController.java | 9 ++ server/odc-server/src/main/resources/data.sql | 3 + .../OnlineSchemaChangeFlowableTask.java | 1 + .../onlineschemachange/OscService.java | 85 ++++++++++++++++--- .../model/OnlineSchemaChangeParameters.java | 1 + ...ineSchemaChangeScheduleTaskParameters.java | 3 + .../model/RateLimiterConfig.java | 37 ++++++++ .../model/UpdateRateLimiterConfigRequest.java | 37 ++++++++ .../oms/openapi/OmsProjectOpenApiService.java | 8 ++ .../openapi/OmsProjectOpenApiServiceImpl.java | 10 +++ .../oms/request/FullTransferConfig.java | 7 +- .../oms/request/IncrTransferConfig.java | 5 +- .../oms/request/ThrottleConfig.java | 28 ++++++ .../request/UpdateProjectConfigRequest.java | 51 +++++++++++ .../pipeline/BaseCreateOmsProjectValve.java | 12 ++- .../ScheduleCheckOmsProjectValve.java | 77 +++++++++++++++++ 16 files changed, 355 insertions(+), 19 deletions(-) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/model/RateLimiterConfig.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/model/UpdateRateLimiterConfigRequest.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/request/ThrottleConfig.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/request/UpdateProjectConfigRequest.java diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/OscController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/OscController.java index 1875204fe9..7609ce7f65 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/OscController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/OscController.java @@ -17,6 +17,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; @@ -26,6 +27,7 @@ import com.oceanbase.odc.service.onlineschemachange.OscService; import com.oceanbase.odc.service.onlineschemachange.model.OscLockDatabaseUserInfo; import com.oceanbase.odc.service.onlineschemachange.model.OscSwapTableVO; +import com.oceanbase.odc.service.onlineschemachange.model.UpdateRateLimiterConfigRequest; import io.swagger.annotations.ApiOperation; @@ -54,4 +56,11 @@ public SuccessResponse swapTable(@PathVariable Long scheduleTask return Responses.success(oscService.swapTable(scheduleTaskId)); } + @ApiOperation(value = "updateRateLimitConfig", notes = "update osc rate limit config") + @RequestMapping(value = "/updateRateLimitConfig", method = RequestMethod.POST) + public SuccessResponse updateRateLimitConfig( + @RequestBody UpdateRateLimiterConfigRequest updateRateLimiterConfig) { + return Responses.success(oscService.updateRateLimiterConfig(updateRateLimiterConfig)); + } + } diff --git a/server/odc-server/src/main/resources/data.sql b/server/odc-server/src/main/resources/data.sql index 631522d9c4..5f2f511efb 100644 --- a/server/odc-server/src/main/resources/data.sql +++ b/server/odc-server/src/main/resources/data.sql @@ -658,6 +658,9 @@ VALUES ( 'odc.rollback.query-data-batch-size', '1000', '生成备份回滚方案批量查询数据的数量' ) ON DUPLICATE KEY UPDATE `id` = `id`; +INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('odc.osc.rate-limit.enabled','false', 'enable OSC rate limit') + ON DUPLICATE KEY UPDATE `id`=`id`; + INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('odc.osc.cloud.enabled-instance-ids', '', 'instances that enable OSC') ON DUPLICATE KEY UPDATE `id`=`id`; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTask.java index ee655ac7b9..8befd39eaf 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTask.java @@ -122,6 +122,7 @@ protected Void start(Long taskId, TaskService taskService, DelegateExecution exe List tasks = parameter.generateSubTaskParameters(connectionConfig, schema).stream() .map(param -> { param.setUid(uid); + param.setRateLimitConfig(parameter.getRateLimitConfig()); return createScheduleTaskEntity(schedule.getId(), param); }).collect(Collectors.toList()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OscService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OscService.java index 516d5443a0..48498b5427 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OscService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OscService.java @@ -24,6 +24,7 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.web.bind.annotation.PathVariable; +import com.fasterxml.jackson.core.type.TypeReference; import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.session.ConnectionSession; @@ -39,22 +40,30 @@ import com.oceanbase.odc.metadb.schedule.ScheduleRepository; import com.oceanbase.odc.metadb.schedule.ScheduleTaskEntity; import com.oceanbase.odc.metadb.schedule.ScheduleTaskRepository; +import com.oceanbase.odc.metadb.task.TaskEntity; import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.flow.FlowInstanceService; import com.oceanbase.odc.service.flow.factory.FlowFactory; import com.oceanbase.odc.service.flow.instance.FlowInstance; +import com.oceanbase.odc.service.flow.task.model.OnlineSchemaChangeTaskResult; import com.oceanbase.odc.service.iam.HorizontalDataPermissionValidator; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeParameters; import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeScheduleTaskResult; import com.oceanbase.odc.service.onlineschemachange.model.OscLockDatabaseUserInfo; import com.oceanbase.odc.service.onlineschemachange.model.OscSwapTableVO; +import com.oceanbase.odc.service.onlineschemachange.model.RateLimiterConfig; import com.oceanbase.odc.service.onlineschemachange.model.SwapTableType; +import com.oceanbase.odc.service.onlineschemachange.model.UpdateRateLimiterConfigRequest; import com.oceanbase.odc.service.onlineschemachange.rename.OscDBUserUtil; +import com.oceanbase.odc.service.schedule.ScheduleService; +import com.oceanbase.odc.service.schedule.ScheduleTaskService; import com.oceanbase.odc.service.schedule.model.JobType; import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; +import com.oceanbase.odc.service.task.TaskService; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -84,6 +93,14 @@ public class OscService { private FlowFactory flowFactory; @Autowired private DatabaseService databaseService; + @Autowired + private FlowInstanceService flowInstanceService; + @Autowired + private ScheduleTaskService scheduleTaskService; + @Autowired + private ScheduleService scheduleService; + @Autowired + private TaskService taskService; @SkipAuthorize("internal authenticated") @@ -115,20 +132,7 @@ public OscSwapTableVO swapTable(@PathVariable Long scheduleTaskId) { OnlineSchemaChangeParameters oscParameters = JsonUtils.fromJson(scheduleEntity.get().getJobParametersJson(), OnlineSchemaChangeParameters.class); - Optional optional = flowFactory.getFlowInstance(oscParameters.getFlowInstanceId()); - FlowInstance flowInstance = optional.orElseThrow( - () -> new NotFoundException(ResourceType.ODC_FLOW_INSTANCE, "id", oscParameters.getFlowInstanceId())); - try { - permissionValidator.checkCurrentOrganization(flowInstance); - } finally { - flowInstance.dealloc(); - } - - // check user permission, only creator can swap table manual - PreConditions.validHasPermission( - Objects.equals(authenticationFacade.currentUserId(), scheduleEntity.get().getCreatorId()), - ErrorCodes.AccessDenied, - "no permission swap table."); + checkPermission(oscParameters.getFlowInstanceId()); OnlineSchemaChangeScheduleTaskResult result = JsonUtils.fromJson(scheduleTask.getResultJson(), OnlineSchemaChangeScheduleTaskResult.class); @@ -161,6 +165,59 @@ public OscSwapTableVO swapTable(@PathVariable Long scheduleTaskId) { return oscSwapTable; } + @Transactional(rollbackFor = Exception.class) + @SkipAuthorize("internal authenticated") + public boolean updateRateLimiterConfig(UpdateRateLimiterConfigRequest req) { + log.info("Get updateRateLimiterConfig from request, req={}", JsonUtils.toJson(req)); + checkPermission(req.getFlowInstanceId()); + TaskEntity task = flowInstanceService.getTaskByFlowInstanceId(req.getFlowInstanceId()); + PreConditions.notNull(task.getParametersJson(), "result json", + "Task result is empty, taskId=" + task.getId() + ",flowInstanceId=" + req.getFlowInstanceId()); + PreConditions.validArgumentState(!task.getStatus().isTerminated(), ErrorCodes.Unsupported, + new Object[] {req.getFlowInstanceId()}, + "modify task rate limiter is unsupported when task is terminated"); + OnlineSchemaChangeTaskResult taskResult = + JsonUtils.fromJson(task.getResultJson(), new TypeReference() {}); + Long scheduleId = Long.parseLong(taskResult.getTasks().get(0).getJobName()); + ScheduleEntity scheduleEntity = scheduleService.nullSafeGetById(scheduleId); + OnlineSchemaChangeParameters parameters = JsonUtils.fromJson( + scheduleEntity.getJobParametersJson(), OnlineSchemaChangeParameters.class); + RateLimiterConfig rateLimiter = parameters.getRateLimitConfig(); + rateLimiter.setDataSizeLimit(req.getRateLimitConfig().getDataSizeLimit()); + rateLimiter.setRowLimit(req.getRateLimitConfig().getRowLimit()); + parameters.setRateLimitConfig(rateLimiter); + + OnlineSchemaChangeParameters inputParameters = + JsonUtils.fromJson(task.getParametersJson(), OnlineSchemaChangeParameters.class); + inputParameters.setRateLimitConfig(rateLimiter); + task.setParametersJson(JsonUtils.toJson(inputParameters)); + taskService.updateParametersJson(task); + String parameterJson = JsonUtils.toJson(parameters); + int rows = scheduleRepository.updateJobParametersById(scheduleEntity.getId(), parameterJson); + if (rows > 0) { + log.info("Update rate limiter config in job parameters completed, scheduleId={}, parameterJson={}.", + scheduleEntity.getId(), parameterJson); + } + return true; + } + + private void checkPermission(Long flowInstanceId) { + Optional optional = flowFactory.getFlowInstance(flowInstanceId); + FlowInstance flowInstance = optional.orElseThrow( + () -> new NotFoundException(ResourceType.ODC_FLOW_INSTANCE, "id", flowInstanceId)); + try { + permissionValidator.checkCurrentOrganization(flowInstance); + } finally { + flowInstance.dealloc(); + } + + // check user permission, only creator can swap table manual + PreConditions.validHasPermission( + Objects.equals(authenticationFacade.currentUserId(), optional.get().getCreatorId()), + ErrorCodes.AccessDenied, + "no permission swap table."); + } + private boolean getLockUserIsRequired(Long connectionId) { ConnectionConfig decryptedConnConfig = connectionService.getForConnectionSkipPermissionCheck(connectionId); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/model/OnlineSchemaChangeParameters.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/model/OnlineSchemaChangeParameters.java index 6898aa6dbd..941cfc20b6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/model/OnlineSchemaChangeParameters.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/model/OnlineSchemaChangeParameters.java @@ -75,6 +75,7 @@ public class OnlineSchemaChangeParameters implements Serializable, TaskParameter private List lockUsers; private SwapTableType swapTableType; private Long flowInstanceId; + private RateLimiterConfig rateLimitConfig = new RateLimiterConfig(); public boolean isContinueOnError() { return this.errorStrategy == TaskErrorStrategy.CONTINUE; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/model/OnlineSchemaChangeScheduleTaskParameters.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/model/OnlineSchemaChangeScheduleTaskParameters.java index 8a20d29e15..e5a92026f3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/model/OnlineSchemaChangeScheduleTaskParameters.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/model/OnlineSchemaChangeScheduleTaskParameters.java @@ -95,6 +95,9 @@ public class OnlineSchemaChangeScheduleTaskParameters { private ReplaceResult replaceResult; + private RateLimiterConfig rateLimitConfig = new RateLimiterConfig(); + + public String getOriginTableNameWithSchema() { return tableNameWithSchema(originTableName); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/model/RateLimiterConfig.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/model/RateLimiterConfig.java new file mode 100644 index 0000000000..9a98bd877a --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/model/RateLimiterConfig.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.onlineschemachange.model; + +import lombok.Data; + +/** + * @author yaobin + * @date 2024-04-28 + * @since 4.2.4 + */ +@Data +public class RateLimiterConfig { + + /** + * limit max row size per second + */ + private Integer rowLimit; + + /** + * limit max IO size per second, Unit is MB + */ + private Integer dataSizeLimit; +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/model/UpdateRateLimiterConfigRequest.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/model/UpdateRateLimiterConfigRequest.java new file mode 100644 index 0000000000..6d0bf7e3d0 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/model/UpdateRateLimiterConfigRequest.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.onlineschemachange.model; + +import javax.validation.Valid; +import javax.validation.constraints.NotNull; + +import lombok.Data; + +/** + * @author yaobin + * @date 2024-04-28 + * @since 4.2.4 + */ +@Valid +@Data +public class UpdateRateLimiterConfigRequest { + + @NotNull + private Long flowInstanceId; + + private RateLimiterConfig rateLimitConfig = new RateLimiterConfig(); + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/openapi/OmsProjectOpenApiService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/openapi/OmsProjectOpenApiService.java index e2e8bf5b62..c36c9e298d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/openapi/OmsProjectOpenApiService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/openapi/OmsProjectOpenApiService.java @@ -21,6 +21,7 @@ import com.oceanbase.odc.service.onlineschemachange.oms.request.ListOmsProjectFullVerifyResultRequest; import com.oceanbase.odc.service.onlineschemachange.oms.request.ListOmsProjectRequest; import com.oceanbase.odc.service.onlineschemachange.oms.request.OmsProjectControlRequest; +import com.oceanbase.odc.service.onlineschemachange.oms.request.UpdateProjectConfigRequest; import com.oceanbase.odc.service.onlineschemachange.oms.response.OmsProjectFullVerifyResultResponse; import com.oceanbase.odc.service.onlineschemachange.oms.response.OmsProjectProgressResponse; import com.oceanbase.odc.service.onlineschemachange.oms.response.OmsProjectResponse; @@ -94,4 +95,11 @@ public interface OmsProjectOpenApiService { */ OmsProjectFullVerifyResultResponse listProjectFullVerifyResult(ListOmsProjectFullVerifyResultRequest request); + /** + * update project config + * + * @param request config + */ + void updateProjectConfig(UpdateProjectConfigRequest request); + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/openapi/OmsProjectOpenApiServiceImpl.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/openapi/OmsProjectOpenApiServiceImpl.java index 0d53c9ceb1..3aafc5d150 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/openapi/OmsProjectOpenApiServiceImpl.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/openapi/OmsProjectOpenApiServiceImpl.java @@ -28,6 +28,7 @@ import com.oceanbase.odc.service.onlineschemachange.oms.request.ListOmsProjectRequest; import com.oceanbase.odc.service.onlineschemachange.oms.request.OmsApiReturnResult; import com.oceanbase.odc.service.onlineschemachange.oms.request.OmsProjectControlRequest; +import com.oceanbase.odc.service.onlineschemachange.oms.request.UpdateProjectConfigRequest; import com.oceanbase.odc.service.onlineschemachange.oms.response.OmsProjectFullVerifyResultResponse; import com.oceanbase.odc.service.onlineschemachange.oms.response.OmsProjectProgressResponse; import com.oceanbase.odc.service.onlineschemachange.oms.response.OmsProjectResponse; @@ -150,4 +151,13 @@ public OmsProjectFullVerifyResultResponse listProjectFullVerifyResult( return omsClient.postOmsInterface(params); } + @Override + public void updateProjectConfig(UpdateProjectConfigRequest request) { + ClientRequestParams params = new ClientRequestParams() + .setRequest(request) + .setAction("UpdateProjectConfig") + .setTypeReference(new TypeReference>() {}); + + omsClient.postOmsInterface(params); + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/request/FullTransferConfig.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/request/FullTransferConfig.java index 7dc8abadcd..1656453a76 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/request/FullTransferConfig.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/request/FullTransferConfig.java @@ -25,7 +25,7 @@ * @since 4.2.0 */ @Data -public class FullTransferConfig { +public class FullTransferConfig implements ThrottleConfig { /** * 在处理源端无唯一索引表德的全量迁移时，是否 truncate 目标表（清空目标表数据）,组件层面默认为 true 目前场景：PolarDB-X 1.0 @@ -46,4 +46,9 @@ public class FullTransferConfig { * 全量校验 配置(STEADY：平稳，NORMAL：正常，FAST：快速)。 */ private String fullVerifySpeedMode = "STEADY"; + + private Integer throttleRps; + + private Integer throttleIOPS; + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/request/IncrTransferConfig.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/request/IncrTransferConfig.java index aa6bd61a64..71752e762b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/request/IncrTransferConfig.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/request/IncrTransferConfig.java @@ -33,7 +33,7 @@ * @since 4.2.0 */ @Data -public class IncrTransferConfig { +public class IncrTransferConfig implements ThrottleConfig { /** * 勾选了增量且没勾选全量时，允许设置增量起始位点，秒级时间戳 最多30天内，实际根据store情况 @@ -73,4 +73,7 @@ public class IncrTransferConfig { */ private Boolean enableIncrSyncStatistics = Boolean.FALSE; + private Integer throttleRps; + + private Integer throttleIOPS; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/request/ThrottleConfig.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/request/ThrottleConfig.java new file mode 100644 index 0000000000..2fa788c555 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/request/ThrottleConfig.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.onlineschemachange.oms.request; + +/** + * @author yaobin + * @date 2024-04-28 + * @since 4.2.4 + */ +public interface ThrottleConfig { + + Integer getThrottleRps(); + + Integer getThrottleIOPS(); +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/request/UpdateProjectConfigRequest.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/request/UpdateProjectConfigRequest.java new file mode 100644 index 0000000000..84e932c4e1 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/request/UpdateProjectConfigRequest.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.onlineschemachange.oms.request; + +import javax.validation.Valid; +import javax.validation.constraints.NotEmpty; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author yaobin + * @date 2024-04-28 + * @since 4.2.4 + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class UpdateProjectConfigRequest extends BaseOmsRequest { + /** + * Oms project id + */ + @NotEmpty + private String id; + /** + * full transfer config + */ + @Valid + private FullTransferConfig fullTransferConfig; + /** + * increment transfer config + */ + @Valid + private IncrTransferConfig incrTransferConfig; +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/pipeline/BaseCreateOmsProjectValve.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/pipeline/BaseCreateOmsProjectValve.java index d6bebb89dd..431704551c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/pipeline/BaseCreateOmsProjectValve.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/pipeline/BaseCreateOmsProjectValve.java @@ -38,6 +38,7 @@ import com.oceanbase.odc.service.onlineschemachange.configuration.OnlineSchemaChangeProperties; import com.oceanbase.odc.service.onlineschemachange.ddl.DdlConstants; import com.oceanbase.odc.service.onlineschemachange.exception.OmsException; +import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeParameters; import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeScheduleTaskParameters; import com.oceanbase.odc.service.onlineschemachange.oms.enums.OmsOceanBaseType; import com.oceanbase.odc.service.onlineschemachange.oms.openapi.DataSourceOpenApiService; @@ -103,7 +104,7 @@ public void invoke(ValveContext valveContext) { PreConditions.notNull(omsDsId, "Oms datasource id"); CreateOmsProjectRequest createProjectRequest = getCreateProjectRequest(omsDsId, - context.getSchedule().getId(), context.getTaskParameter()); + context.getSchedule().getId(), context.getTaskParameter(), context.getParameter()); String projectId = projectOpenApiService.createProject(createProjectRequest); @@ -155,7 +156,8 @@ private Map getStringObjectMap(Long scheduleTaskId) { } private CreateOmsProjectRequest getCreateProjectRequest(String omsDsId, Long scheduleId, - OnlineSchemaChangeScheduleTaskParameters oscScheduleTaskParameters) { + OnlineSchemaChangeScheduleTaskParameters oscScheduleTaskParameters, + OnlineSchemaChangeParameters oscParameters) { CreateOmsProjectRequest request = new CreateOmsProjectRequest(); doCreateProjectRequest(omsDsId, scheduleId, oscScheduleTaskParameters, request); if (oscProperties.isEnableFullVerify()) { @@ -185,8 +187,12 @@ private CreateOmsProjectRequest getCreateProjectRequest(String omsDsId, Long sch CommonTransferConfig commonTransferConfig = new CommonTransferConfig(); request.setCommonTransferConfig(commonTransferConfig); FullTransferConfig fullTransferConfig = new FullTransferConfig(); - request.setFullTransferConfig(fullTransferConfig); IncrTransferConfig incrTransferConfig = new IncrTransferConfig(); + fullTransferConfig.setThrottleIOPS(oscParameters.getRateLimitConfig().getDataSizeLimit()); + incrTransferConfig.setThrottleIOPS(oscParameters.getRateLimitConfig().getDataSizeLimit()); + fullTransferConfig.setThrottleRps(oscParameters.getRateLimitConfig().getRowLimit()); + incrTransferConfig.setThrottleRps(oscParameters.getRateLimitConfig().getRowLimit()); + request.setFullTransferConfig(fullTransferConfig); request.setIncrTransferConfig(incrTransferConfig); request.setUid(oscScheduleTaskParameters.getUid()); return request; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/pipeline/ScheduleCheckOmsProjectValve.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/pipeline/ScheduleCheckOmsProjectValve.java index 2300e17104..2f05969b67 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/pipeline/ScheduleCheckOmsProjectValve.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/pipeline/ScheduleCheckOmsProjectValve.java @@ -16,6 +16,7 @@ package com.oceanbase.odc.service.onlineschemachange.pipeline; import java.util.List; +import java.util.Objects; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -31,14 +32,19 @@ import com.oceanbase.odc.service.onlineschemachange.exception.OscException; import com.oceanbase.odc.service.onlineschemachange.logger.DefaultTableFactory; import com.oceanbase.odc.service.onlineschemachange.model.FullVerificationResult; +import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeParameters; import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeScheduleTaskParameters; import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeScheduleTaskResult; import com.oceanbase.odc.service.onlineschemachange.model.PrecheckResult; import com.oceanbase.odc.service.onlineschemachange.model.SwapTableType; +import com.oceanbase.odc.service.onlineschemachange.oms.enums.OmsProjectStatusEnum; import com.oceanbase.odc.service.onlineschemachange.oms.enums.OmsStepName; import com.oceanbase.odc.service.onlineschemachange.oms.openapi.OmsProjectOpenApiService; +import com.oceanbase.odc.service.onlineschemachange.oms.request.FullTransferConfig; +import com.oceanbase.odc.service.onlineschemachange.oms.request.IncrTransferConfig; import com.oceanbase.odc.service.onlineschemachange.oms.request.ListOmsProjectFullVerifyResultRequest; import com.oceanbase.odc.service.onlineschemachange.oms.request.OmsProjectControlRequest; +import com.oceanbase.odc.service.onlineschemachange.oms.request.UpdateProjectConfigRequest; import com.oceanbase.odc.service.onlineschemachange.oms.response.OmsProjectFullVerifyResultResponse; import com.oceanbase.odc.service.onlineschemachange.oms.response.OmsProjectProgressResponse; import com.oceanbase.odc.service.onlineschemachange.oms.response.OmsProjectStepVO; @@ -73,6 +79,7 @@ public void invoke(ValveContext valveContext) { log.debug("Start execute {}, schedule task id {}", getClass().getSimpleName(), scheduleTask.getId()); OnlineSchemaChangeScheduleTaskParameters taskParameter = context.getTaskParameter(); + OnlineSchemaChangeParameters inputParameters = context.getParameter(); OmsProjectControlRequest projectRequest = getProjectRequest(taskParameter); List projectSteps = projectOpenApiService.describeProjectSteps(projectRequest); @@ -95,6 +102,7 @@ public void invoke(ValveContext valveContext) { return null; }) .getCheckerResult(); + updateOmsProjectConfig(scheduleTask.getId(), taskParameter, inputParameters, progress.getStatus()); OnlineSchemaChangeScheduleTaskResult result = new OnlineSchemaChangeScheduleTaskResult(taskParameter); @@ -111,6 +119,75 @@ public void invoke(ValveContext valveContext) { context.getParameter().getSwapTableType(), scheduleTask); } + private void updateOmsProjectConfig(Long scheduleTaskId, OnlineSchemaChangeScheduleTaskParameters taskParameters, + OnlineSchemaChangeParameters inputParameters, OmsProjectStatusEnum omsProjectStatus) { + // if rate limiter parameters is changed, try to stop and restart project + if (Objects.equals(inputParameters.getRateLimitConfig(), taskParameters.getRateLimitConfig())) { + return; + } + log.info("Input rate limiter has changed, currentOmsProjectStatus={}, rateLimiterConfig={}, " + + "oldRateLimiterConfig={}.", + omsProjectStatus.name(), JsonUtils.toJson(taskParameters.getRateLimitConfig()), + JsonUtils.toJson(inputParameters.getRateLimitConfig())); + + if (omsProjectStatus == OmsProjectStatusEnum.RUNNING) { + OmsProjectControlRequest controlRequest = new OmsProjectControlRequest(); + controlRequest.setId(taskParameters.getOmsProjectId()); + controlRequest.setUid(taskParameters.getUid()); + log.info("Try to stop oms project, omsProjectId={}, scheduleTaskId={}.", + taskParameters.getOmsProjectId(), scheduleTaskId); + try { + projectOpenApiService.stopProject(controlRequest); + log.info("Stop oms project completed, omsProjectId={}, scheduleTaskId={}.", + taskParameters.getOmsProjectId(), scheduleTaskId); + } catch (Exception e) { + log.warn("Stop oms project failed, omsProjectId={}, scheduleTaskId={}.", + taskParameters.getOmsProjectId(), scheduleTaskId, e); + } + + } else if (omsProjectStatus == OmsProjectStatusEnum.SUSPEND) { + try { + doUpdateOmsProjectConfig(scheduleTaskId, taskParameters, inputParameters); + } catch (Exception e) { + log.warn("Update oms project config failed, omsProjectId={}, scheduleTaskId={}.", + taskParameters.getOmsProjectId(), scheduleTaskId, e); + } + } + } + + private void doUpdateOmsProjectConfig(Long scheduleTaskId, OnlineSchemaChangeScheduleTaskParameters taskParameters, + OnlineSchemaChangeParameters oscParameters) { + OmsProjectControlRequest controlRequest = new OmsProjectControlRequest(); + controlRequest.setId(taskParameters.getOmsProjectId()); + controlRequest.setUid(taskParameters.getUid()); + UpdateProjectConfigRequest request = new UpdateProjectConfigRequest(); + request.setId(taskParameters.getOmsProjectId()); + FullTransferConfig fullTransferConfig = new FullTransferConfig(); + IncrTransferConfig incrTransferConfig = new IncrTransferConfig(); + fullTransferConfig.setThrottleIOPS(oscParameters.getRateLimitConfig().getDataSizeLimit()); + incrTransferConfig.setThrottleIOPS(oscParameters.getRateLimitConfig().getDataSizeLimit()); + fullTransferConfig.setThrottleRps(oscParameters.getRateLimitConfig().getRowLimit()); + incrTransferConfig.setThrottleRps(oscParameters.getRateLimitConfig().getRowLimit()); + request.setFullTransferConfig(fullTransferConfig); + request.setIncrTransferConfig(incrTransferConfig); + + log.info("Try to update oms project, omsProjectId={}, scheduleTaskId={}," + + " request={}.", taskParameters.getOmsProjectId(), scheduleTaskId, JsonUtils.toJson(request)); + projectOpenApiService.updateProjectConfig(request); + log.info("Update oms project completed, Try to resume project, omsProjectId={}," + + " scheduleTaskId={}", taskParameters.getOmsProjectId(), scheduleTaskId); + + projectOpenApiService.resumeProject(controlRequest); + log.info("Resume oms project completed, omsProjectId={}, scheduleTaskId={}", + taskParameters.getOmsProjectId(), scheduleTaskId); + // update task parameters rate limit same as schedule + taskParameters.setRateLimitConfig(oscParameters.getRateLimitConfig()); + int rows = scheduleTaskRepository.updateTaskParameters(scheduleTaskId, JsonUtils.toJson(taskParameters)); + if (rows > 0) { + log.info("Update throttle completed, scheduleTaskId={}", scheduleTaskId); + } + } + private void handleOmsProjectStepResult(ValveContext valveContext, ProjectStepResult projectStepResult, OnlineSchemaChangeScheduleTaskResult result, SwapTableType swapTableType, ScheduleTaskEntity scheduleTask) { From ecaacad339946e862b1e355be31ae9b877aca968 Mon Sep 17 00:00:00 2001 From: IL MARE Date: Thu, 13 Jun 2024 09:47:14 +0800 Subject: [PATCH 10/64] builds: upgrade version from 4.3.0 to 4.3.1 #2735 --- .gitmodules | 2 +- client | 2 +- distribution/odc-server-VER.txt | 2 +- libs/db-browser/pom.xml | 2 +- pom.xml | 4 ++-- server/3rd-party/Libinjection/pom.xml | 2 +- server/integration-test/pom.xml | 2 +- server/odc-common/pom.xml | 2 +- server/odc-core/pom.xml | 2 +- server/odc-migrate/pom.xml | 2 +- server/odc-server/pom.xml | 2 +- server/odc-service/pom.xml | 2 +- server/odc-test/pom.xml | 2 +- server/plugins/connect-plugin-api/pom.xml | 2 +- server/plugins/connect-plugin-doris/pom.xml | 2 +- server/plugins/connect-plugin-mysql/pom.xml | 2 +- server/plugins/connect-plugin-ob-mysql/pom.xml | 2 +- server/plugins/connect-plugin-ob-oracle/pom.xml | 2 +- server/plugins/connect-plugin-oracle/pom.xml | 2 +- server/plugins/pom.xml | 2 +- server/plugins/sample-plugin-api/pom.xml | 2 +- server/plugins/sample-plugin/pom.xml | 2 +- server/plugins/schema-plugin-api/pom.xml | 2 +- server/plugins/schema-plugin-doris/pom.xml | 2 +- server/plugins/schema-plugin-mysql/pom.xml | 2 +- server/plugins/schema-plugin-ob-mysql/pom.xml | 2 +- server/plugins/schema-plugin-ob-oracle/pom.xml | 2 +- server/plugins/schema-plugin-odp-sharding-ob-mysql/pom.xml | 2 +- server/plugins/schema-plugin-oracle/pom.xml | 2 +- server/plugins/task-plugin-api/pom.xml | 2 +- server/plugins/task-plugin-doris/pom.xml | 2 +- server/plugins/task-plugin-mysql/pom.xml | 2 +- server/plugins/task-plugin-ob-mysql/pom.xml | 2 +- server/plugins/task-plugin-ob-oracle/pom.xml | 2 +- server/plugins/task-plugin-oracle/pom.xml | 2 +- server/starters/desktop-starter/pom.xml | 2 +- server/starters/pom.xml | 2 +- server/starters/web-starter/pom.xml | 2 +- 38 files changed, 39 insertions(+), 39 deletions(-) diff --git a/.gitmodules b/.gitmodules index 533eb90c06..3274a3ccb2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "client"] path = client url = https://github.com/oceanbase/odc-client.git - branch = dev-4.3.0 + branch = dev-4.3.1 [submodule "build-resource"] path = build-resource url = https://github.com/oceanbase/odc-build-resource.git diff --git a/client b/client index 3a8a151f6a..1c8484f58e 160000 --- a/client +++ b/client @@ -1 +1 @@ -Subproject commit 3a8a151f6a37c063bb12a03baa913969026fe71a +Subproject commit 1c8484f58e08b2dec5dc377ff7af249185104580 diff --git a/distribution/odc-server-VER.txt b/distribution/odc-server-VER.txt index 80895903a1..f77856a6f1 100644 --- a/distribution/odc-server-VER.txt +++ b/distribution/odc-server-VER.txt @@ -1 +1 @@ -4.3.0 +4.3.1 diff --git a/libs/db-browser/pom.xml b/libs/db-browser/pom.xml index f87790ad24..be783dec24 100644 --- a/libs/db-browser/pom.xml +++ b/libs/db-browser/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.oceanbase db-browser - 1.1.3 + 1.1.4 db-browser https://github.com/oceanbase/odc/tree/main/libs/db-browser diff --git a/pom.xml b/pom.xml index e61b6f2e32..a21ba229d5 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ pom OceanBase Developer Center https://github.com/oceanbase/odc - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT server/3rd-party/Libinjection server/odc-test @@ -93,7 +93,7 @@ 4.20.19.ALL 4.10.0 2.10.0 - 1.1.3 + 1.1.4 1.3.0 3.10.0 1.64 diff --git a/server/3rd-party/Libinjection/pom.xml b/server/3rd-party/Libinjection/pom.xml index d2e2b6195b..876769c4de 100644 --- a/server/3rd-party/Libinjection/pom.xml +++ b/server/3rd-party/Libinjection/pom.xml @@ -8,7 +8,7 @@ com.oceanbase odc-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../../../pom.xml Libinjection diff --git a/server/integration-test/pom.xml b/server/integration-test/pom.xml index 37591a8aff..2ab6717c83 100644 --- a/server/integration-test/pom.xml +++ b/server/integration-test/pom.xml @@ -7,7 +7,7 @@ com.oceanbase odc-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../../pom.xml integration-test diff --git a/server/odc-common/pom.xml b/server/odc-common/pom.xml index f8cc24e37c..98f576e62c 100644 --- a/server/odc-common/pom.xml +++ b/server/odc-common/pom.xml @@ -7,7 +7,7 @@ com.oceanbase odc-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../../pom.xml odc-common diff --git a/server/odc-core/pom.xml b/server/odc-core/pom.xml index c6a8db957a..56b4844741 100644 --- a/server/odc-core/pom.xml +++ b/server/odc-core/pom.xml @@ -5,7 +5,7 @@ com.oceanbase odc-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../../pom.xml odc-core diff --git a/server/odc-migrate/pom.xml b/server/odc-migrate/pom.xml index 4fc6d9c78b..a9baf734fb 100644 --- a/server/odc-migrate/pom.xml +++ b/server/odc-migrate/pom.xml @@ -5,7 +5,7 @@ com.oceanbase odc-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../../pom.xml odc-migrate diff --git a/server/odc-server/pom.xml b/server/odc-server/pom.xml index 64ed093616..029a313b9c 100644 --- a/server/odc-server/pom.xml +++ b/server/odc-server/pom.xml @@ -6,7 +6,7 @@ com.oceanbase odc-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../../pom.xml odc-server diff --git a/server/odc-service/pom.xml b/server/odc-service/pom.xml index 4d3e30c67b..b94a963a88 100644 --- a/server/odc-service/pom.xml +++ b/server/odc-service/pom.xml @@ -5,7 +5,7 @@ com.oceanbase odc-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../../pom.xml odc-service diff --git a/server/odc-test/pom.xml b/server/odc-test/pom.xml index 4156ee5315..e83ea9fd09 100644 --- a/server/odc-test/pom.xml +++ b/server/odc-test/pom.xml @@ -7,7 +7,7 @@ com.oceanbase odc-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../../pom.xml odc-test diff --git a/server/plugins/connect-plugin-api/pom.xml b/server/plugins/connect-plugin-api/pom.xml index 2bbce789ca..2e600eb387 100644 --- a/server/plugins/connect-plugin-api/pom.xml +++ b/server/plugins/connect-plugin-api/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../pom.xml 4.0.0 diff --git a/server/plugins/connect-plugin-doris/pom.xml b/server/plugins/connect-plugin-doris/pom.xml index ea5a3b367c..38b9c6f6a7 100644 --- a/server/plugins/connect-plugin-doris/pom.xml +++ b/server/plugins/connect-plugin-doris/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../pom.xml diff --git a/server/plugins/connect-plugin-mysql/pom.xml b/server/plugins/connect-plugin-mysql/pom.xml index f373b84a41..b69725a632 100644 --- a/server/plugins/connect-plugin-mysql/pom.xml +++ b/server/plugins/connect-plugin-mysql/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../pom.xml diff --git a/server/plugins/connect-plugin-ob-mysql/pom.xml b/server/plugins/connect-plugin-ob-mysql/pom.xml index e13cd3493e..ea607712b2 100644 --- a/server/plugins/connect-plugin-ob-mysql/pom.xml +++ b/server/plugins/connect-plugin-ob-mysql/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../pom.xml 4.0.0 diff --git a/server/plugins/connect-plugin-ob-oracle/pom.xml b/server/plugins/connect-plugin-ob-oracle/pom.xml index 00de712b19..b5f8efd2d5 100644 --- a/server/plugins/connect-plugin-ob-oracle/pom.xml +++ b/server/plugins/connect-plugin-ob-oracle/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../pom.xml 4.0.0 diff --git a/server/plugins/connect-plugin-oracle/pom.xml b/server/plugins/connect-plugin-oracle/pom.xml index cbfc70c06f..085a892975 100644 --- a/server/plugins/connect-plugin-oracle/pom.xml +++ b/server/plugins/connect-plugin-oracle/pom.xml @@ -21,7 +21,7 @@ plugin-parent com.oceanbase - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../pom.xml 4.0.0 diff --git a/server/plugins/pom.xml b/server/plugins/pom.xml index 3dc59fa74e..5c0bc930c3 100644 --- a/server/plugins/pom.xml +++ b/server/plugins/pom.xml @@ -5,7 +5,7 @@ com.oceanbase odc-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../../pom.xml pom diff --git a/server/plugins/sample-plugin-api/pom.xml b/server/plugins/sample-plugin-api/pom.xml index 70d6390329..a6f2b0a3cb 100644 --- a/server/plugins/sample-plugin-api/pom.xml +++ b/server/plugins/sample-plugin-api/pom.xml @@ -5,7 +5,7 @@ com.oceanbase plugin-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../pom.xml sample-plugin-api diff --git a/server/plugins/sample-plugin/pom.xml b/server/plugins/sample-plugin/pom.xml index d1a90ed585..84fe81c0c5 100644 --- a/server/plugins/sample-plugin/pom.xml +++ b/server/plugins/sample-plugin/pom.xml @@ -5,7 +5,7 @@ com.oceanbase plugin-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../pom.xml sample-plugin diff --git a/server/plugins/schema-plugin-api/pom.xml b/server/plugins/schema-plugin-api/pom.xml index e6eae2e2b2..0596c22a41 100644 --- a/server/plugins/schema-plugin-api/pom.xml +++ b/server/plugins/schema-plugin-api/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../pom.xml diff --git a/server/plugins/schema-plugin-doris/pom.xml b/server/plugins/schema-plugin-doris/pom.xml index fc9621794e..04da7c864f 100644 --- a/server/plugins/schema-plugin-doris/pom.xml +++ b/server/plugins/schema-plugin-doris/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../pom.xml diff --git a/server/plugins/schema-plugin-mysql/pom.xml b/server/plugins/schema-plugin-mysql/pom.xml index fd8775155c..510e223036 100644 --- a/server/plugins/schema-plugin-mysql/pom.xml +++ b/server/plugins/schema-plugin-mysql/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../pom.xml diff --git a/server/plugins/schema-plugin-ob-mysql/pom.xml b/server/plugins/schema-plugin-ob-mysql/pom.xml index 63c5352fc3..8922b0678c 100644 --- a/server/plugins/schema-plugin-ob-mysql/pom.xml +++ b/server/plugins/schema-plugin-ob-mysql/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../pom.xml schema-plugin-ob-mysql diff --git a/server/plugins/schema-plugin-ob-oracle/pom.xml b/server/plugins/schema-plugin-ob-oracle/pom.xml index ce8c65b4b7..d629216fba 100644 --- a/server/plugins/schema-plugin-ob-oracle/pom.xml +++ b/server/plugins/schema-plugin-ob-oracle/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../pom.xml diff --git a/server/plugins/schema-plugin-odp-sharding-ob-mysql/pom.xml b/server/plugins/schema-plugin-odp-sharding-ob-mysql/pom.xml index 475a3b63f8..59e608f0bb 100644 --- a/server/plugins/schema-plugin-odp-sharding-ob-mysql/pom.xml +++ b/server/plugins/schema-plugin-odp-sharding-ob-mysql/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../pom.xml diff --git a/server/plugins/schema-plugin-oracle/pom.xml b/server/plugins/schema-plugin-oracle/pom.xml index 327dc2f94a..447a17b775 100644 --- a/server/plugins/schema-plugin-oracle/pom.xml +++ b/server/plugins/schema-plugin-oracle/pom.xml @@ -23,7 +23,7 @@ com.oceanbase plugin-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../pom.xml diff --git a/server/plugins/task-plugin-api/pom.xml b/server/plugins/task-plugin-api/pom.xml index 05677d862f..83fc38bd70 100644 --- a/server/plugins/task-plugin-api/pom.xml +++ b/server/plugins/task-plugin-api/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../pom.xml 4.0.0 diff --git a/server/plugins/task-plugin-doris/pom.xml b/server/plugins/task-plugin-doris/pom.xml index c43b8e2215..44247d9f02 100644 --- a/server/plugins/task-plugin-doris/pom.xml +++ b/server/plugins/task-plugin-doris/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT 4.0.0 diff --git a/server/plugins/task-plugin-mysql/pom.xml b/server/plugins/task-plugin-mysql/pom.xml index 59788080b4..22e9344ad5 100644 --- a/server/plugins/task-plugin-mysql/pom.xml +++ b/server/plugins/task-plugin-mysql/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT 4.0.0 diff --git a/server/plugins/task-plugin-ob-mysql/pom.xml b/server/plugins/task-plugin-ob-mysql/pom.xml index 187c78ac78..5955d81011 100644 --- a/server/plugins/task-plugin-ob-mysql/pom.xml +++ b/server/plugins/task-plugin-ob-mysql/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT 4.0.0 diff --git a/server/plugins/task-plugin-ob-oracle/pom.xml b/server/plugins/task-plugin-ob-oracle/pom.xml index aac33b3ace..ba4ec53391 100644 --- a/server/plugins/task-plugin-ob-oracle/pom.xml +++ b/server/plugins/task-plugin-ob-oracle/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT 4.0.0 diff --git a/server/plugins/task-plugin-oracle/pom.xml b/server/plugins/task-plugin-oracle/pom.xml index d9ca24bb96..053d5eacb4 100644 --- a/server/plugins/task-plugin-oracle/pom.xml +++ b/server/plugins/task-plugin-oracle/pom.xml @@ -21,7 +21,7 @@ plugin-parent com.oceanbase - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT 4.0.0 diff --git a/server/starters/desktop-starter/pom.xml b/server/starters/desktop-starter/pom.xml index 643211729e..e00ddf1374 100644 --- a/server/starters/desktop-starter/pom.xml +++ b/server/starters/desktop-starter/pom.xml @@ -5,7 +5,7 @@ starter-parent com.oceanbase - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../pom.xml desktop-starter diff --git a/server/starters/pom.xml b/server/starters/pom.xml index b91db13a37..f86157eefe 100644 --- a/server/starters/pom.xml +++ b/server/starters/pom.xml @@ -5,7 +5,7 @@ com.oceanbase odc-parent - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../../pom.xml pom diff --git a/server/starters/web-starter/pom.xml b/server/starters/web-starter/pom.xml index 7a5ec565a0..a305357143 100644 --- a/server/starters/web-starter/pom.xml +++ b/server/starters/web-starter/pom.xml @@ -5,7 +5,7 @@ starter-parent com.oceanbase - 4.3.0-SNAPSHOT + 4.3.1-SNAPSHOT ../pom.xml web-starter From 088c0fc250d8c27cf153eef7194438d1dfb1206d Mon Sep 17 00:00:00 2001 From: niyuhang <32453647+Niyuhang2@users.noreply.github.com> Date: Thu, 13 Jun 2024 16:31:43 +0800 Subject: [PATCH 11/64] feat(CI): support run build release by ob farm (#2738) --- .github/obfarm/action.yaml | 9 +- .github/obfarm/obfarm.py | 10 +- .github/workflows/build_release.yaml | 207 +++++---------------------- 3 files changed, 49 insertions(+), 177 deletions(-) diff --git a/.github/obfarm/action.yaml b/.github/obfarm/action.yaml index ef699cc8fb..30b3013206 100644 --- a/.github/obfarm/action.yaml +++ b/.github/obfarm/action.yaml @@ -11,7 +11,11 @@ inputs: timeout: description: 'timeout' required: false - default: '3600' + default: '4800' + jobname: + description: 'jobname' + required: false + default: 'farm' outputs: success: description: 'the status for the task' @@ -22,5 +26,4 @@ runs: - ${{ inputs.pipeline_id }} - ${{ inputs.project }} - ${{ inputs.timeout }} - - + - ${{ inputs.jobname }} \ No newline at end of file diff --git a/.github/obfarm/obfarm.py b/.github/obfarm/obfarm.py index 836625f42c..269e1c9dc4 100644 --- a/.github/obfarm/obfarm.py +++ b/.github/obfarm/obfarm.py @@ -202,9 +202,13 @@ def get_task_stage_output(oss_proxy: OssProxy, github_pipeline_id, start): return "" -def main(pipeline_id, project, timeout): +def main(pipeline_id, project, timeout, jobname): print("create a new task") print("working....") + global RESULT_FILE_KEY + RESULT_FILE_KEY = "{}/results/".format(jobname) + global TASK_QUEUE_FILE_KEY + TASK_QUEUE_FILE_KEY = "{}/jobs/{{}}.json".format(jobname) oss_proxy = OssProxy("https://farm-ce.oss-cn-heyuan.aliyuncs.com") github_proxy = GithubProxy() job_info = github_proxy.get_job_by_id(project, pipeline_id) @@ -226,8 +230,8 @@ def set_output(output): if __name__ == "__main__": print(sys.argv) - if len(sys.argv) < 4: + if len(sys.argv) < 5: print("缺失相关参数") OUTPUT.update({"success": -1}) sys.exit(1) - main(sys.argv[1], sys.argv[2], sys.argv[3]) + main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]) \ No newline at end of file diff --git a/.github/workflows/build_release.yaml b/.github/workflows/build_release.yaml index ba8b20b2bf..4311b62953 100644 --- a/.github/workflows/build_release.yaml +++ b/.github/workflows/build_release.yaml @@ -122,7 +122,7 @@ jobs: echo "::notice ::BUILD DOCKER IMAGE DOWNLOAD URL https://farm-use-for-odc.obs.cn-east-3.myhuaweicloud.com/odc-image/$GITHUB_RUN_ID/odc-${{ env.odc_docker_image_tag }}-arm64.tar.gz" build-web-x86_64: - name: Build Web Artifact (x86_64) + name: Build Web Artifact Release(AMD64) needs: [ calculate-version ] runs-on: ubuntu-latest env: @@ -131,184 +131,49 @@ jobs: steps: - name: Checkout workspace uses: actions/checkout@v4 + - name: action by obfarm++odc_build_amd++BUILD_DOCKER_IMAGE=1;BUILD_RPM=1;BUILD_RPM_VERSION=${{ inputs.rpm_release }};BUILD_DOCKER_TAG=${{ env.odc_docker_image_tag }} + uses: ./.github/obfarm/ + id: build-web-arm64 with: - submodules: "recursive" - - name: Setup JDK 8 - uses: actions/setup-java@v4 - with: - java-version: "8" - distribution: "temurin" - cache: maven - - name: Setup node 16 - uses: actions/setup-node@v4 - with: - node-version: "16" - - name: Build front static resources - run: | - echo "Current directory: "`pwd` - echo "Start build front static resources" - pushd client - echo "Run npm install pnpm -g" - npm install pnpm@8 -g - echo "Run pnpm install" - pnpm install - echo "Run npm run build:odc" - npm run build:odc - popd - echo "Build front static resources success" - echo "Start copy resources files" - static_resources_path="server/odc-server/src/main/resources/static" - if [ ! -d "${static_resources_path}" ]; then - echo "mkdir -p ${static_resources_path}" - mkdir -p "${static_resources_path}" - fi - rm --force --recursive --verbose ${static_resources_path}/* - cp --force --recursive --verbose client/dist/renderer/* ${static_resources_path} - echo "Copy resources files success" - - name: Set release version - id: set_release_version - run: | - main_version="$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout | cut -d - -f 1)" - new_version="${main_version}-${odc_rpm_release_number}" - echo "new_version=${new_version}" >> $GITHUB_OUTPUT - echo "RPM's version is "${new_version} - mvn versions:set -DnewVersion="${new_version}" - mvn versions:commit - - name: Build jar & rpm (x86_64) - run: | - echo "Start prepare oceanbase-client" - pushd import - echo "Current dir is "`pwd` - cp ../build-resource/obclient/2.2.4/linux_x86/obclient.tar.gz obclient.tar.gz - popd - echo "Prepare oceanbase-client success" - echo "Start build rpm package" - mvn help:system - mvn clean install -Dmaven.test.skip=true - mvn --file server/odc-server/pom.xml rpm:rpm -Drpm.prefix=/opt - echo "Build rpm package success" - rm --force --recursive --verbose distribution/docker/resources/odc-*.rpm - mkdir -p distribution/docker/resources/ - mv --verbose server/odc-server/target/rpm/odc-server/RPMS/*/odc-*.rpm distribution/docker/resources/ - mv --verbose server/odc-server/target/*-executable.jar distribution/jar/odc.jar - cp -fv distribution/jar/odc.jar distribution/jar/odc-slim.jar - zip -d distribution/jar/odc-slim.jar "BOOT-INF/classes/static/*" - - name: Upload jar - uses: actions/upload-artifact@v4 - with: - name: odc-artifact-jar - path: | - distribution/plugins/*.jar - distribution/starters/*.jar - distribution/jar/*.jar - - name: Upload rpm (x86_64) - uses: actions/upload-artifact@v4 - with: - name: odc-server-${{ steps.set_release_version.outputs.new_version }}.x86_64.rpm - path: distribution/docker/resources/odc-*.rpm - - name: Build docker image (x86_64) - run: | - sed -e "s/DATE_CHANGE/$(date)/" -i distribution/docker/odc/Dockerfile - echo "odc_docker_image_tag=${odc_docker_image_tag}" - pushd distribution/docker - docker build -t docker.io/oceanbase/odc:${odc_docker_image_tag} -f odc/Dockerfile . - docker save -o resources/odc-${odc_docker_image_tag}.tar.gz docker.io/oceanbase/odc:${odc_docker_image_tag} - popd - - name: Upload docker image (x86_64) - uses: actions/upload-artifact@v4 + pipeline_id: ${{ github.run_id }} + project: ${{ github.repository }} + jobname: odc_build_amd + + build-web-arm: + name: Build Web Artifact Release(ARM64) + needs: [ calculate-version ] + runs-on: ubuntu-latest + env: + odc_rpm_release_number: ${{ needs.calculate-version.outputs.odc_rpm_release_number }} + odc_docker_image_tag: ${{ needs.calculate-version.outputs.odc_docker_image_tag }} + steps: + - name: Checkout workspace + uses: actions/checkout@v4 + - name: action by obfarm++odc_build_arm++BUILD_DOCKER_IMAGE=1;BUILD_RPM=1;BUILD_RPM_VERSION=${{ inputs.rpm_release }};BUILD_DOCKER_TAG=${{ env.odc_docker_image_tag }} + uses: ./.github/obfarm/ + id: build-web-arm64 with: - name: odc-${{ env.odc_docker_image_tag }}.tar.gz - path: distribution/docker/resources/odc-*.tar.gz + pipeline_id: ${{ github.run_id }} + project: ${{ github.repository }} + jobname: odc_build_arm build-client: name: Build Client Artifact - needs: [ build-web-x86_64 ] - runs-on: macos-latest - strategy: - fail-fast: false - matrix: - target: [ win, mac, linux_x86, linux_aarch64 ] + needs: [ calculate-version ] + runs-on: ubuntu-latest + env: + odc_rpm_release_number: ${{ needs.calculate-version.outputs.odc_rpm_release_number }} + odc_docker_image_tag: ${{ needs.calculate-version.outputs.odc_docker_image_tag }} steps: - name: Checkout workspace uses: actions/checkout@v4 + - name: action by obfarm++odc_build_client + uses: ./.github/obfarm/ + id: build-client with: - submodules: "recursive" - - name: Download resources - uses: actions/download-artifact@v4 - with: - name: odc-artifact-jar - path: jar-dist - - name: Change directory - run: | - mkdir -p client/libraries/java - cp jar-dist/jar/odc-slim.jar client/libraries/java/odc.jar - mkdir -p client/libraries/java/plugins - cp -R jar-dist/plugins/. client/libraries/java/plugins - mkdir -p client/libraries/java/starters - cp -R jar-dist/starters/. client/libraries/java/starters - - name: Setup node 16 - uses: actions/setup-node@v4 - with: - node-version: "16" - - name: Install dependencies - uses: pnpm/action-setup@v4 - with: - version: 8 - run_install: false - - name: Get pnpm store directory - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - uses: actions/cache@v4 - name: Setup pnpm cache - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - name: Build artifact - env: - # MACOS_CERTIFICATE: ${{ secrets.PROD_MACOS_CERTIFICATE }} - # MACOS_CERTIFICATE_PWD: ${{ secrets.PROD_MACOS_CERTIFICATE_PWD }} - # MACOS_CERTIFICATE_NAME: ${{ secrets.PROD_MACOS_CERTIFICATE_NAME }} - # MACOS_CI_KEYCHAIN_PWD: ${{ secrets.PROD_MACOS_CI_KEYCHAIN_PWD }} - # APPLE_ID: ${{ secrets.PROD_MACOS_NOTARIZATION_APPLE_ID }} - # APPLE_TEAM_ID: ${{ secrets.PROD_MACOS_NOTARIZATION_TEAM_ID }} - # APPLE_ID_PASSWORD: ${{ secrets.PROD_MACOS_NOTARIZATION_PWD }} - # APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.PROD_MACOS_NOTARIZATION_PWD }} - WIN_CSC_LINK: ${{ secrets.PROD_WIN_CSC_LINK }} - WIN_CSC_KEY_PASSWORD: ${{ secrets.PROD_WIN_CSC_KEY_PASSWORD }} - run: | - # Turn our base64-encoded certificate back to a regular .p12 file - cd client - # echo $MACOS_CERTIFICATE | base64 --decode > certificate.p12 - # We need to create a new keychain, otherwise using the certificate will prompt - # with a UI dialog asking for the certificate password, which we can't - # use in a headless CI environment - # security create-keychain -p "$MACOS_CI_KEYCHAIN_PWD" build.keychain - # security default-keychain -s build.keychain - # security unlock-keychain -p "$MACOS_CI_KEYCHAIN_PWD" build.keychain - # security import certificate.p12 -k build.keychain -P "$MACOS_CERTIFICATE_PWD" -T /usr/bin/codesign - # security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$MACOS_CI_KEYCHAIN_PWD" build.keychain - # echo "Cert Imported" - echo "node-linker=hoisted" >> .npmrc - pnpm install - # We need to create a new keychain, otherwise using the certificate will prompt - # with a UI dialog asking for the certificate password, which we can't - # use in a headless CI environment - export ELECTRON_MIRROR=https://npmmirror.com/mirrors/electron/ - export ODC_BUILD_SKIP_JAR=true - export CSC_IDENTITY_AUTO_DISCOVERY=false - node ./scripts/client/build.js ${{ matrix.target }} - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: odc-client-pkg-${{ matrix.target }} - path: | - client/release/*.dmg - client/release/*.deb - client/release/*.exe - client/release/*.AppImage + pipeline_id: ${{ github.run_id }} + project: ${{ github.repository }} + jobname: odc_build_client release: name: Release (Skip for now) @@ -324,4 +189,4 @@ jobs: runs-on: ubuntu-latest steps: - name: Tag release - run: echo "Skip for now 🤪" + run: echo "Skip for now 🤪" \ No newline at end of file From d7fb046e8ce07cf060953f7ff452a8a758237e54 Mon Sep 17 00:00:00 2001 From: LuckyLeo Date: Mon, 17 Jun 2024 16:42:37 +0800 Subject: [PATCH 12/64] feat(sql-execute): supports service layer for query profile (#2423) * add service for query profile * fix NPE * rename QueryProfileManager.java * support parse ob sql plan * support parse logic plan into graph * add warning as error message * fix UT * support set sqlId * support distributed plan * remove i18n * fix an error * fix audit missing * fix UT * fix UT * response to CR comments * format code * remove const * add exception catch * remove import * add comments * fix cast parameter failed --- client | 2 +- .../session/ConnectConsoleServiceTest.java | 37 +- .../odc/common/util/TimespanFormatUtil.java | 85 +++++ .../common/util/TimespanFormatUtilTest.java | 77 ++++ .../odc/core/shared/constant/ErrorCodes.java | 1 + .../odc/core/shared}/model/OBSqlPlan.java | 5 +- .../odc/core/shared}/model/Operator.java | 11 +- .../odc/core/shared/model/PlanNode.java | 2 + .../odc/core/shared}/model/PredicateKey.java | 7 +- .../odc/core/shared/model/QueryStatus.java} | 26 +- .../core/shared}/model/SqlPlanMonitor.java | 18 +- .../oceanbase/odc/core/sql/util/OBUtils.java | 127 ++++++- .../i18n/BusinessMessages.properties | 1 - .../i18n/BusinessMessages_zh_CN.properties | 1 - .../resources/i18n/ErrorMessages.properties | 1 + .../i18n/ErrorMessages_zh_CN.properties | 1 + .../i18n/ErrorMessages_zh_TW.properties | 1 + .../init-config/default-audit-event-meta.yml | 2 +- .../controller/v1/SqlDiagnoseController.java | 14 +- .../v2/ConnectSessionController.java | 31 -- .../odc/config/ScheduleConfiguration.java | 15 + .../odc/service/audit/AuditEventAspect.java | 17 +- .../service/diagnose/SqlDiagnoseService.java | 9 +- .../queryprofile/OBQueryProfileManager.java | 108 ++++++ .../queryprofile/display/PlanGraph.java | 34 -- .../display/PlanGraphOperator.java | 37 -- .../queryprofile/model/SqlPlanGraph.java | 59 --- .../queryprofile/model/SqlProfile.java | 51 --- .../session/ConnectConsoleService.java | 165 ++------ .../service/session/OBExecutionListener.java | 28 +- .../service/session/OdcStatementCallBack.java | 1 + .../session/model/AsyncExecuteContext.java | 7 +- .../session/model/AsyncExecuteResultResp.java | 2 + .../service/session/util/SqlRewriteUtil.java | 1 - .../api/SqlDiagnoseExtensionPoint.java | 5 +- .../connect/model/diagnose/PlanGraph.java | 100 +++++ .../model/diagnose}/PlanGraphEdge.java | 2 +- .../model/diagnose/PlanGraphOperator.java | 95 +++++ .../connect/model/diagnose}/SqlExplain.java | 3 +- .../doris/DorisDiagnoseExtensionPoint.java | 3 +- .../connect/doris/DorisExtensionTest.java | 2 +- .../mysql/MySQLDiagnoseExtensionPoint.java | 8 +- .../connect/mysql/MySQLExtensionTest.java | 2 +- .../obmysql/OBMySQLConnectionExtension.java | 5 +- .../obmysql/OBMySQLDiagnoseExtension.java | 167 +++++++- .../obmysql/diagnose/DiagnoseUtil.java | 19 +- .../obmysql/diagnose}/PlanGraphBuilder.java | 111 ++++-- .../obmysql/diagnose}/PlanGraphMapper.java | 24 +- .../diagnose}/PlanParameterSubstitutor.java | 2 +- .../obmysql/diagnose/ProfileConstants.java | 44 +++ .../obmysql/diagnose/QueryProfileHelper.java | 358 ++++++++++++++++++ .../EnablePlanMonitorInitializer.java | 46 +++ .../connect/obmysql/OBMySQLExtensionTest.java | 2 +- .../queryprofile/PlanGraphBuilderTest.java | 56 ++- .../PlanParameterSubstitutorTest.java | 4 +- .../oboracle/OBOracleConnectionExtension.java | 2 + .../oboracle/OBOracleDiagnoseExtension.java | 52 ++- .../oboracle/OBOracleExtensionTest.java | 2 +- 58 files changed, 1622 insertions(+), 476 deletions(-) create mode 100644 server/odc-common/src/main/java/com/oceanbase/odc/common/util/TimespanFormatUtil.java create mode 100644 server/odc-common/src/test/java/com/oceanbase/odc/common/util/TimespanFormatUtilTest.java rename server/{odc-service/src/main/java/com/oceanbase/odc/service/queryprofile => odc-core/src/main/java/com/oceanbase/odc/core/shared}/model/OBSqlPlan.java (89%) rename server/{odc-service/src/main/java/com/oceanbase/odc/service/queryprofile => odc-core/src/main/java/com/oceanbase/odc/core/shared}/model/Operator.java (80%) rename server/{odc-service/src/main/java/com/oceanbase/odc/service/queryprofile => odc-core/src/main/java/com/oceanbase/odc/core/shared}/model/PredicateKey.java (88%) rename server/{odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/display/SqlProfile.java => odc-core/src/main/java/com/oceanbase/odc/core/shared/model/QueryStatus.java} (58%) rename server/{odc-service/src/main/java/com/oceanbase/odc/service/queryprofile => odc-core/src/main/java/com/oceanbase/odc/core/shared}/model/SqlPlanMonitor.java (67%) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/OBQueryProfileManager.java delete mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/display/PlanGraph.java delete mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/display/PlanGraphOperator.java delete mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/model/SqlPlanGraph.java delete mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/model/SqlProfile.java create mode 100644 server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/model/diagnose/PlanGraph.java rename server/{odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/display => plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/model/diagnose}/PlanGraphEdge.java (93%) create mode 100644 server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/model/diagnose/PlanGraphOperator.java rename server/{odc-core/src/main/java/com/oceanbase/odc/core/shared/model => plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/model/diagnose}/SqlExplain.java (91%) rename server/{odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/helper => plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose}/PlanGraphBuilder.java (54%) rename server/{odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/helper => plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose}/PlanGraphMapper.java (68%) rename server/{odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/helper => plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose}/PlanParameterSubstitutor.java (96%) create mode 100644 server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/ProfileConstants.java create mode 100644 server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/QueryProfileHelper.java create mode 100644 server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/initializer/EnablePlanMonitorInitializer.java rename server/{odc-service/src/test/java/com/oceanbase/odc/service => plugins/connect-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/connect/obmysql}/queryprofile/PlanGraphBuilderTest.java (52%) rename server/{odc-service/src/test/java/com/oceanbase/odc/service => plugins/connect-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/connect/obmysql}/queryprofile/PlanParameterSubstitutorTest.java (91%) diff --git a/client b/client index 1c8484f58e..7508ae9f90 160000 --- a/client +++ b/client @@ -1 +1 @@ -Subproject commit 1c8484f58e08b2dec5dc377ff7af249185104580 +Subproject commit 7508ae9f90e71c2cb67d7a50de4898ef7d0204b4 diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/session/ConnectConsoleServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/session/ConnectConsoleServiceTest.java index f4d6283f31..d0ec0ce5db 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/session/ConnectConsoleServiceTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/session/ConnectConsoleServiceTest.java @@ -61,6 +61,7 @@ import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.db.session.DefaultDBSessionManage; import com.oceanbase.odc.service.dml.ValueEncodeType; +import com.oceanbase.odc.service.session.model.AsyncExecuteContext; import com.oceanbase.odc.service.session.model.BinaryContent; import com.oceanbase.odc.service.session.model.SqlAsyncExecuteReq; import com.oceanbase.odc.service.session.model.SqlAsyncExecuteResp; @@ -89,8 +90,9 @@ public class ConnectConsoleServiceTest extends ServiceTestEnv { public void getAsyncResult_killSessionSql_successResult() throws Exception { String sql = "kill session /*"; injectAsyncJdbcExecutor(JdbcGeneralResult.successResult(SqlTuple.newTuple(sql))); - SqlAsyncExecuteResp resp = consoleService.execute(sessionid, getSqlAsyncExecuteReq(sql)); - List resultList = consoleService.getAsyncResult(sessionid, resp.getRequestId()); + SqlAsyncExecuteResp resp = consoleService.streamExecute(sessionid, getSqlAsyncExecuteReq(sql)); + injectExecuteContext(sessionid, resp.getRequestId(), JdbcGeneralResult.successResult(SqlTuple.newTuple(sql))); + List resultList = consoleService.getMoreResults(sessionid, resp.getRequestId()).getResults(); Assert.assertFalse(resultList.isEmpty()); } @@ -109,8 +111,9 @@ public void getAsyncResult_wrongKillSessionSql_failedResult() throws Exception { String sql = "kill session /*"; JdbcGeneralResult failedResult = JdbcGeneralResult.failedResult(SqlTuple.newTuple(sql), new Exception("test")); injectAsyncJdbcExecutor(failedResult); - SqlAsyncExecuteResp resp = consoleService.execute(sessionid, getSqlAsyncExecuteReq(sql)); - List resultList = consoleService.getAsyncResult(sessionid, resp.getRequestId()); + SqlAsyncExecuteResp resp = consoleService.streamExecute(sessionid, getSqlAsyncExecuteReq(sql)); + injectExecuteContext(sessionid, resp.getRequestId(), failedResult); + List resultList = consoleService.getMoreResults(sessionid, resp.getRequestId()).getResults(); Assert.assertFalse(resultList.isEmpty()); Assert.assertSame(SqlExecuteStatus.FAILED, resultList.get(0).getStatus()); @@ -120,8 +123,8 @@ public void getAsyncResult_wrongKillSessionSql_failedResult() throws Exception { public void getAsyncResult_delimiter_getResultSucceed() throws Exception { String sql = "delimiter $$"; injectAsyncJdbcExecutor(JdbcGeneralResult.successResult(SqlTuple.newTuple(sql))); - SqlAsyncExecuteResp resp = consoleService.execute(sessionid, getSqlAsyncExecuteReq(sql)); - List resultList = consoleService.getAsyncResult(sessionid, resp.getRequestId()); + SqlAsyncExecuteResp resp = consoleService.streamExecute(sessionid, getSqlAsyncExecuteReq(sql)); + List resultList = consoleService.getMoreResults(sessionid, resp.getRequestId()).getResults(); Assert.assertFalse(resultList.isEmpty()); } @@ -130,8 +133,9 @@ public void getAsyncResult_delimiter_getResultSucceed() throws Exception { public void getAsyncResult_commonSQLForOracle_getResultSucceed() throws Exception { String sql = "select * from tableaas"; injectAsyncJdbcExecutor(JdbcGeneralResult.successResult(SqlTuple.newTuple(sql))); - SqlAsyncExecuteResp resp = consoleService.execute(sessionid, getSqlAsyncExecuteReq(sql)); - List resultList = consoleService.getAsyncResult(sessionid, resp.getRequestId()); + SqlAsyncExecuteResp resp = consoleService.streamExecute(sessionid, getSqlAsyncExecuteReq(sql)); + injectExecuteContext(sessionid, resp.getRequestId(), JdbcGeneralResult.successResult(SqlTuple.newTuple(sql))); + List resultList = consoleService.getMoreResults(sessionid, resp.getRequestId()).getResults(); Assert.assertFalse(resultList.isEmpty()); } @@ -141,8 +145,9 @@ public void getAsyncResult_commonSQLForMysql_getSucceed() throws Exception { String sql = "select * from tableaas"; injectAsyncJdbcExecutor(JdbcGeneralResult.successResult(SqlTuple.newTuple(sql)), ConnectType.OB_MYSQL); - SqlAsyncExecuteResp resp = consoleService.execute(sessionid, getSqlAsyncExecuteReq(sql)); - List resultList = consoleService.getAsyncResult(sessionid, resp.getRequestId()); + SqlAsyncExecuteResp resp = consoleService.streamExecute(sessionid, getSqlAsyncExecuteReq(sql)); + injectExecuteContext(sessionid, resp.getRequestId(), JdbcGeneralResult.successResult(SqlTuple.newTuple(sql))); + List resultList = consoleService.getMoreResults(sessionid, resp.getRequestId()).getResults(); Assert.assertFalse(resultList.isEmpty()); } @@ -152,8 +157,9 @@ public void generateResult_editableResultSet_isEditable() throws Exception { String sql = "select * from table_test"; JdbcGeneralResult executeResult = getJdbcGeneralResultWithQueryData(sql); injectAsyncJdbcExecutor(executeResult, ConnectType.OB_MYSQL); - SqlAsyncExecuteResp resp = consoleService.execute(sessionid, getSqlAsyncExecuteReq(sql)); - List resultList = consoleService.getAsyncResult(sessionid, resp.getRequestId()); + SqlAsyncExecuteResp resp = consoleService.streamExecute(sessionid, getSqlAsyncExecuteReq(sql)); + injectExecuteContext(sessionid, resp.getRequestId(), executeResult); + List resultList = consoleService.getMoreResults(sessionid, resp.getRequestId()).getResults(); Assert.assertTrue(resultList.get(0).getResultSetMetaData().isEditable()); } @@ -209,6 +215,13 @@ private ConnectionConfig buildTestConnection(ConnectType connectType) { return connection; } + private void injectExecuteContext(String sessionId, String requestId, JdbcGeneralResult result) { + ConnectionSession connectionSession = sessionService.nullSafeGet(sessionid); + AsyncExecuteContext context = + (AsyncExecuteContext) ConnectionSessionUtil.getExecuteContext(connectionSession, requestId); + context.addSqlExecutionResults(Collections.singletonList(result)); + } + private JdbcGeneralResult getJdbcGeneralResultWithQueryData(String sql) throws SQLException { JdbcGeneralResult executeResult = JdbcGeneralResult.successResult(SqlTuple.newTuple(sql)); diff --git a/server/odc-common/src/main/java/com/oceanbase/odc/common/util/TimespanFormatUtil.java b/server/odc-common/src/main/java/com/oceanbase/odc/common/util/TimespanFormatUtil.java new file mode 100644 index 0000000000..55583c500d --- /dev/null +++ b/server/odc-common/src/main/java/com/oceanbase/odc/common/util/TimespanFormatUtil.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.common.util; + +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +public class TimespanFormatUtil { + + public static String formatTimespan(Duration dValue) { + return formatTimespan(dValue, " "); + } + + public static String formatTimespan(Duration dValue, String delimiter) { + if (dValue == null) { + return "0"; + } + return formatTimespan(dValue.toNanos() / 1000, TimeUnit.MICROSECONDS, delimiter); + } + + public static String formatTimespan(long time, TimeUnit timeUnit) { + return formatTimespan(time, timeUnit, " "); + } + + /** + * 
+     * Examples:
+     *  - (555555, TimeUnit.MICROSECONDS, "") ==> returns "555.555ms"
+     *  - (60, TimeUnit.SECONDS, " ") ==> "1.000 min"
+     *  - (24, TimeUnit.HOURS, "-") ==> "1.000-d"
+     *
+ * + * @param delimiter The separator to use between the number and unit. + */ + public static String formatTimespan(long time, TimeUnit timeUnit, String delimiter) { + if (time == 0) { + return "0"; + } + if (delimiter.contains(".")) { + throw new IllegalArgumentException("delimiter should not contains '.'"); + } + TimespanUnit result = TimespanUnit.MICROSECONDS; + double converted = TimeUnit.MICROSECONDS.convert(time, timeUnit); + for (TimespanUnit unit : TimespanUnit.values()) { + result = unit; + long amount = unit.amount; + if (result == TimespanUnit.DAYS || converted < amount) { + break; + } + converted /= amount; + } + return String.format("%.2f%s%s", converted, delimiter, result.text); + } + + private enum TimespanUnit { + MICROSECONDS("us", 1000), + MILLISECONDS("ms", 1000), + SECONDS("s", 60), + MINUTES("min", 60), + HOURS("h", 24), + DAYS("d", 7); + + final String text; + final long amount; + + TimespanUnit(String unit, long amount) { + this.text = unit; + this.amount = amount; + } + } + +} diff --git a/server/odc-common/src/test/java/com/oceanbase/odc/common/util/TimespanFormatUtilTest.java b/server/odc-common/src/test/java/com/oceanbase/odc/common/util/TimespanFormatUtilTest.java new file mode 100644 index 0000000000..36e55ef294 --- /dev/null +++ b/server/odc-common/src/test/java/com/oceanbase/odc/common/util/TimespanFormatUtilTest.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.common.util; + +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalUnit; +import java.util.Arrays; +import java.util.Collection; +import java.util.concurrent.TimeUnit; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +/** + * @author: liuyizhuo.lyz + * @date: 2024/4/19 + */ +@RunWith(Parameterized.class) +public class TimespanFormatUtilTest { + + @Parameter(0) + public long time; + @Parameter(1) + public TimeUnit timeUnit; + @Parameter(2) + public TemporalUnit temporalUnit; + @Parameter(3) + public String separator; + @Parameter(4) + public String output; + + + @Parameters(name = "{index}: var[{0}]={1}") + public static Collection data() { + return Arrays.asList(new Object[][] { + {1000, TimeUnit.MICROSECONDS, ChronoUnit.MICROS, "", "1.00ms"}, + {1550000, TimeUnit.MICROSECONDS, ChronoUnit.MICROS, "", "1.55s"}, + {180, TimeUnit.SECONDS, ChronoUnit.SECONDS, "", "3.00min"}, + {5400, TimeUnit.SECONDS, ChronoUnit.SECONDS, "", "1.50h"}, + {86400, TimeUnit.SECONDS, ChronoUnit.SECONDS, " ", "1.00 d"}, + {123000, TimeUnit.MICROSECONDS, ChronoUnit.MICROS, " ", "123.00 ms"}, + {6780000, TimeUnit.MILLISECONDS, ChronoUnit.MILLIS, " ", "1.88 h"}, + {4000, TimeUnit.SECONDS, ChronoUnit.SECONDS, "-", "1.11-h"}, + {1440, TimeUnit.MINUTES, ChronoUnit.MINUTES, "-", "1.00-d"}, + {1000000, TimeUnit.MICROSECONDS, ChronoUnit.MICROS, "-", "1.00-s"} + }); + } + + @Test + public void testFormatTimespan_TimeUnit() { + Assert.assertEquals(output, TimespanFormatUtil.formatTimespan(time, timeUnit, separator)); + } + + @Test + public void testFormatTimespan_Duration() { + Assert.assertEquals(output, TimespanFormatUtil.formatTimespan(Duration.of(time, temporalUnit), separator)); + } + +} diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java index a0c9dc5699..9d226764e5 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java @@ -253,6 +253,7 @@ public enum ErrorCodes implements ErrorCode { ObGetFullLinkTraceFailed, ObFullLinkTraceNotSupported, ObFullLinkTraceNotEnabled, + ObQueryProfileNotSupported, ObPreCheckDdlFailed, ObCopySchemaFailed, diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/model/OBSqlPlan.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/OBSqlPlan.java similarity index 89% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/model/OBSqlPlan.java rename to server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/OBSqlPlan.java index caeeb81b5f..644a33aa2a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/model/OBSqlPlan.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/OBSqlPlan.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.queryprofile.model; +package com.oceanbase.odc.core.shared.model; import lombok.Data; @@ -26,6 +26,9 @@ public class OBSqlPlan { private String id; private String parentId; + private String depth; + private String cost; + private String cardinality; private String operator; private String objectOwner; private String objectName; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/model/Operator.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/Operator.java similarity index 80% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/model/Operator.java rename to server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/Operator.java index f3ca8a0ee9..e0061b8c7e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/model/Operator.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/Operator.java @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.queryprofile.model; +package com.oceanbase.odc.core.shared.model; +import java.util.LinkedHashMap; import java.util.Map; import com.oceanbase.odc.common.graph.GraphVertex; -import com.oceanbase.odc.service.queryprofile.model.SqlProfile.Status; import lombok.Getter; import lombok.NonNull; @@ -32,9 +32,10 @@ @Setter public class Operator extends GraphVertex { private String title; - private Status status; - private Map overview; - private Map statistics; + private QueryStatus status; + private Long duration; + private Map overview = new LinkedHashMap<>(); + private Map statistics = new LinkedHashMap<>(); public Operator(@NonNull String id, String name) { super(id, name); diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/PlanNode.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/PlanNode.java index c27de08616..d50d8cb32f 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/PlanNode.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/PlanNode.java @@ -33,6 +33,8 @@ public class PlanNode { private String operator; private String rowCount; private String cost; + private String realCost; + private String realRowCount; private int depth; private String outputFilter; private LinkedHashMap children = new LinkedHashMap<>(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/model/PredicateKey.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/PredicateKey.java similarity index 88% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/model/PredicateKey.java rename to server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/PredicateKey.java index d083ca301e..7a59dae9ac 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/model/PredicateKey.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/PredicateKey.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.queryprofile.model; +package com.oceanbase.odc.core.shared.model; import java.util.HashMap; import java.util.Map; @@ -29,6 +29,11 @@ public class PredicateKey { MAP.put("range", "Range"); MAP.put("range_cond", "Range condition"); MAP.put("partitions", "Scan partitions"); + MAP.put("is_index_back", "Index look up back"); + MAP.put("is_global_index", "Is global index"); + MAP.put("filter_before_indexback", "Filter before index look up"); + MAP.put("prefix_columns_cnt", "Prefix columns count"); + MAP.put("skip_scan_range", "Skip scan range"); // join MAP.put("nl_params_", "Nested loop params"); MAP.put("conds", "Join conditions"); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/display/SqlProfile.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/QueryStatus.java similarity index 58% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/display/SqlProfile.java rename to server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/QueryStatus.java index c7d74c2d95..d79708b1d2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/display/SqlProfile.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/QueryStatus.java @@ -13,24 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.queryprofile.display; +package com.oceanbase.odc.core.shared.model; -import java.util.Date; - -import com.oceanbase.odc.service.queryprofile.model.SqlProfile.Status; - -import lombok.Data; - -/** - * @author liuyizhuo.lyz - * @date 2024/4/11 - */ -@Data -public class SqlProfile { - private Date startTime; - private Date endTime; - private Status status; - private Long duration; - private String traceId; - private String sqlText; +public enum QueryStatus { + PREPARING, + RUNNING, + FINISHED, + FAILED, + ; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/model/SqlPlanMonitor.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/SqlPlanMonitor.java similarity index 67% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/model/SqlPlanMonitor.java rename to server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/SqlPlanMonitor.java index b7d381bd9c..79f826d18b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/model/SqlPlanMonitor.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/SqlPlanMonitor.java @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.queryprofile.model; +package com.oceanbase.odc.core.shared.model; -import java.util.Date; +import java.sql.Timestamp; +import java.util.Map; import lombok.Data; @@ -28,14 +29,19 @@ public class SqlPlanMonitor { private String svrIp; private String svrPort; - private Date firstRefreshTime; - private Date lastRefreshTime; - private Date firstChangeTime; - private Date lastChangeTime; + private String processName; + private Timestamp firstRefreshTime; + private Timestamp lastRefreshTime; + private Timestamp firstChangeTime; + private Timestamp lastChangeTime; + private Timestamp currentTime; private String planLineId; + private Long workareaMaxMem; + private Long workareaMaxTempSeg; private Long starts; private Long outputRows; private Long dbTime; private Long userIOWaitTime; + private Map otherstats; } diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/util/OBUtils.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/util/OBUtils.java index 286024fb23..f5941bc07f 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/util/OBUtils.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/util/OBUtils.java @@ -22,7 +22,9 @@ import java.sql.Statement; import java.sql.Types; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Validate; @@ -39,7 +41,9 @@ import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.exception.BadRequestException; import com.oceanbase.odc.core.shared.exception.UnexpectedException; +import com.oceanbase.odc.core.shared.model.OBSqlPlan; import com.oceanbase.odc.core.shared.model.SqlExecDetail; +import com.oceanbase.odc.core.shared.model.SqlPlanMonitor; import com.oceanbase.odc.core.sql.execute.SyncJdbcExecutor; import com.oceanbase.odc.core.sql.execute.model.SqlExecTime; import com.oceanbase.tools.dbbrowser.util.MySQLSqlBuilder; @@ -440,7 +444,8 @@ public static String queryTraceIdFromASH(@NonNull Statement statement, * OceanBase only supports ASH views in versions higher than 4.0. Therefore, this method is not * applicable to earlier versions, please use sql_audit instead. */ - public static String queryPlanIdByTraceId(@NonNull Statement statement, String traceId, ConnectType connectType) + public static String queryPlanIdByTraceIdFromASH(@NonNull Statement statement, String traceId, + ConnectType connectType) throws SQLException { DialectType dialectType = connectType.getDialectType(); SqlBuilder sqlBuilder = getBuilder(connectType) @@ -456,4 +461,124 @@ public static String queryPlanIdByTraceId(@NonNull Statement statement, String t } } + public static String queryPlanIdByTraceIdFromAudit(@NonNull Statement statement, String traceId, + ConnectType connectType) + throws SQLException { + DialectType dialectType = connectType.getDialectType(); + SqlBuilder sqlBuilder = getBuilder(connectType) + .append("select plan_id from ") + .append(dialectType.isMysql() ? "oceanbase" : "sys") + .append(".v$ob_sql_audit where trace_id=") + .value(traceId) + .append(" and is_inner_sql=0"); + try (ResultSet rs = statement.executeQuery(sqlBuilder.toString())) { + if (!rs.next()) { + throw new SQLException("No result found in sql_audit."); + } + return rs.getString(1); + } + } + + public static List queryOBSqlPlanByPlanId(@NonNull Statement statement, @NonNull String planId, + ConnectType connectType) throws SQLException { + DialectType dialectType = connectType.getDialectType(); + SqlBuilder sqlBuilder = getBuilder(connectType) + .append("select id, parent_id, operator, object_owner, object_name, object_alias, ") + .append("depth, cost, cardinality, ") + .append("other, access_predicates, filter_predicates, projection, special_predicates from ") + .append(dialectType.isMysql() ? "oceanbase" : "sys") + .append(".v$ob_sql_plan where plan_id=") + .value(planId) + .append(" order by id asc"); + List records = new ArrayList<>(); + try (ResultSet rs = statement.executeQuery(sqlBuilder.toString())) { + while (rs.next()) { + OBSqlPlan plan = new OBSqlPlan(); + plan.setId(rs.getString("id")); + plan.setParentId(rs.getString("parent_id")); + plan.setDepth(rs.getString("depth")); + plan.setCost(rs.getString("cost")); + plan.setCardinality(rs.getString("cardinality")); + plan.setOperator(rs.getString("operator")); + plan.setObjectOwner(rs.getString("object_owner")); + plan.setObjectName(rs.getString("object_name")); + plan.setObjectAlias(rs.getString("object_alias")); + plan.setOther(rs.getString("other")); + plan.setAccessPredicates(rs.getString("access_predicates")); + plan.setFilterPredicates(rs.getString("filter_predicates")); + plan.setProjection(rs.getString("projection")); + plan.setSpecialPredicates(rs.getString("special_predicates")); + records.add(plan); + } + } + Verify.notEmpty(records, "plan records"); + return records; + } + + public static List querySqlPlanMonitorStats(@NonNull Statement statement, @NonNull String traceId, + ConnectType connectType, Map statId2Name) throws SQLException { + DialectType dialectType = connectType.getDialectType(); + SqlBuilder sqlBuilder = getBuilder(connectType) + .append("select svr_ip,svr_port,first_refresh_time,last_refresh_time,first_change_time,") + .append("last_change_time,plan_line_id,starts,output_rows,db_time,user_io_wait_time,workarea_max_mem,") + .append("workarea_max_tempseg,process_name,"); + for (int i = 1; i <= 10; i++) { + sqlBuilder.append("otherstat_").append(i).append("_id,"); + sqlBuilder.append("otherstat_").append(i).append("_value,"); + } + sqlBuilder.append("current_timestamp(6) current_ts from ") + .append(dialectType.isMysql() ? "oceanbase" : "sys") + .append(".v$sql_plan_monitor where trace_id=") + .value(traceId); + List records = new ArrayList<>(); + try (ResultSet rs = statement.executeQuery(sqlBuilder.toString())) { + while (rs.next()) { + SqlPlanMonitor record = new SqlPlanMonitor(); + record.setSvrIp(rs.getString("svr_ip")); + record.setSvrPort(rs.getString("svr_port")); + record.setProcessName(rs.getString("process_name")); + record.setFirstRefreshTime(rs.getTimestamp("first_refresh_time")); + record.setLastRefreshTime(rs.getTimestamp("last_refresh_time")); + record.setFirstChangeTime(rs.getTimestamp("first_change_time")); + record.setLastChangeTime(rs.getTimestamp("last_change_time")); + record.setCurrentTime(rs.getTimestamp("current_ts")); + record.setWorkareaMaxMem(rs.getLong("workarea_max_mem")); + record.setWorkareaMaxTempSeg(rs.getLong("workarea_max_tempseg")); + record.setPlanLineId(rs.getString("plan_line_id")); + record.setStarts(rs.getLong("starts")); + record.setOutputRows(rs.getLong("output_rows")); + record.setDbTime(rs.getLong("db_time")); + record.setUserIOWaitTime(rs.getLong("user_io_wait_time")); + Map otherstats = new HashMap<>(); + for (int i = 1; i <= 10; i++) { + String key = "otherstat_" + i + "_id"; + String statKey = rs.getString(key); + if (statKey == null || otherstats.containsKey(statKey) || !statId2Name.containsKey(statKey)) { + continue; + } + otherstats.put(statId2Name.get(statKey), rs.getString("otherstat_" + i + "_value")); + } + record.setOtherstats(otherstats); + records.add(record); + } + } + Verify.notEmpty(records, "sql plan monitor"); + return records; + } + + public static Map querySPMStatNames(@NonNull Statement statement, ConnectType connectType) + throws SQLException { + SqlBuilder sqlBuilder = getBuilder(connectType) + .append("select id, name from ") + .append(connectType.getDialectType().isMysql() ? "oceanbase" : "sys") + .append(".v$sql_monitor_statname"); + Map statId2Name = new HashMap<>(); + try (ResultSet rs = statement.executeQuery(sqlBuilder.toString())) { + while (rs.next()) { + statId2Name.put(rs.getString("id"), rs.getString("name")); + } + } + return statId2Name; + } + } diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages.properties index 30f9313597..ad9cad6d06 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages.properties @@ -888,4 +888,3 @@ com.oceanbase.odc.PartitionPlanVariableKey.INTERVAL=interval com.oceanbase.odc.PartitionPlanVariableKey.LAST_PARTITION_VALUE=The value of the positional expression corresponding to the last partition rule com.oceanbase.odc.partitionplan.TimeDataType=Time type com.oceanbase.odc.partitionplan.NumberDataType=Number type - diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties index a6880ae204..a93bad1a14 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties @@ -825,4 +825,3 @@ com.oceanbase.odc.PartitionPlanVariableKey.INTERVAL=间隔 com.oceanbase.odc.PartitionPlanVariableKey.LAST_PARTITION_VALUE=最后一个分区规则对应位置表达式的值 com.oceanbase.odc.partitionplan.TimeDataType=时间类型 com.oceanbase.odc.partitionplan.NumberDataType=数字类型 - diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages.properties index a6a28ba4ab..20c689eab1 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages.properties @@ -195,3 +195,4 @@ com.oceanbase.odc.ErrorCodes.InvalidSqlExpression=An error occurred during the e com.oceanbase.odc.ErrorCodes.PartitionKeyDataTypeMismatch=The data type does not match. The expected data type for partition key {0} is {1}, but the actual data type is {2}. com.oceanbase.odc.ErrorCodes.TimeDataTypePrecisionMismatch=The time type precision does not match. The configured time interval precision is greater than the maximum precision of the partition key {0} data type. com.oceanbase.odc.ErrorCodes.DatabaseAccessDenied=Database access is denied because the current user does not have {0} permissions for the database. +com.oceanbase.odc.ErrorCodes.ObQueryProfileNotSupported=Query profile is only available for OceanBase Database with versions equal to or higher than {0}. diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties index 079879d31d..0a34f7797e 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties @@ -194,3 +194,4 @@ com.oceanbase.odc.ErrorCodes.PartitionPlanNoDropPreviewSqlGenerated=没有分区 com.oceanbase.odc.ErrorCodes.PartitionKeyDataTypeMismatch = 数据类型不匹配，分区键 {0} 的预期数据类型为 {1}，实际数据类型为 {2} com.oceanbase.odc.ErrorCodes.TimeDataTypePrecisionMismatch = 时间类型精度不匹配，配置的时间间隔精度大于分区键 {0} 数据类型的最大精度 com.oceanbase.odc.ErrorCodes.DatabaseAccessDenied=数据库访问被拒绝，因为当前用户没有数据库的{0}权限 +com.oceanbase.odc.ErrorCodes.ObQueryProfileNotSupported=执行详情仅在高于或等于 {0} 的 OceanBase 版本可用 diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties index ec5f45fe8e..1929e3496d 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties @@ -194,3 +194,4 @@ com.oceanbase.odc.ErrorCodes.PartitionPlanNoDropPreviewSqlGenerated=沒有分割 com.oceanbase.odc.ErrorCodes.PartitionKeyDataTypeMismatch = 資料類型不匹配，分區鍵 {0} 的預期資料類型為 {1}，實際資料類型為 {2} com.oceanbase.odc.ErrorCodes.TimeDataTypePrecisionMismatch = 時間類型精度不匹配，配置的時間間隔精度大於分區鍵 {0} 資料類型的最大精度 com.oceanbase.odc.ErrorCodes.DatabaseAccessDenied=資料庫存取被拒絕，因為目前使用者沒有資料庫的{0}權限 +com.oceanbase.odc.ErrorCodes.ObQueryProfileNotSupported=執行詳情僅在高於或等於 {0} 的 OceanBase 版本可用 diff --git a/server/odc-migrate/src/main/resources/init-config/default-audit-event-meta.yml b/server/odc-migrate/src/main/resources/init-config/default-audit-event-meta.yml index db323b8c20..a5e85ac1de 100644 --- a/server/odc-migrate/src/main/resources/init-config/default-audit-event-meta.yml +++ b/server/odc-migrate/src/main/resources/init-config/default-audit-event-meta.yml @@ -108,7 +108,7 @@ # SQL 窗口执行 - type: "DATABASE_OPERATION" action: "OTHERS" - method_signature: "com.oceanbase.odc.server.web.controller.v2.ConnectSessionController.asyncSqlExecute" + method_signature: "com.oceanbase.odc.server.web.controller.v2.ConnectSessionController.streamExecute" sid_extract_expression: "{#sessionId}" in_connection: 1 enabled: 1 diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/SqlDiagnoseController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/SqlDiagnoseController.java index 8c2b6c914e..2a9103662c 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/SqlDiagnoseController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/SqlDiagnoseController.java @@ -24,9 +24,10 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; +import com.oceanbase.odc.core.shared.PreConditions; import com.oceanbase.odc.core.shared.model.SqlExecDetail; -import com.oceanbase.odc.core.shared.model.SqlExplain; import com.oceanbase.odc.core.shared.model.TraceSpan; +import com.oceanbase.odc.plugin.connect.model.diagnose.SqlExplain; import com.oceanbase.odc.service.common.model.ResourceSql; import com.oceanbase.odc.service.common.response.OdcResult; import com.oceanbase.odc.service.common.response.Responses; @@ -95,4 +96,15 @@ public SuccessResponse getFullLinkTraceJson(@PathVariable String sid, @R sessionService.nullSafeGet(SidUtils.getSessionId(sid)), sql)); } + @ApiOperation(value = "getQueryProfile") + @RequestMapping(value = "/getQueryProfile/{sid}", method = RequestMethod.POST) + @StatefulRoute(stateName = StateName.DB_SESSION, stateIdExpression = "#sid") + public SuccessResponse getQueryProfile(@PathVariable String sid, + @RequestBody ResourceSql resource) throws IOException { + String traceId = resource.getTag(); + PreConditions.notEmpty(traceId, "trace id"); + return Responses.success(diagnoseService.getQueryProfile( + sessionService.nullSafeGet(SidUtils.getSessionId(sid)), traceId)); + } + } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ConnectSessionController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ConnectSessionController.java index 257684ad50..b3407c7146 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ConnectSessionController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ConnectSessionController.java @@ -104,23 +104,6 @@ public SuccessResponse createSessionByDatabase(@PathVariable return Responses.success(sessionService.createByDatabaseId(databaseId)); } - /** - * 异步执行sql
- * asyncExecute 和 asyncExecuteQueryData 功能完全一样，区分是为了根据不同场景配置拦截策略
- * - asyncExecute 为 SQL 窗口场景
- * - asyncExecuteQueryData 为 表数据查询场景 - * - * @param sessionId - * @return - */ - @ApiOperation(value = "asyncSqlExecute", notes = "异步执行sql") - @RequestMapping(value = {"/sessions/{sessionId}/sqls/asyncExecute"}, method = RequestMethod.POST) - @StatefulRoute(stateName = StateName.DB_SESSION, stateIdExpression = "#sessionId") - public SuccessResponse asyncSqlExecute(@PathVariable String sessionId, - @RequestBody SqlAsyncExecuteReq req) throws Exception { - return Responses.success(consoleService.execute(SidUtils.getSessionId(sessionId), req)); - } - @RequestMapping(value = {"/sessions/{sessionId}/sqls/streamExecute"}, method = RequestMethod.POST) @StatefulRoute(stateName = StateName.DB_SESSION, stateIdExpression = "#sessionId") public SuccessResponse streamExecute(@PathVariable String sessionId, @@ -128,20 +111,6 @@ public SuccessResponse streamExecute(@PathVariable String s return Responses.success(consoleService.streamExecute(SidUtils.getSessionId(sessionId), req, true)); } - /** - * 获取异步执行sql的结果 Todo 这里的sqlIds后续需要改成一个string类型的requestId，异步api请求需要有超时机制 - * - * @param sessionId - * @return - */ - @ApiOperation(value = "getAsyncSqlExecute", notes = "异步执行获取结果sql") - @RequestMapping(value = "/sessions/{sessionId}/sqls/getResult", method = RequestMethod.GET) - @StatefulRoute(stateName = StateName.DB_SESSION, stateIdExpression = "#sessionId") - public SuccessResponse> getAsyncSqlExecute(@PathVariable String sessionId, - @RequestParam String requestId) { - return Responses.success(consoleService.getAsyncResult(SidUtils.getSessionId(sessionId), requestId, null)); - } - @RequestMapping(value = "/sessions/{sessionId}/sqls/getMoreResults", method = RequestMethod.GET) @StatefulRoute(stateName = StateName.DB_SESSION, stateIdExpression = "#sessionId") public SuccessResponse getMoreResults(@PathVariable String sessionId, diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/config/ScheduleConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/config/ScheduleConfiguration.java index 99615351d5..ccd132fac5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/config/ScheduleConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/config/ScheduleConfiguration.java @@ -283,6 +283,21 @@ public ThreadPoolTaskExecutor logicalTableExtractTaskExecutor() { return executor; } + @Bean(name = "queryProfileMonitorExecutor") + public ThreadPoolTaskExecutor queryProfileMonitorExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + int poolSize = Math.max(SystemUtils.availableProcessors() * 2, 8); + executor.setCorePoolSize(poolSize); + executor.setMaxPoolSize(poolSize); + executor.setThreadNamePrefix("query-profile-monitor-"); + executor.setWaitForTasksToCompleteOnShutdown(false); + executor.setAwaitTerminationSeconds(5); + executor.setTaskDecorator(new TraceDecorator<>()); + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy()); + executor.initialize(); + log.info("queryProfileMonitorExecutor initialized"); + return executor; + } @Scheduled(fixedDelay = REFRESH_CONFIG_RATE_MILLIS) public void refreshSysConfig() { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/audit/AuditEventAspect.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/audit/AuditEventAspect.java index 29865acba2..4789187478 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/audit/AuditEventAspect.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/audit/AuditEventAspect.java @@ -78,6 +78,7 @@ import com.oceanbase.odc.service.flow.model.FlowInstanceDetailResp; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.session.ConnectSessionService; +import com.oceanbase.odc.service.session.model.AsyncExecuteResultResp; import com.oceanbase.odc.service.session.model.SqlAsyncExecuteResp; import com.oceanbase.odc.service.session.model.SqlExecuteResult; import com.oceanbase.odc.service.session.model.SqlTuplesWithViolation; @@ -132,7 +133,7 @@ public class AuditEventAspect { @Pointcut("execution(public * com.oceanbase.odc.server.web.controller.*.*.*(..))") public void eventAudit() {} - @Pointcut("execution(public * com.oceanbase.odc.server.web.controller.v2.ConnectSessionController.getAsyncSqlExecute(..))") + @Pointcut("execution(public * com.oceanbase.odc.server.web.controller.v2.ConnectSessionController.getMoreResults(..))") public void getAsyncSqlExecuteResult() {} @Around("eventAudit()") @@ -164,14 +165,16 @@ public Object aroundEventAudit(ProceedingJoinPoint joinPoint) throws Throwable { @AfterReturning(value = "getAsyncSqlExecuteResult()", returning = "returnValue") public void afterSqlExecute(Object returnValue) { try { - - Object data = null; - if (returnValue instanceof SuccessResponse) { - data = ((SuccessResponse) returnValue).getData(); + if (!(returnValue instanceof SuccessResponse)) { + return; + } + AsyncExecuteResultResp data = (AsyncExecuteResultResp) ((SuccessResponse) returnValue).getData(); + List results = data.getResults(); + if (results.isEmpty()) { + return; } List events = new ArrayList<>(); - for (Object obj : (List) data) { - SqlExecuteResult result = (SqlExecuteResult) obj; + for (SqlExecuteResult result : results) { AuditEventResult auditEventResult = getAuditEventResultFromResult(result); if (AuditEventResult.UNFINISHED != auditEventResult) { AuditEventAction action = AuditUtils.getSqlTypeFromResult(result.getSqlType()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/diagnose/SqlDiagnoseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/diagnose/SqlDiagnoseService.java index 7373337168..750e84c7c6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/diagnose/SqlDiagnoseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/diagnose/SqlDiagnoseService.java @@ -36,15 +36,16 @@ import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.exception.NotFoundException; import com.oceanbase.odc.core.shared.model.SqlExecDetail; -import com.oceanbase.odc.core.shared.model.SqlExplain; import com.oceanbase.odc.core.shared.model.TraceSpan; import com.oceanbase.odc.core.sql.execute.cache.model.BinaryContentMetaData; +import com.oceanbase.odc.plugin.connect.model.diagnose.SqlExplain; import com.oceanbase.odc.service.common.model.ResourceSql; import com.oceanbase.odc.service.diagnose.fulllinktrace.JaegerConverter; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.objectstorage.ObjectStorageFacade; import com.oceanbase.odc.service.objectstorage.model.ObjectMetadata; import com.oceanbase.odc.service.plugin.ConnectionPluginUtil; +import com.oceanbase.odc.service.queryprofile.OBQueryProfileManager; @Service @SkipAuthorize("inside connect session") @@ -53,6 +54,8 @@ public class SqlDiagnoseService { private ObjectStorageFacade objectStorageFacade; @Autowired private AuthenticationFacade authenticationFacade; + @Autowired + private OBQueryProfileManager profileManager; public SqlExplain explain(ConnectionSession session, ResourceSql odcSql) { return session.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY) @@ -107,4 +110,8 @@ public String getFullLinkTraceDownloadUrl(ConnectionSession session, ResourceSql return objectStorageFacade.getDownloadUrl(metadata.getBucketName(), metadata.getObjectId()); } + public SqlExplain getQueryProfile(ConnectionSession session, String traceId) throws IOException { + return profileManager.getProfile(traceId, session); + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/OBQueryProfileManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/OBQueryProfileManager.java new file mode 100644 index 0000000000..37d4bd2153 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/OBQueryProfileManager.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.queryprofile; + +import static com.oceanbase.odc.core.session.ConnectionSessionConstants.BACKEND_DS_KEY; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.jdbc.core.ConnectionCallback; +import org.springframework.jdbc.core.StatementCallback; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.stereotype.Component; +import org.springframework.util.StreamUtils; + +import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.common.util.VersionUtils; +import com.oceanbase.odc.core.session.ConnectionSession; +import com.oceanbase.odc.core.session.ConnectionSessionUtil; +import com.oceanbase.odc.core.shared.constant.ErrorCodes; +import com.oceanbase.odc.core.shared.exception.BadRequestException; +import com.oceanbase.odc.core.shared.exception.UnexpectedException; +import com.oceanbase.odc.core.sql.execute.cache.BinaryDataManager; +import com.oceanbase.odc.core.sql.execute.cache.model.BinaryContentMetaData; +import com.oceanbase.odc.plugin.connect.model.diagnose.SqlExplain; +import com.oceanbase.odc.service.plugin.ConnectionPluginUtil; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + +/** + * @author: liuyizhuo.lyz + * @date: 2024/4/19 + */ +@Slf4j +@Component +public class OBQueryProfileManager { + private static final String PROFILE_KEY_PREFIX = "query-profile-"; + public static final String ENABLE_QUERY_PROFILE_VERSION = "4.2.4"; + + @Autowired + @Qualifier("queryProfileMonitorExecutor") + private ThreadPoolTaskExecutor executor; + + public void submit(ConnectionSession session, @NonNull String traceId) { + executor.execute(() -> { + if (session.isExpired() + || ConnectionSessionUtil.getBinaryContentMetadata(session, PROFILE_KEY_PREFIX + traceId) != null) { + return; + } + try { + SqlExplain profile = session.getSyncJdbcExecutor(BACKEND_DS_KEY).execute( + (ConnectionCallback) conn -> ConnectionPluginUtil + .getDiagnoseExtension(session.getConnectType().getDialectType()) + .getQueryProfileByTraceId(conn, traceId)); + BinaryDataManager binaryDataManager = ConnectionSessionUtil.getBinaryDataManager(session); + BinaryContentMetaData metaData = binaryDataManager.write( + new ByteArrayInputStream(JsonUtils.toJson(profile).getBytes())); + String key = PROFILE_KEY_PREFIX + traceId; + ConnectionSessionUtil.setBinaryContentMetadata(session, key, metaData); + } catch (Exception e) { + log.warn("Failed to cache profile.", e); + } + }); + } + + public SqlExplain getProfile(@NonNull String traceId, ConnectionSession session) { + if (VersionUtils.isLessThan(ConnectionSessionUtil.getVersion(session), ENABLE_QUERY_PROFILE_VERSION)) { + throw new BadRequestException(ErrorCodes.ObQueryProfileNotSupported, + new Object[] {ENABLE_QUERY_PROFILE_VERSION}, + ErrorCodes.ObQueryProfileNotSupported + .getLocalizedMessage(new Object[] {ENABLE_QUERY_PROFILE_VERSION})); + } + try { + BinaryContentMetaData metadata = + ConnectionSessionUtil.getBinaryContentMetadata(session, PROFILE_KEY_PREFIX + traceId); + if (metadata != null) { + InputStream stream = ConnectionSessionUtil.getBinaryDataManager(session).read(metadata); + return JsonUtils.fromJson(StreamUtils.copyToString(stream, StandardCharsets.UTF_8), SqlExplain.class); + } + return session.getSyncJdbcExecutor(BACKEND_DS_KEY).execute( + (StatementCallback) stmt -> ConnectionPluginUtil + .getDiagnoseExtension(session.getConnectType().getDialectType()) + .getQueryProfileByTraceId(stmt.getConnection(), traceId)); + } catch (Exception e) { + log.warn("Failed to get profile with OB trace_id={}.", traceId, e); + throw new UnexpectedException( + String.format("Failed to get profile with OB trace_id=%s. Reason:%s", traceId, e.getMessage())); + } + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/display/PlanGraph.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/display/PlanGraph.java deleted file mode 100644 index e8855285d6..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/display/PlanGraph.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.queryprofile.display; - -import java.util.List; -import java.util.Map; - -import lombok.Data; - -/** - * @author liuyizhuo.lyz - * @date 2024/4/11 - */ -@Data -public class PlanGraph { - private List edges; - private List vertexes; - private Map statistics; - private Map overview; - private Map> topNodes; -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/display/PlanGraphOperator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/display/PlanGraphOperator.java deleted file mode 100644 index 96aae9f367..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/display/PlanGraphOperator.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.queryprofile.display; - -import java.util.Map; - -import com.oceanbase.odc.service.queryprofile.model.SqlProfile.Status; - -import lombok.Data; - -/** - * @author liuyizhuo.lyz - * @date 2024/4/11 - */ -@Data -public class PlanGraphOperator { - private String graphId; - private String name; - private String title; - private Status status; - private Map attributes; - private Map statistics; - private Map overview; -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/model/SqlPlanGraph.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/model/SqlPlanGraph.java deleted file mode 100644 index 756f8f9695..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/model/SqlPlanGraph.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.queryprofile.model; - -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; - -import com.oceanbase.odc.common.graph.Graph; -import com.oceanbase.odc.common.graph.GraphVertex; - -import lombok.Getter; -import lombok.NonNull; - -/** - * @author liuyizhuo.lyz - * @date 2024/4/11 - */ -@Getter -public class SqlPlanGraph extends Graph { - private final Map overview = new LinkedHashMap<>(); - private final Map statistics = new LinkedHashMap<>(); - private final Map operatorMap = new HashMap<>(); - - public void setStatistic(String key, String value) { - if (value != null) { - statistics.put(key, value); - } - } - - public void setOverview(String key, String value) { - if (value != null) { - overview.put(key, value); - } - } - - @Override - public GraphVertex insertVertex(@NonNull GraphVertex vertex) { - operatorMap.put(vertex.getGraphId(), vertex); - return super.insertVertex(vertex); - } - - public GraphVertex getVertex(@NonNull String id) { - return operatorMap.get(id); - } -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/model/SqlProfile.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/model/SqlProfile.java deleted file mode 100644 index 5dc0b4dc84..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/model/SqlProfile.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.queryprofile.model; - -import java.util.Date; - -import com.oceanbase.odc.core.session.ConnectionSession; - -import lombok.Data; - -/** - * @author liuyizhuo.lyz - * @date 2024/4/10 - */ -@Data -public class SqlProfile { - private Date startTime; - private Date endTime; - private Status status; - private Long duration; - private String traceId; - private String planId; - private String sqlText; - private SqlPlanGraph graph; - - private Long creatorId; - private Long organizationId; - private Long projectId; - - ConnectionSession session; - - public enum Status { - PREPARING, - RUNNING, - FINISHED, - FAILED, - } -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java index 2132998e0d..9f1ae6ae53 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java @@ -23,10 +23,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import javax.validation.Valid; @@ -92,6 +89,7 @@ import com.oceanbase.odc.service.feature.AllFeatures; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; +import com.oceanbase.odc.service.queryprofile.OBQueryProfileManager; import com.oceanbase.odc.service.session.interceptor.SqlCheckInterceptor; import com.oceanbase.odc.service.session.interceptor.SqlConsoleInterceptor; import com.oceanbase.odc.service.session.interceptor.SqlExecuteInterceptorService; @@ -127,7 +125,7 @@ @SkipAuthorize("inside connect session") public class ConnectConsoleService { - public static final int DEFAULT_GET_RESULT_TIMEOUT_SECONDS = 3; + public static final int DEFAULT_GET_RESULT_TIMEOUT_SECONDS = 1; public static final String SHOW_TABLE_COLUMN_INFO = "SHOW_TABLE_COLUMN_INFO"; @Autowired @@ -146,6 +144,8 @@ public class ConnectConsoleService { private UserConfigFacade userConfigFacade; @Autowired private AuthenticationFacade authenticationFacade; + @Autowired + private OBQueryProfileManager profileManager; public SqlExecuteResult queryTableOrViewData(@NotNull String sessionId, @NotNull @Valid QueryTableOrViewDataReq req) throws Exception { @@ -190,7 +190,8 @@ public SqlExecuteResult queryTableOrViewData(@NotNull String sessionId, asyncExecuteReq.setShowTableColumnInfo(true); asyncExecuteReq.setContinueExecutionOnError(true); asyncExecuteReq.setFullLinkTraceEnabled(false); - SqlAsyncExecuteResp resp = execute(sessionId, asyncExecuteReq, false); + // SqlAsyncExecuteResp resp = execute(sessionId, asyncExecuteReq, false); + SqlAsyncExecuteResp resp = streamExecute(sessionId, asyncExecuteReq, false); List unauthorizedDBResources = resp.getUnauthorizedDBResources(); if (CollectionUtils.isNotEmpty(unauthorizedDBResources)) { @@ -203,7 +204,10 @@ public SqlExecuteResult queryTableOrViewData(@NotNull String sessionId, String requestId = resp.getRequestId(); ConnectionConfig connConfig = (ConnectionConfig) ConnectionSessionUtil.getConnectionConfig(connectionSession); - List results = getAsyncResult(sessionId, requestId, connConfig.queryTimeoutSeconds()); + + List results = + getMoreResults(sessionId, requestId, connConfig.queryTimeoutSeconds()).getResults(); + if (CollectionUtils.isEmpty(results)) { String sqlId = resp.getSqls().get(0).getSqlTuple().getSqlId(); throw new RequestTimeoutException(String @@ -226,92 +230,9 @@ public SqlExecuteResult queryTableOrViewData(@NotNull String sessionId, return result; } - public SqlAsyncExecuteResp execute(@NotNull String sessionId, @NotNull @Valid SqlAsyncExecuteReq request) + public SqlAsyncExecuteResp streamExecute(@NotNull String sessionId, @NotNull @Valid SqlAsyncExecuteReq request) throws Exception { - return execute(sessionId, request, true); - } - - public SqlAsyncExecuteResp execute(@NotNull String sessionId, - @NotNull @Valid SqlAsyncExecuteReq request, boolean needSqlRuleCheck) throws Exception { - ConnectionSession connectionSession = sessionService.nullSafeGet(sessionId, true); - - long maxSqlLength = sessionProperties.getMaxSqlLength(); - if (maxSqlLength > 0) { - PreConditions.lessThanOrEqualTo("sqlLength", LimitMetric.SQL_LENGTH, - StringUtils.length(request.getSql()), maxSqlLength); - } - SqlAsyncExecuteResp result = filterKillSession(connectionSession, request); - if (result != null) { - return result; - } - List sqls = request.ifSplitSqls() - ? SqlUtils.splitWithOffset(connectionSession, request.getSql(), - sessionProperties.isOracleRemoveCommentPrefix()) - : Collections.singletonList(new OffsetString(0, request.getSql())); - if (sqls.size() == 0) { - /** - * if a sql only contains delimiter setting(eg. delimiter $$), code will do this - */ - SqlTuple sqlTuple = SqlTuple.newTuple(request.getSql()); - String id = ConnectionSessionUtil.setFutureJdbc(connectionSession, - FutureResult.successResultList(JdbcGeneralResult.successResult(sqlTuple)), null); - return SqlAsyncExecuteResp.newSqlAsyncExecuteResp(id, Collections.singletonList(sqlTuple)); - } - - long maxSqlStatementCount = sessionProperties.getMaxSqlStatementCount(); - if (maxSqlStatementCount > 0) { - PreConditions.lessThanOrEqualTo("sqlStatementCount", - LimitMetric.SQL_STATEMENT_COUNT, sqls.size(), maxSqlStatementCount); - } - - List sqlTuples = generateSqlTuple(sqls, connectionSession, request); - SqlAsyncExecuteResp response = SqlAsyncExecuteResp.newSqlAsyncExecuteResp(sqlTuples); - Map context = new HashMap<>(); - context.put(SHOW_TABLE_COLUMN_INFO, request.getShowTableColumnInfo()); - context.put(SqlCheckInterceptor.NEED_SQL_CHECK_KEY, needSqlRuleCheck); - context.put(SqlConsoleInterceptor.NEED_SQL_CONSOLE_CHECK, needSqlRuleCheck); - AsyncExecuteContext executeContext = new AsyncExecuteContext(sqlTuples, context); - List stages = sqlTuples.stream() - .map(s -> s.getSqlWatch().start(SqlExecuteStages.SQL_PRE_CHECK)) - .collect(Collectors.toList()); - try { - if (!sqlInterceptService.preHandle(request, response, connectionSession, executeContext)) { - return response; - } - } finally { - for (TraceStage stage : stages) { - try { - stage.close(); - } catch (Exception e) { - // eat exception - } - } - } - Integer queryLimit = checkQueryLimit(request.getQueryLimit()); - boolean continueExecutionOnError = - Objects.nonNull(request.getContinueExecutionOnError()) ? request.getContinueExecutionOnError() - : userConfigFacade.isContinueExecutionOnError(); - boolean stopOnError = !continueExecutionOnError; - OdcStatementCallBack statementCallBack = new OdcStatementCallBack(sqlTuples, connectionSession, - request.getAutoCommit(), queryLimit, stopOnError); - - statementCallBack.setDbmsoutputMaxRows(sessionProperties.getDbmsOutputMaxRows()); - - boolean fullLinkTraceEnabled = - Objects.nonNull(request.getFullLinkTraceEnabled()) ? request.getFullLinkTraceEnabled() - : userConfigFacade.isFullLinkTraceEnabled(); - statementCallBack.setUseFullLinkTrace(fullLinkTraceEnabled); - - statementCallBack.setFullLinkTraceTimeout(sessionProperties.getFullLinkTraceTimeoutSeconds()); - statementCallBack.setMaxCachedSize(sessionProperties.getResultSetMaxCachedSize()); - statementCallBack.setMaxCachedLines(sessionProperties.getResultSetMaxCachedLines()); - statementCallBack.setLocale(LocaleContextHolder.getLocale()); - - Future> futureResult = connectionSession.getAsyncJdbcExecutor( - ConnectionSessionConstants.CONSOLE_DS_KEY).execute(statementCallBack); - String id = ConnectionSessionUtil.setFutureJdbc(connectionSession, futureResult, context); - response.setRequestId(id); - return response; + return streamExecute(sessionId, request, true); } public SqlAsyncExecuteResp streamExecute(@NotNull String sessionId, @@ -336,8 +257,13 @@ public SqlAsyncExecuteResp streamExecute(@NotNull String sessionId, * if a sql only contains delimiter setting(eg. delimiter $$), code will do this */ SqlTuple sqlTuple = SqlTuple.newTuple(request.getSql()); - String id = ConnectionSessionUtil.setFutureJdbc(connectionSession, - FutureResult.successResultList(JdbcGeneralResult.successResult(sqlTuple)), null); + AsyncExecuteContext executeContext = + new AsyncExecuteContext(Collections.singletonList(sqlTuple), new HashMap<>()); + Future> successFuture = FutureResult.successResultList( + JdbcGeneralResult.successResult(sqlTuple)); + executeContext.setFuture(successFuture); + executeContext.addSqlExecutionResults(successFuture.get()); + String id = ConnectionSessionUtil.setExecuteContext(connectionSession, executeContext); return SqlAsyncExecuteResp.newSqlAsyncExecuteResp(id, Collections.singletonList(sqlTuple)); } @@ -390,7 +316,7 @@ public SqlAsyncExecuteResp streamExecute(@NotNull String sessionId, statementCallBack.setMaxCachedLines(sessionProperties.getResultSetMaxCachedLines()); statementCallBack.setLocale(LocaleContextHolder.getLocale()); if (connectionSession.getDialectType().isOceanbase() && sqlTuples.size() <= 10) { - statementCallBack.getListeners().add(new OBExecutionListener(connectionSession)); + statementCallBack.getListeners().add(new OBExecutionListener(connectionSession, profileManager)); } Future> futureResult = connectionSession.getAsyncJdbcExecutor( @@ -401,49 +327,19 @@ public SqlAsyncExecuteResp streamExecute(@NotNull String sessionId, return response; } - public List getAsyncResult(@NotNull String sessionId, @NotNull String requestId) { - return getAsyncResult(sessionId, requestId, DEFAULT_GET_RESULT_TIMEOUT_SECONDS); - } - - public List getAsyncResult(@NotNull String sessionId, String requestId, Integer timeoutSeconds) { - PreConditions.validArgumentState(Objects.nonNull(requestId), ErrorCodes.SqlRegulationRuleBlocked, null, null); - ConnectionSession connectionSession = sessionService.nullSafeGet(sessionId); - Future> listFuture = - ConnectionSessionUtil.getFutureJdbcResult(connectionSession, requestId); - int timeout = Objects.isNull(timeoutSeconds) ? DEFAULT_GET_RESULT_TIMEOUT_SECONDS : timeoutSeconds; - try { - List resultList = listFuture.get(timeout, TimeUnit.SECONDS); - Map context = ConnectionSessionUtil.getFutureJdbcContext(connectionSession, requestId); - ConnectionSessionUtil.removeFutureJdbc(connectionSession, requestId); - return resultList.stream().map(jdbcGeneralResult -> { - Map ctx = context == null ? new HashMap<>() : context; - SqlExecuteResult result = generateResult(connectionSession, jdbcGeneralResult, ctx); - try (TraceStage stage = result.getSqlTuple().getSqlWatch().start(SqlExecuteStages.SQL_AFTER_CHECK)) { - sqlInterceptService.afterCompletion(result, connectionSession, new AsyncExecuteContext(null, ctx)); - } catch (Exception e) { - throw new IllegalStateException(e); - } - return result; - }).collect(Collectors.toList()); - } catch (InterruptedException | ExecutionException e) { - throw new IllegalStateException(e); - } catch (TimeoutException timeoutException) { - if (log.isDebugEnabled()) { - log.debug("Get sql execution result timed out, sessionId={}, requestId={}", sessionId, requestId, - timeoutException); - } - return Collections.emptyList(); - } + public AsyncExecuteResultResp getMoreResults(@NotNull String sessionId, String requestId) { + return getMoreResults(sessionId, requestId, null); } - public AsyncExecuteResultResp getMoreResults(@NotNull String sessionId, String requestId) { + public AsyncExecuteResultResp getMoreResults(@NotNull String sessionId, String requestId, Integer timeoutSeconds) { PreConditions.validArgumentState(Objects.nonNull(requestId), ErrorCodes.SqlRegulationRuleBlocked, null, null); ConnectionSession connectionSession = sessionService.nullSafeGet(sessionId); AsyncExecuteContext context = (AsyncExecuteContext) ConnectionSessionUtil.getExecuteContext(connectionSession, requestId); + int timeout = Objects.isNull(timeoutSeconds) ? DEFAULT_GET_RESULT_TIMEOUT_SECONDS : timeoutSeconds; boolean shouldRemoveContext = context.isFinished(); try { - List resultList = context.getMoreSqlExecutionResults(); + List resultList = context.getMoreSqlExecutionResults(timeout); List results = resultList.stream().map(jdbcGeneralResult -> { SqlExecuteResult result = generateResult(connectionSession, jdbcGeneralResult, context.getContextMap()); try (TraceStage stage = result.getSqlTuple().getSqlWatch().start(SqlExecuteStages.SQL_AFTER_CHECK)) { @@ -643,7 +539,8 @@ private boolean validateSqlSemantics(String sql, ConnectionSession session) { * @param request odc sql object * @return result of sql execution */ - private SqlAsyncExecuteResp filterKillSession(ConnectionSession connectionSession, SqlAsyncExecuteReq request) { + private SqlAsyncExecuteResp filterKillSession(ConnectionSession connectionSession, SqlAsyncExecuteReq request) + throws Exception { String sqlScript = request.getSql().trim().toLowerCase(); if (!sqlScript.startsWith("kill ") || !sqlScript.contains("/*")) { return null; @@ -652,7 +549,13 @@ private SqlAsyncExecuteResp filterKillSession(ConnectionSession connectionSessio Arrays.stream(sqlScript.split(";")).filter(StringUtils::isNotBlank).collect(Collectors.toList())); List results = defaultDbSessionManage.executeKillSession(connectionSession, sqlTuples, sqlScript); - String id = ConnectionSessionUtil.setFutureJdbc(connectionSession, FutureResult.successResult(results), null); + + AsyncExecuteContext executeContext = + new AsyncExecuteContext(sqlTuples, new HashMap<>()); + Future> successFuture = FutureResult.successResult(results); + executeContext.setFuture(successFuture); + executeContext.addSqlExecutionResults(successFuture.get()); + String id = ConnectionSessionUtil.setExecuteContext(connectionSession, executeContext); return SqlAsyncExecuteResp.newSqlAsyncExecuteResp(id, sqlTuples); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/OBExecutionListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/OBExecutionListener.java index 9c2792ad1f..46179c450d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/OBExecutionListener.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/OBExecutionListener.java @@ -17,6 +17,7 @@ import static com.oceanbase.odc.core.session.ConnectionSessionConstants.BACKEND_DS_KEY; import static com.oceanbase.odc.core.session.ConnectionSessionConstants.CONSOLE_DS_KEY; +import static com.oceanbase.odc.service.queryprofile.OBQueryProfileManager.ENABLE_QUERY_PROFILE_VERSION; import java.util.Collections; import java.util.List; @@ -30,8 +31,11 @@ import com.oceanbase.odc.core.session.ConnectionSessionUtil; import com.oceanbase.odc.core.sql.execute.model.JdbcGeneralResult; import com.oceanbase.odc.core.sql.execute.model.SqlTuple; +import com.oceanbase.odc.core.sql.parser.AbstractSyntaxTree; import com.oceanbase.odc.core.sql.util.OBUtils; +import com.oceanbase.odc.service.queryprofile.OBQueryProfileManager; import com.oceanbase.odc.service.session.model.AsyncExecuteContext; +import com.oceanbase.tools.dbbrowser.parser.constant.SqlType; /** * @author: liuyizhuo.lyz @@ -39,13 +43,14 @@ */ public class OBExecutionListener implements SqlExecutionListener { private static final Long DEFAULT_QUERY_TRACE_ID_WAIT_MILLIS = 1100L; - private static final String ENABLE_QUERY_PROFILE_VERSION = "4.2"; private final ConnectionSession session; private final List sessionIds; + private final OBQueryProfileManager profileManager; - public OBExecutionListener(ConnectionSession session) { + public OBExecutionListener(ConnectionSession session, OBQueryProfileManager profileManager) { this.session = session; + this.profileManager = profileManager; sessionIds = getSessionIds(); } @@ -53,7 +58,12 @@ public OBExecutionListener(ConnectionSession session) { public void onExecutionStart(SqlTuple sqlTuple, AsyncExecuteContext context) {} @Override - public void onExecutionEnd(SqlTuple sqlTuple, List results, AsyncExecuteContext context) {} + public void onExecutionEnd(SqlTuple sqlTuple, List results, AsyncExecuteContext context) { + JdbcGeneralResult firstResult = results.get(0); + if (StringUtils.isNotEmpty(firstResult.getTraceId()) && isSelect(sqlTuple)) { + profileManager.submit(session, firstResult.getTraceId()); + } + } @Override public void onExecutionCancelled(SqlTuple sqlTuple, List results, AsyncExecuteContext context) {} @@ -64,7 +74,7 @@ public void onExecutionStartAfter(SqlTuple sqlTuple, AsyncExecuteContext context } String traceId = session.getSyncJdbcExecutor(BACKEND_DS_KEY).execute((StatementCallback) stmt -> OBUtils .queryTraceIdFromASH(stmt, sessionIds, session.getConnectType())); - if (traceId != null) { + if (StringUtils.isNotEmpty(traceId)) { context.setCurrentExecutingSqlTraceId(traceId); } } @@ -85,4 +95,14 @@ private List getSessionIds() { return session.getSyncJdbcExecutor(CONSOLE_DS_KEY).execute((StatementCallback>) stmt -> OBUtils .querySessionIdsByProxySessId(stmt, proxySessId, session.getConnectType())); } + + private boolean isSelect(SqlTuple sqlTuple) { + try { + AbstractSyntaxTree ast = sqlTuple.getAst(); + return ast.getParseResult().getSqlType() == SqlType.SELECT; + } catch (Exception e) { + // eat exception + return false; + } + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/OdcStatementCallBack.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/OdcStatementCallBack.java index c4949a3b15..8e4ea8b22c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/OdcStatementCallBack.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/OdcStatementCallBack.java @@ -569,6 +569,7 @@ private void rollback(Connection connection) { private void onExecutionStart(SqlTuple sqlTuple) { if (context != null) { context.setCurrentExecutingSql(sqlTuple.getExecutedSql()); + context.setCurrentExecutingSqlId(sqlTuple.getSqlId()); context.incrementTotalExecutedSqlCount(); context.setCurrentExecutingSqlTraceId(null); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/AsyncExecuteContext.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/AsyncExecuteContext.java index 11027990fe..048857d6b8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/AsyncExecuteContext.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/AsyncExecuteContext.java @@ -44,6 +44,7 @@ public class AsyncExecuteContext { private Future> future; private String currentExecutingSqlTraceId; private String currentExecutingSql; + private String currentExecutingSqlId; private int totalExecutedSqlCount = 0; public AsyncExecuteContext(List sqlTuples, Map contextMap) { @@ -70,8 +71,12 @@ public int getToBeExecutedSqlCount() { /** * only return the incremental results */ - public List getMoreSqlExecutionResults() { + public List getMoreSqlExecutionResults(long timeoutMillis) { List copiedResults = new ArrayList<>(); + + long expect = System.currentTimeMillis() + timeoutMillis; + while (System.currentTimeMillis() <= expect && results.isEmpty()) { + } while (!results.isEmpty()) { copiedResults.add(results.poll()); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/AsyncExecuteResultResp.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/AsyncExecuteResultResp.java index ae94614894..54a0c344fe 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/AsyncExecuteResultResp.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/AsyncExecuteResultResp.java @@ -27,6 +27,7 @@ public class AsyncExecuteResultResp { List results; private String traceId; + private String sqlId; private int total; private int count; private boolean finished; @@ -36,6 +37,7 @@ public AsyncExecuteResultResp(boolean finished, AsyncExecuteContext context, Lis this.finished = finished; this.results = results; traceId = context.getCurrentExecutingSqlTraceId(); + sqlId = context.getCurrentExecutingSqlId(); total = context.getToBeExecutedSqlCount(); count = context.getTotalExecutedSqlCount(); sql = context.getCurrentExecutingSql(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/SqlRewriteUtil.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/SqlRewriteUtil.java index 9771386909..116eb03768 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/SqlRewriteUtil.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/SqlRewriteUtil.java @@ -44,7 +44,6 @@ @Slf4j public class SqlRewriteUtil { - private static final String SELECT_ODC_INTERNAL_ROWID_STMT = ", ROWID AS \"" + OdcConstants.ODC_INTERNAL_ROWID + "\" "; diff --git a/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/SqlDiagnoseExtensionPoint.java b/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/SqlDiagnoseExtensionPoint.java index 9a40c0c980..0d2f75c10f 100644 --- a/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/SqlDiagnoseExtensionPoint.java +++ b/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/SqlDiagnoseExtensionPoint.java @@ -22,7 +22,7 @@ import org.pf4j.ExtensionPoint; import com.oceanbase.odc.core.shared.model.SqlExecDetail; -import com.oceanbase.odc.core.shared.model.SqlExplain; +import com.oceanbase.odc.plugin.connect.model.diagnose.SqlExplain; import lombok.NonNull; @@ -42,4 +42,7 @@ public interface SqlDiagnoseExtensionPoint extends ExtensionPoint { SqlExecDetail getExecutionDetailById(Connection connection, @NonNull String id) throws SQLException; SqlExecDetail getExecutionDetailBySql(Connection connection, @NonNull String sql) throws SQLException; + + SqlExplain getQueryProfileByTraceId(Connection connection, @NonNull String traceId) throws SQLException; + } diff --git a/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/model/diagnose/PlanGraph.java b/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/model/diagnose/PlanGraph.java new file mode 100644 index 0000000000..3a71f7714c --- /dev/null +++ b/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/model/diagnose/PlanGraph.java @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.plugin.connect.model.diagnose; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import com.fasterxml.jackson.annotation.JsonGetter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.oceanbase.odc.common.unit.BinarySizeUnit; +import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.core.shared.model.QueryStatus; + +import lombok.Data; + +/** + * @author liuyizhuo.lyz + * @date 2024/4/11 + */ +@Data +public class PlanGraph { + private String traceId; + private String planId; + private Long duration; + private QueryStatus status; + private List vertexes = new ArrayList<>(); + /** + * Represents information of the execution overview. The reason for using {@link Object} as the + * parameter type here is that value may be a number or a String. Only when key="CPU time" or "I/O + * wait time", value is of numerical type. + */ + private Map overview = new LinkedHashMap<>(); + private Map statistics; + private Map> topNodes; + + @JsonIgnore + private final Map graphId2Operator = new HashMap<>(); + + public void setVertexes(List vertexes) { + this.vertexes = vertexes; + vertexes.forEach(v -> graphId2Operator.put(v.getGraphId(), v)); + } + + public void addStatistics(String key, String value) { + if (value == null) { + return; + } + if (statistics == null) { + statistics = new LinkedHashMap<>(); + } + if (StringUtils.isNumeric(value)) { + long val = Long.parseLong(value) + Long.parseLong(statistics.getOrDefault(key, "0")); + statistics.put(key, val + ""); + } else { + statistics.put(key, value); + } + } + + public void putOverview(String key, Object value) { + if (value != null) { + overview.put(key, value); + } + } + + public PlanGraphOperator getOperator(String id) { + return graphId2Operator.get(id); + } + + @JsonGetter + public Map getStatistics() { + if (statistics == null || statistics.isEmpty()) { + return statistics; + } + for (Entry entry : statistics.entrySet()) { + if (entry.getKey().contains("byte") && StringUtils.isNumeric(entry.getValue())) { + long bytes = Long.parseLong(entry.getValue()); + entry.setValue(BinarySizeUnit.B.of(bytes).toString()); + } + } + return statistics; + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/display/PlanGraphEdge.java b/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/model/diagnose/PlanGraphEdge.java similarity index 93% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/display/PlanGraphEdge.java rename to server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/model/diagnose/PlanGraphEdge.java index f6aeb3a8b7..3bc07be635 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/display/PlanGraphEdge.java +++ b/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/model/diagnose/PlanGraphEdge.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.queryprofile.display; +package com.oceanbase.odc.plugin.connect.model.diagnose; import lombok.Data; diff --git a/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/model/diagnose/PlanGraphOperator.java b/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/model/diagnose/PlanGraphOperator.java new file mode 100644 index 0000000000..7da76afa05 --- /dev/null +++ b/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/model/diagnose/PlanGraphOperator.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.plugin.connect.model.diagnose; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import com.fasterxml.jackson.annotation.JsonGetter; +import com.oceanbase.odc.common.unit.BinarySizeUnit; +import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.core.shared.model.QueryStatus; + +import lombok.Data; +import lombok.NonNull; + +/** + * @author liuyizhuo.lyz + * @date 2024/4/11 + */ +@Data +public class PlanGraphOperator { + private String graphId; + private String name; + private String title; + private QueryStatus status; + private Long duration; + private Map> attributes; + private Map statistics; + /** + * Represents information of the execution overview. The reason for using {@link Object} as the + * parameter type here is that value may be a number or a String. Only when key="CPU time" or "I/O + * wait time", value is of numerical type. + */ + private Map overview; + private List inEdges; + private List outEdges; + private Map subNodes; + + public void putAttribute(@NonNull String key, List value) { + if (attributes == null) { + attributes = new LinkedHashMap<>(); + } + if (value != null) { + attributes.put(key, value); + } + } + + public void putOverview(@NonNull String key, Object value) { + if (overview == null) { + overview = new LinkedHashMap<>(); + } + if (value != null) { + overview.put(key, value); + } + } + + public void putStatistics(@NonNull String key, String value) { + if (statistics == null) { + statistics = new LinkedHashMap<>(); + } + if (StringUtils.isNotEmpty(value)) { + statistics.put(key, value); + } + } + + @JsonGetter + public Map getStatistics() { + if (statistics == null || statistics.isEmpty()) { + return statistics; + } + for (Entry entry : statistics.entrySet()) { + if (entry.getKey().contains("byte") && StringUtils.isNumeric(entry.getValue())) { + long bytes = Long.parseLong(entry.getValue()); + entry.setValue(BinarySizeUnit.B.of(bytes).toString()); + } + } + return statistics; + } + +} diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/SqlExplain.java b/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/model/diagnose/SqlExplain.java similarity index 91% rename from server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/SqlExplain.java rename to server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/model/diagnose/SqlExplain.java index 4726231572..c6780e39cb 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/SqlExplain.java +++ b/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/model/diagnose/SqlExplain.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.core.shared.model; +package com.oceanbase.odc.plugin.connect.model.diagnose; import lombok.Data; @@ -30,4 +30,5 @@ public class SqlExplain { private String outline; private String originalText; private Boolean showFormatInfo; + private PlanGraph graph; } diff --git a/server/plugins/connect-plugin-doris/src/main/java/com/oceanbase/odc/plugin/connect/doris/DorisDiagnoseExtensionPoint.java b/server/plugins/connect-plugin-doris/src/main/java/com/oceanbase/odc/plugin/connect/doris/DorisDiagnoseExtensionPoint.java index fb2936b379..875be038c9 100644 --- a/server/plugins/connect-plugin-doris/src/main/java/com/oceanbase/odc/plugin/connect/doris/DorisDiagnoseExtensionPoint.java +++ b/server/plugins/connect-plugin-doris/src/main/java/com/oceanbase/odc/plugin/connect/doris/DorisDiagnoseExtensionPoint.java @@ -21,8 +21,8 @@ import org.pf4j.Extension; import com.oceanbase.odc.core.shared.model.SqlExecDetail; -import com.oceanbase.odc.core.shared.model.SqlExplain; import com.oceanbase.odc.plugin.connect.api.SqlDiagnoseExtensionPoint; +import com.oceanbase.odc.plugin.connect.model.diagnose.SqlExplain; import com.oceanbase.odc.plugin.connect.mysql.MySQLDiagnoseExtensionPoint; import lombok.NonNull; @@ -59,4 +59,5 @@ public SqlExecDetail getExecutionDetailById(Connection connection, @NonNull Stri public SqlExecDetail getExecutionDetailBySql(Connection connection, @NonNull String sql) throws SQLException { throw new UnsupportedOperationException("Not supported for doris mode"); } + } diff --git a/server/plugins/connect-plugin-doris/src/test/java/com/oceanbase/odc/plugin/connect/doris/DorisExtensionTest.java b/server/plugins/connect-plugin-doris/src/test/java/com/oceanbase/odc/plugin/connect/doris/DorisExtensionTest.java index 03982ed4a2..0baa0576c6 100644 --- a/server/plugins/connect-plugin-doris/src/test/java/com/oceanbase/odc/plugin/connect/doris/DorisExtensionTest.java +++ b/server/plugins/connect-plugin-doris/src/test/java/com/oceanbase/odc/plugin/connect/doris/DorisExtensionTest.java @@ -33,12 +33,12 @@ import com.oceanbase.odc.core.datasource.ConnectionInitializer; import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.constant.OdcConstants; -import com.oceanbase.odc.core.shared.model.SqlExplain; import com.oceanbase.odc.plugin.connect.api.ConnectionExtensionPoint; import com.oceanbase.odc.plugin.connect.api.SessionExtensionPoint; import com.oceanbase.odc.plugin.connect.api.TestResult; import com.oceanbase.odc.plugin.connect.model.ConnectionPropertiesBuilder; import com.oceanbase.odc.plugin.connect.model.JdbcUrlProperty; +import com.oceanbase.odc.plugin.connect.model.diagnose.SqlExplain; import com.oceanbase.odc.test.database.TestDBConfiguration; import com.oceanbase.odc.test.database.TestDBConfigurations; diff --git a/server/plugins/connect-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/connect/mysql/MySQLDiagnoseExtensionPoint.java b/server/plugins/connect-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/connect/mysql/MySQLDiagnoseExtensionPoint.java index 4426115029..8d11debb5f 100644 --- a/server/plugins/connect-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/connect/mysql/MySQLDiagnoseExtensionPoint.java +++ b/server/plugins/connect-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/connect/mysql/MySQLDiagnoseExtensionPoint.java @@ -32,8 +32,8 @@ import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.exception.OBException; import com.oceanbase.odc.core.shared.model.SqlExecDetail; -import com.oceanbase.odc.core.shared.model.SqlExplain; import com.oceanbase.odc.plugin.connect.api.SqlDiagnoseExtensionPoint; +import com.oceanbase.odc.plugin.connect.model.diagnose.SqlExplain; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -92,4 +92,10 @@ public SqlExecDetail getExecutionDetailById(Connection connection, @NonNull Stri public SqlExecDetail getExecutionDetailBySql(Connection connection, @NonNull String sql) throws SQLException { throw new UnsupportedOperationException("Not supported for mysql mode"); } + + @Override + public SqlExplain getQueryProfileByTraceId(Connection connection, @NonNull String traceId) throws SQLException { + throw new UnsupportedOperationException("Not supported for mysql mode"); + } + } diff --git a/server/plugins/connect-plugin-mysql/src/test/java/com/oceanbase/odc/plugin/connect/mysql/MySQLExtensionTest.java b/server/plugins/connect-plugin-mysql/src/test/java/com/oceanbase/odc/plugin/connect/mysql/MySQLExtensionTest.java index 0632f6ea04..c99cb341fa 100644 --- a/server/plugins/connect-plugin-mysql/src/test/java/com/oceanbase/odc/plugin/connect/mysql/MySQLExtensionTest.java +++ b/server/plugins/connect-plugin-mysql/src/test/java/com/oceanbase/odc/plugin/connect/mysql/MySQLExtensionTest.java @@ -32,7 +32,6 @@ import com.oceanbase.odc.core.datasource.ConnectionInitializer; import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.constant.OdcConstants; -import com.oceanbase.odc.core.shared.model.SqlExplain; import com.oceanbase.odc.core.sql.execute.model.SqlExecTime; import com.oceanbase.odc.plugin.connect.api.ConnectionExtensionPoint; import com.oceanbase.odc.plugin.connect.api.InformationExtensionPoint; @@ -41,6 +40,7 @@ import com.oceanbase.odc.plugin.connect.api.TraceExtensionPoint; import com.oceanbase.odc.plugin.connect.model.ConnectionPropertiesBuilder; import com.oceanbase.odc.plugin.connect.model.JdbcUrlProperty; +import com.oceanbase.odc.plugin.connect.model.diagnose.SqlExplain; import com.oceanbase.odc.test.database.TestDBConfiguration; import com.oceanbase.odc.test.database.TestDBConfigurations; diff --git a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLConnectionExtension.java b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLConnectionExtension.java index db48a50453..a7da7d85e8 100644 --- a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLConnectionExtension.java +++ b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLConnectionExtension.java @@ -23,7 +23,7 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; -import java.util.Collections; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -44,6 +44,7 @@ import com.oceanbase.odc.plugin.connect.api.ConnectionExtensionPoint; import com.oceanbase.odc.plugin.connect.api.TestResult; import com.oceanbase.odc.plugin.connect.model.JdbcUrlProperty; +import com.oceanbase.odc.plugin.connect.obmysql.initializer.EnablePlanMonitorInitializer; import com.oceanbase.odc.plugin.connect.obmysql.initializer.EnableTraceInitializer; import lombok.NonNull; @@ -90,7 +91,7 @@ public String getDriverClassName() { @Override public List getConnectionInitializers() { - return Collections.singletonList(new EnableTraceInitializer()); + return Arrays.asList(new EnableTraceInitializer(), new EnablePlanMonitorInitializer()); } @Override diff --git a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLDiagnoseExtension.java b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLDiagnoseExtension.java index 1a2d7c9748..be188bec58 100644 --- a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLDiagnoseExtension.java +++ b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLDiagnoseExtension.java @@ -15,6 +15,11 @@ */ package com.oceanbase.odc.plugin.connect.obmysql; +import static com.oceanbase.odc.plugin.connect.obmysql.diagnose.ProfileConstants.DB_TIME; +import static com.oceanbase.odc.plugin.connect.obmysql.diagnose.ProfileConstants.IS_HIT_PLAN_CACHE; +import static com.oceanbase.odc.plugin.connect.obmysql.diagnose.ProfileConstants.PLAN_TYPE; +import static com.oceanbase.odc.plugin.connect.obmysql.diagnose.ProfileConstants.QUEUE_TIME; + import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -23,19 +28,33 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.collections4.CollectionUtils; import org.pf4j.Extension; import com.alibaba.fastjson.JSON; +import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.common.util.TimespanFormatUtil; import com.oceanbase.odc.common.util.VersionUtils; +import com.oceanbase.odc.core.shared.Verify; +import com.oceanbase.odc.core.shared.constant.ConnectType; import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.exception.OBException; +import com.oceanbase.odc.core.shared.exception.UnexpectedException; +import com.oceanbase.odc.core.shared.model.OBSqlPlan; import com.oceanbase.odc.core.shared.model.PlanNode; import com.oceanbase.odc.core.shared.model.SqlExecDetail; -import com.oceanbase.odc.core.shared.model.SqlExplain; +import com.oceanbase.odc.core.shared.model.SqlPlanMonitor; +import com.oceanbase.odc.core.sql.util.OBUtils; import com.oceanbase.odc.plugin.connect.api.SqlDiagnoseExtensionPoint; +import com.oceanbase.odc.plugin.connect.model.diagnose.PlanGraph; +import com.oceanbase.odc.plugin.connect.model.diagnose.SqlExplain; import com.oceanbase.odc.plugin.connect.obmysql.diagnose.DiagnoseUtil; +import com.oceanbase.odc.plugin.connect.obmysql.diagnose.PlanGraphBuilder; +import com.oceanbase.odc.plugin.connect.obmysql.diagnose.QueryProfileHelper; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -48,6 +67,7 @@ @Slf4j @Extension public class OBMySQLDiagnoseExtension implements SqlDiagnoseExtensionPoint { + private static final Pattern OPERATOR_PATTERN = Pattern.compile(".+\\|(OPERATOR +)\\|.+"); @Override public SqlExplain getExplain(Statement statement, @NonNull String sql) throws SQLException { @@ -92,6 +112,7 @@ public SqlExplain getExplain(Statement statement, @NonNull String sql) throws SQ // 设置原始的执行计划信息 explain.setOriginalText(text); explain.setShowFormatInfo(true); + explain.setGraph(getSqlPlanGraphBySql(statement, sql)); } catch (Exception e) { log.warn("Fail to parse explain result, origin plan text: {}", text, e); throw OBException.executeFailed(ErrorCodes.ObGetPlanExplainFailed, @@ -238,7 +259,7 @@ protected SqlExplain innerGetPhysicalPlan(Connection connection, String querySql @Override public SqlExecDetail getExecutionDetailById(Connection connection, @NonNull String id) throws SQLException { // v$sql_audit中对于分区表会有多条记录，需要过滤 - String appendSql = "TRACE_ID = '" + id + "' AND LENGTH(QUERY_SQL) > 0;"; + String appendSql = "TRACE_ID = '" + id + "' AND LENGTH(QUERY_SQL) > 0 AND IS_INNER_SQL = 0;"; return innerGetExecutionDetail(connection, appendSql, id); } @@ -248,6 +269,147 @@ public SqlExecDetail getExecutionDetailBySql(Connection connection, @NonNull Str return innerGetExecutionDetail(connection, appendSql, null); } + @Override + public SqlExplain getQueryProfileByTraceId(Connection connection, @NonNull String traceId) throws SQLException { + try (Statement stmt = connection.createStatement()) { + String planId = getPlanIdByTraceId(stmt, traceId); + Verify.notEmpty(planId, "plan id"); + PlanGraph graph = getPlanGraph(stmt, planId); + graph.setTraceId(traceId); + QueryProfileHelper.refreshGraph(graph, getSqlPlanMonitorRecords(stmt, traceId)); + + try { + SqlExecDetail execDetail = getExecutionDetailById(connection, traceId); + Verify.notNull(execDetail, "exec detail"); + graph.putOverview(QUEUE_TIME, + TimespanFormatUtil.formatTimespan(execDetail.getQueueTime(), TimeUnit.MICROSECONDS)); + graph.putOverview(PLAN_TYPE, execDetail.getPlanType()); + graph.putOverview(IS_HIT_PLAN_CACHE, execDetail.isHitPlanCache() + ""); + if (graph.getDuration() == 0) { + graph.setDuration(execDetail.getExecTime()); + graph.putOverview(DB_TIME, + TimespanFormatUtil.formatTimespan(execDetail.getExecTime(), TimeUnit.MICROSECONDS)); + } + } catch (Exception e) { + log.warn("Failed to query sql audit with OB trace_id={}.", traceId, e); + } + SqlExplain explain = innerGetSqlExplainByDbmsXplan(stmt, planId); + explain.setGraph(graph); + return explain; + } + } + + protected String getPlanIdByTraceId(Statement stmt, String traceId) throws SQLException { + try { + return OBUtils.queryPlanIdByTraceIdFromASH(stmt, traceId, ConnectType.OB_MYSQL); + } catch (SQLException e) { + return OBUtils.queryPlanIdByTraceIdFromAudit(stmt, traceId, ConnectType.OB_MYSQL); + } + } + + protected String getPhysicalPlanByDbmsXplan(Statement stmt, String planId) throws SQLException { + try (ResultSet rs = stmt.executeQuery("select dbms_xplan.display_cursor(" + planId + ")")) { + if (!rs.next()) { + throw new SQLException("Failed to query plan by dbms_xplan.display_cursor"); + } + return rs.getString(1); + } + } + + protected PlanGraph getPlanGraph(Statement stmt, String planId) throws SQLException { + List planRecords = OBUtils.queryOBSqlPlanByPlanId(stmt, planId, ConnectType.OB_MYSQL); + return PlanGraphBuilder.buildPlanGraph(planRecords); + } + + protected List getSqlPlanMonitorRecords(Statement stmt, String traceId) throws SQLException { + Map statId2Name = OBUtils.querySPMStatNames(stmt, ConnectType.OB_MYSQL); + return OBUtils.querySqlPlanMonitorStats( + stmt, traceId, ConnectType.OB_MYSQL, statId2Name); + } + + /** + * 
+     *     ===========================================================================================================
+     *     |ID|OPERATOR              |NAME    |EST.ROWS|EST.TIME(us)|REAL.ROWS|REAL.TIME(us)|IO TIME(us)|CPU TIME(us)|
+     *     -----------------------------------------------------------------------------------------------------------
+     *     |0 |PX COORDINATOR MERGE  |        |697     |50767       |2        |545699       |509932     |546311      |
+     *     |1 |+-EXCHANGE OUT DISTR  |:EX10007|697     |50370       |2        |543539       |0          |59          |
+     *     |2 |  +-SORT              |        |697     |50259       |2        |543539       |0          |78          |
+     *     |9 |  | +-SHARED HASH JOIN|        |2570    |47498       |2549     |525890       |0          |7917        |
+     *     ===========================================================================================================
+     *     Outputs & filters:
+     *     -------------------------------------
+     *       0 - output...
+     *
+ */ + private SqlExplain innerGetSqlExplainByDbmsXplan(Statement stmt, String planId) throws SQLException { + SqlExplain sqlExplain = new SqlExplain(); + sqlExplain.setShowFormatInfo(true); + + String planText = getPhysicalPlanByDbmsXplan(stmt, planId); + sqlExplain.setOriginalText(planText); + String[] segs = planText.split("Outputs & filters")[0].split("\n"); + String headerLine = segs[1]; + Matcher matcher = OPERATOR_PATTERN.matcher(headerLine); + if (!matcher.matches()) { + throw new UnexpectedException("Invalid explain:" + planText); + } + int operatorStartIndex = matcher.start(1); + int operatorStrLen = matcher.end(1) - operatorStartIndex; + + PlanNode tree = null; + for (int i = 0; i < segs.length - 4; i++) { + PlanNode node = new PlanNode(); + node.setId(i); + + String line = segs[i + 3]; + String temp = line.substring(operatorStartIndex); + String operatorStr = temp.substring(0, operatorStrLen); + DiagnoseUtil.recognizeNodeDepthAndOperator(node, operatorStr); + + String[] others = temp.substring(operatorStrLen).split("\\|"); + node.setName(others[1]); + node.setRowCount(others[2]); + node.setCost(others[3]); + node.setRealRowCount(others[4]); + node.setRealCost(others[5]); + + PlanNode tmpPlanNode = DiagnoseUtil.buildPlanTree(tree, node); + if (tree == null) { + tree = tmpPlanNode; + } + + } + sqlExplain.setExpTree(JsonUtils.toJson(tree)); + return sqlExplain; + } + + private PlanGraph getSqlPlanGraphBySql(Statement statement, @NonNull String sql) throws SQLException { + String explain = "explain format=json " + sql; + StringBuilder planJson = new StringBuilder(); + try (ResultSet rs = statement.executeQuery(explain)) { + while (rs.next()) { + planJson.append(rs.getString(1)); + } + } + explain = "explain " + sql; + List queryPlan = new ArrayList<>(); + try (ResultSet rs = statement.executeQuery(explain)) { + while (rs.next()) { + queryPlan.add(rs.getString(1).trim()); + } + } + String planText = String.join("\n", queryPlan); + String[] segs = planText.split("Outputs & filters"); + String[] outputSegs = segs[1].split("Used Hint")[0].split("[0-9]+ - output"); + Map outputFilters = new HashMap<>(); + for (int i = 1; i < outputSegs.length; i++) { + outputFilters.put(i - 1 + "", "output" + outputSegs[i]); + } + return PlanGraphBuilder.buildPlanGraph( + JsonUtils.fromJsonMap(planJson.toString(), String.class, Object.class), outputFilters); + } + protected SqlExecDetail innerGetExecutionDetail(Connection connection, String appendSql, String traceId) throws SQLException { OBMySQLInformationExtension informationExtension = new OBMySQLInformationExtension(); @@ -270,5 +432,4 @@ protected SqlExecDetail innerGetExecutionDetail(Connection connection, String ap String.format("Failed to get execution detail, traceId=%s, message=%s", traceId, e.getMessage())); } } - } diff --git a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/DiagnoseUtil.java b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/DiagnoseUtil.java index 5e87c451c8..312f1423b4 100644 --- a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/DiagnoseUtil.java +++ b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/DiagnoseUtil.java @@ -86,10 +86,27 @@ public static int recognizeNodeDepth(String operator) { return depth; } + public static void recognizeNodeDepthAndOperator(PlanNode node, String operator) { + int index = operator.indexOf("+-"); + if (index != -1) { + node.setDepth(index / 2); + node.setOperator(operator.substring(index + 2).trim()); + return; + } + index = operator.indexOf("|-"); + if (index != -1) { + node.setDepth(index / 2); + node.setOperator(operator.substring(index + 2).trim()); + return; + } + node.setDepth(-1); + node.setOperator(operator.trim()); + } + public static SqlExecDetail toSQLExecDetail(ResultSet resultSet) throws SQLException { SqlExecDetail detail = new SqlExecDetail(); if (!resultSet.next()) { - return detail; + throw new IllegalStateException("No result found in sql audit."); } detail.setTraceId(resultSet.getString(1)); detail.setSqlId(resultSet.getString(2)); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/helper/PlanGraphBuilder.java b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/PlanGraphBuilder.java similarity index 54% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/helper/PlanGraphBuilder.java rename to server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/PlanGraphBuilder.java index 5a6b376b9d..07a79aa513 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/helper/PlanGraphBuilder.java +++ b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/PlanGraphBuilder.java @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.queryprofile.helper; +package com.oceanbase.odc.plugin.connect.obmysql.diagnose; + +import static java.util.Collections.singletonList; -import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedList; @@ -24,13 +25,13 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.oceanbase.odc.common.graph.Graph; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.core.shared.exception.UnexpectedException; -import com.oceanbase.odc.service.queryprofile.model.OBSqlPlan; -import com.oceanbase.odc.service.queryprofile.model.Operator; -import com.oceanbase.odc.service.queryprofile.model.PredicateKey; -import com.oceanbase.odc.service.queryprofile.model.SqlPlanGraph; -import com.oceanbase.odc.service.queryprofile.model.SqlProfile.Status; +import com.oceanbase.odc.core.shared.model.OBSqlPlan; +import com.oceanbase.odc.core.shared.model.Operator; +import com.oceanbase.odc.core.shared.model.QueryStatus; +import com.oceanbase.odc.plugin.connect.model.diagnose.PlanGraph; import lombok.extern.slf4j.Slf4j; @@ -40,17 +41,17 @@ */ @Slf4j public class PlanGraphBuilder { - private static final Pattern PARAMETER_PATTERN = Pattern.compile("^\\s+(:\\d+) => ('.+')&"); + private static final Pattern PARAMETER_PATTERN = Pattern.compile("\\s+(:\\d+) => (.+)"); private static final Pattern VALUE_GROUP_PATTERN = Pattern.compile("\\[([^]]+)]"); private static final String EMPTY_PREDICATE = "nil"; - public static SqlPlanGraph buildPlanGraph(List records) { - SqlPlanGraph graph = new SqlPlanGraph(); + public static PlanGraph buildPlanGraph(List records) { + Graph graph = new Graph(); Map map = new HashMap<>(); Map parameters = new HashMap<>(); for (OBSqlPlan record : records) { Operator operator = parseResult(record, parameters); - operator.setStatus(Status.PREPARING); + operator.setStatus(QueryStatus.PREPARING); graph.insertVertex(operator); map.put(record.getId(), operator); if ("-1".equals(record.getParentId())) { @@ -62,32 +63,74 @@ public static SqlPlanGraph buildPlanGraph(List records) { } graph.insertEdge(map.get(record.getParentId()), operator, 0f); } - return graph; + return PlanGraphMapper.toVO(graph); + } + + /** + * build by query plan in json format + */ + public static PlanGraph buildPlanGraph(Map map, Map outputFilters) { + Graph graph = new Graph(); + parsePlanByJsonMap(map, graph, new HashMap<>(), outputFilters, "-1"); + return PlanGraphMapper.toVO(graph); + } + + private static void parsePlanByJsonMap(Map jsonMap, Graph graph, + Map id2Operator, Map outputFilter, String parentId) { + String id = Integer.toString((int) jsonMap.get("ID")); + Operator operator = new Operator(id, (String) jsonMap.get("OPERATOR")); + graph.insertVertex(operator); + id2Operator.put(operator.getGraphId(), operator); + if (!"-1".equals(parentId)) { + graph.insertEdge(id2Operator.get(parentId), operator, (int) jsonMap.get("EST.ROWS")); + } + operator.setStatus(QueryStatus.PREPARING); + String name = (String) jsonMap.get("NAME"); + if (StringUtils.isNotEmpty(name)) { + operator.setTitle(name); + operator.setAttribute("Object name", singletonList(name)); + } + String durationKey = jsonMap.containsKey("EST.TIME(us)") ? "EST.TIME(us)" : "COST"; + long dbTime = (int) jsonMap.get(durationKey); + operator.setDuration(dbTime); + operator.getOverview().put(durationKey, dbTime + ""); + Map> special = parsePredicates(outputFilter.get(id), new HashMap<>()); + operator.getAttributes().putAll(special); + jsonMap.entrySet().stream() + .filter(entry -> entry.getKey().startsWith("CHILD_")) + .forEach(entry -> parsePlanByJsonMap((Map) entry.getValue(), graph, id2Operator, outputFilter, id)); } private static Operator parseResult(OBSqlPlan record, Map parameters) { Operator operator = new Operator(record.getId(), record.getOperator()); // set object info String objectName = parseObjectName(record); - operator.setTitle(objectName); - operator.setAttribute("Full object name", objectName); - operator.setAttribute("Alias", record.getObjectAlias()); + if (StringUtils.isNotEmpty(objectName)) { + operator.setTitle(objectName); + operator.setAttribute("Full object name", singletonList(objectName)); + } + if (StringUtils.isNotEmpty(record.getObjectAlias()) + && StringUtils.equals(record.getObjectAlias(), record.getObjectName())) { + operator.setAttribute("Alias", singletonList(record.getObjectAlias())); + } // init parameters if (StringUtils.isNotEmpty(record.getOther()) && parameters.isEmpty()) { parseParameters(record.getOther(), parameters); } // parse access predicates - if (record.getAccessPredicates() != null) { + if (StringUtils.isNotEmpty(record.getAccessPredicates())) { Map> access = parsePredicates(record.getAccessPredicates(), parameters); - operator.setAttribute("Access predicates", String.join(",", access.get("access"))); + operator.setAttribute("Access predicates", + singletonList(String.join(",", access.get("access")))); } // parse filter predicates - if (record.getFilterPredicates() != null) { + if (StringUtils.isNotEmpty(record.getFilterPredicates())) { Map> filter = parsePredicates(record.getFilterPredicates(), parameters); - operator.setAttribute("Filter predicates", String.join(" AND ", filter.get("filter"))); + operator.setAttribute("Filter predicates", + singletonList(String.join(" AND ", filter.get("filter")))); } // parse special predicates - if (record.getSpecialPredicates() != null) { + if (StringUtils.isNotEmpty(record.getSpecialPredicates())) { Map> special = parsePredicates(record.getSpecialPredicates(), parameters); operator.getAttributes().putAll(special); } @@ -97,6 +140,9 @@ private static Operator parseResult(OBSqlPlan record, Map parame public static Map> parsePredicates(String predicates, Map parameters) { Map> map = new LinkedHashMap<>(); + if (StringUtils.isEmpty(predicates)) { + return map; + } int depth = 0; char[] cs = predicates.toCharArray(); StringBuilder keyBuilder = new StringBuilder(); @@ -105,22 +151,31 @@ public static Map> parsePredicates(String predicates, Map values = new LinkedList<>(); Matcher matcher = VALUE_GROUP_PATTERN.matcher(predicate); while (matcher.find()) { @@ -128,13 +183,14 @@ public static Map> parsePredicates(String predicates, Map parameters private static String parseObjectName(OBSqlPlan record) { String name = record.getObjectName(); - return record.getObjectOwner() == null ? name : String.format("%s.%s", record.getObjectOwner(), name); + return StringUtils.isEmpty(record.getObjectOwner()) ? name + : String.format("%s.%s", record.getObjectOwner(), name); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/helper/PlanGraphMapper.java b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/PlanGraphMapper.java similarity index 68% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/helper/PlanGraphMapper.java rename to server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/PlanGraphMapper.java index 1c749fb54b..7d250c5b0a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/helper/PlanGraphMapper.java +++ b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/PlanGraphMapper.java @@ -13,16 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.queryprofile.helper; +package com.oceanbase.odc.plugin.connect.obmysql.diagnose; +import java.util.Map; import java.util.stream.Collectors; +import com.oceanbase.odc.common.graph.Graph; import com.oceanbase.odc.common.graph.GraphEdge; -import com.oceanbase.odc.service.queryprofile.display.PlanGraph; -import com.oceanbase.odc.service.queryprofile.display.PlanGraphEdge; -import com.oceanbase.odc.service.queryprofile.display.PlanGraphOperator; -import com.oceanbase.odc.service.queryprofile.model.Operator; -import com.oceanbase.odc.service.queryprofile.model.SqlPlanGraph; +import com.oceanbase.odc.core.shared.model.Operator; +import com.oceanbase.odc.plugin.connect.model.diagnose.PlanGraph; +import com.oceanbase.odc.plugin.connect.model.diagnose.PlanGraphEdge; +import com.oceanbase.odc.plugin.connect.model.diagnose.PlanGraphOperator; /** * @author liuyizhuo.lyz @@ -30,14 +31,10 @@ */ public class PlanGraphMapper { - public static PlanGraph toVO(SqlPlanGraph graph) { + public static PlanGraph toVO(Graph graph) { PlanGraph vo = new PlanGraph(); - vo.setOverview(graph.getOverview()); - vo.setStatistics(graph.getStatistics()); vo.setVertexes(graph.getVertexList().stream() .map(vertex -> mapVertex((Operator) vertex)).collect(Collectors.toList())); - vo.setEdges(graph.getEdgeList().stream() - .map(PlanGraphMapper::mapEdge).collect(Collectors.toList())); return vo; } @@ -55,9 +52,12 @@ private static PlanGraphOperator mapVertex(Operator vertex) { vo.setName(vertex.getName()); vo.setTitle(vertex.getTitle()); vo.setStatus(vertex.getStatus()); - vo.setAttributes(vertex.getAttributes()); + vo.setDuration(vertex.getDuration()); + vo.setAttributes((Map) vertex.getAttributes()); vo.setOverview(vertex.getOverview()); vo.setStatistics(vertex.getStatistics()); + vo.setInEdges(vertex.getInEdges().stream().map(PlanGraphMapper::mapEdge).collect(Collectors.toList())); + vo.setOutEdges(vertex.getOutEdges().stream().map(PlanGraphMapper::mapEdge).collect(Collectors.toList())); return vo; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/helper/PlanParameterSubstitutor.java b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/PlanParameterSubstitutor.java similarity index 96% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/helper/PlanParameterSubstitutor.java rename to server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/PlanParameterSubstitutor.java index 005b6014ae..178cd9b764 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/helper/PlanParameterSubstitutor.java +++ b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/PlanParameterSubstitutor.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.queryprofile.helper; +package com.oceanbase.odc.plugin.connect.obmysql.diagnose; import java.util.Map; diff --git a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/ProfileConstants.java b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/ProfileConstants.java new file mode 100644 index 0000000000..1bdea88bdc --- /dev/null +++ b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/ProfileConstants.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.plugin.connect.obmysql.diagnose; + +public class ProfileConstants { + + + public static final String DB_TIME = "DB time"; + public static final String CHANGE_TIME = "Change time"; + public static final String QUEUE_TIME = "Queue time"; + public static final String PARALLEL = "Parallel"; + public static final String PROCESS_NAME = "Process name"; + public static final String SKEWNESS = "Skewness"; + public static final String PLAN_TYPE = "Plan type"; + public static final String IS_HIT_PLAN_CACHE = "Is hit plan cache"; + + public static final String CPU_TIME = "CPU time"; + public static final String IO_WAIT_TIME = "I/O wait time"; + + public static final String OUTPUT_ROWS = "Output rows"; + public static final String IO_READ_BYTES = "total io bytes read from disk"; + public static final String BYTES_IN_TOTAL = "total bytes processed by ssstore"; + public static final String SSSTORE_ROWS_IN_TOTAL = "total rows processed by ssstore"; + public static final String MEMSTORE_ROWS_IN_TOTAL = "total rows processed by memstore"; + public static final String START_TIMES = "Start times"; + public static final String WORKAREA_MAX_MEN = "Max memory"; + public static final String WORKAREA_MAX_TEMPSEG = "Max disk"; + + public static final String OTHER_STATS = "Other Statistics"; + +} diff --git a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/QueryProfileHelper.java b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/QueryProfileHelper.java new file mode 100644 index 0000000000..6a22824472 --- /dev/null +++ b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/QueryProfileHelper.java @@ -0,0 +1,358 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.plugin.connect.obmysql.diagnose; + +import static com.oceanbase.odc.plugin.connect.obmysql.diagnose.ProfileConstants.BYTES_IN_TOTAL; +import static com.oceanbase.odc.plugin.connect.obmysql.diagnose.ProfileConstants.CHANGE_TIME; +import static com.oceanbase.odc.plugin.connect.obmysql.diagnose.ProfileConstants.CPU_TIME; +import static com.oceanbase.odc.plugin.connect.obmysql.diagnose.ProfileConstants.DB_TIME; +import static com.oceanbase.odc.plugin.connect.obmysql.diagnose.ProfileConstants.IO_READ_BYTES; +import static com.oceanbase.odc.plugin.connect.obmysql.diagnose.ProfileConstants.IO_WAIT_TIME; +import static com.oceanbase.odc.plugin.connect.obmysql.diagnose.ProfileConstants.MEMSTORE_ROWS_IN_TOTAL; +import static com.oceanbase.odc.plugin.connect.obmysql.diagnose.ProfileConstants.OTHER_STATS; +import static com.oceanbase.odc.plugin.connect.obmysql.diagnose.ProfileConstants.OUTPUT_ROWS; +import static com.oceanbase.odc.plugin.connect.obmysql.diagnose.ProfileConstants.PARALLEL; +import static com.oceanbase.odc.plugin.connect.obmysql.diagnose.ProfileConstants.PROCESS_NAME; +import static com.oceanbase.odc.plugin.connect.obmysql.diagnose.ProfileConstants.SKEWNESS; +import static com.oceanbase.odc.plugin.connect.obmysql.diagnose.ProfileConstants.SSSTORE_ROWS_IN_TOTAL; +import static com.oceanbase.odc.plugin.connect.obmysql.diagnose.ProfileConstants.START_TIMES; +import static com.oceanbase.odc.plugin.connect.obmysql.diagnose.ProfileConstants.WORKAREA_MAX_MEN; +import static com.oceanbase.odc.plugin.connect.obmysql.diagnose.ProfileConstants.WORKAREA_MAX_TEMPSEG; + +import java.sql.Timestamp; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Collectors; + +import org.apache.commons.collections4.MapUtils; + +import com.google.common.collect.Comparators; +import com.oceanbase.odc.common.util.TimespanFormatUtil; +import com.oceanbase.odc.core.shared.Verify; +import com.oceanbase.odc.core.shared.model.QueryStatus; +import com.oceanbase.odc.core.shared.model.SqlPlanMonitor; +import com.oceanbase.odc.plugin.connect.model.diagnose.PlanGraph; +import com.oceanbase.odc.plugin.connect.model.diagnose.PlanGraphEdge; +import com.oceanbase.odc.plugin.connect.model.diagnose.PlanGraphOperator; + +import lombok.extern.slf4j.Slf4j; + +/** + * @author: liuyizhuo.lyz + * @date: 2024/6/4 + */ +@Slf4j +public class QueryProfileHelper { + + public static void refreshGraph(PlanGraph graph, List spmStats) { + String traceId = graph.getTraceId(); + try { + replaceSPMStatsIntoProfile(spmStats, graph); + } catch (Exception e) { + log.warn("Failed to get runtime statistics with OB trace_id={}.", traceId, e); + } + + Map> topNodes = new HashMap<>(); + graph.setTopNodes(topNodes); + topNodes.put("duration", graph.getVertexes() + .stream().filter(o -> o.getDuration() != null) + .collect(Comparators.greatest(5, Comparator.comparingLong(PlanGraphOperator::getDuration))) + .stream().map(PlanGraphOperator::getGraphId).collect(Collectors.toList())); + } + + private static void replaceSPMStatsIntoProfile(List records, PlanGraph graph) { + boolean needFilter = false; + boolean rootOperatorExists = false; + Timestamp startTs = new Timestamp(0); + // some inner sql of ob would share trace id with user's query, so we filter out these dirty records + // by first_refresh_time + for (SqlPlanMonitor spm : records) { + if (!"0".equals(spm.getPlanLineId())) { + continue; + } + if (rootOperatorExists) { + needFilter = true; + } else { + rootOperatorExists = true; + } + Verify.notNull(spm.getFirstRefreshTime(), "firstRefreshTime"); + if (startTs.compareTo(spm.getFirstRefreshTime()) < 1) { + startTs = spm.getFirstRefreshTime(); + } + } + + Map> planLineId2Spm = new TreeMap<>(Comparator.comparingInt(Integer::parseInt)); + for (SqlPlanMonitor spm : records) { + if (!needFilter || spm.getFirstRefreshTime() == null || startTs.compareTo(spm.getFirstRefreshTime()) < 1) { + List list = planLineId2Spm.computeIfAbsent(spm.getPlanLineId(), s -> new ArrayList<>()); + list.add(spm); + } + } + + if (planLineId2Spm.get("0").size() != 1) { + log.warn("There are more than one root operator in v$sql_plan_monitor with trace_id={}.", + graph.getTraceId()); + } + + for (List spms : planLineId2Spm.values()) { + PlanGraphOperator operator = graph.getOperator(spms.get(0).getPlanLineId()); + try { + if (spms.size() == 1) { + SqlPlanMonitor spm = spms.get(0); + setRuntimeStatistics(spm, operator, graph); + } else { + mergeSPMRecords(spms, operator, graph); + } + } catch (Exception e) { + log.warn("Failed to set runtime statistics with OB trace_id={} and plan_line_id={}", graph.getTraceId(), + spms.get(0).getPlanLineId(), e); + } + } + + SqlPlanMonitor rootOperator = planLineId2Spm.get("0").get(0); + if (rootOperator.getLastRefreshTime() == null) { + graph.setStatus(QueryStatus.RUNNING); + Duration totalTs = Duration.between(rootOperator.getFirstRefreshTime().toInstant(), + rootOperator.getCurrentTime().toInstant()); + graph.putOverview(DB_TIME, TimespanFormatUtil.formatTimespan(totalTs)); + } else { + graph.setStatus(QueryStatus.FINISHED); + Duration totalTs = Duration.between(rootOperator.getFirstRefreshTime().toInstant(), + rootOperator.getLastRefreshTime().toInstant()); + graph.putOverview(DB_TIME, TimespanFormatUtil.formatTimespan(totalTs)); + } + } + + private static void mergeSPMRecords(List spms, PlanGraphOperator operator, PlanGraph graph) { + List subOperators = new ArrayList<>(); + long totalCPUTs = 0; + long totalIOWaitTs = 0; + long starts = 0; + float outputs = 0; + long maxMem = 0; + long maxDisk = 0; + long ioBytes = 0; + long ssstoreBytes = 0; + long ssstoreRows = 0; + long memstoreRows = 0; + long maxChangeTs = -1; + long minChangeTs = Integer.MAX_VALUE; + boolean allNodeFinished = true; + boolean allNodePreparing = true; + + for (SqlPlanMonitor spm : spms) { + PlanGraphOperator child = new PlanGraphOperator(); + Long dbTime = spm.getDbTime(); + Long userIOWaitTime = spm.getUserIOWaitTime(); + Long cpuTime = dbTime - userIOWaitTime; + totalCPUTs += cpuTime; + totalIOWaitTs += userIOWaitTime; + child.setDuration(cpuTime); + child.putOverview(CPU_TIME, cpuTime); + child.putOverview(IO_WAIT_TIME, userIOWaitTime); + child.setTitle(String.format("%s(%s)", spm.getSvrIp(), spm.getProcessName())); + child.setName(operator.getName()); + child.putOverview(PROCESS_NAME, spm.getProcessName()); + + if (spm.getLastChangeTime() != null) { + child.setStatus(QueryStatus.FINISHED); + if (spm.getFirstChangeTime() != null) { + Duration changeTime = Duration.between(spm.getFirstChangeTime().toInstant(), + spm.getLastChangeTime().toInstant()); + child.putOverview(CHANGE_TIME, TimespanFormatUtil.formatTimespan(changeTime)); + long changeTs = changeTime.toNanos() / 1000; + if (changeTs != 0) { + maxChangeTs = Math.max(changeTs, maxChangeTs); + minChangeTs = Math.min(changeTs, minChangeTs); + } + } + } else if (spm.getLastRefreshTime() != null) { + child.setStatus(QueryStatus.FINISHED); + } else { + allNodeFinished = false; + // If the operator is a calculating type like `SCALAR GROUP BY`, when it's running, the + // FIRST_CHANGE_TIME and LAST_CHANGE_TIME may be null. Then we can determine its status by db_time. + // And since there maybe some deviation, we set 100 us as the threshold for judging the status. + if (spm.getFirstChangeTime() == null && spm.getDbTime() < 100) { + child.setStatus(QueryStatus.PREPARING); + } else { + child.setStatus(QueryStatus.RUNNING); + allNodePreparing = false; + } + } + + if (allNodeFinished) { + operator.setStatus(QueryStatus.FINISHED); + } else { + if (allNodePreparing) { + operator.setStatus(QueryStatus.PREPARING); + } else { + operator.setStatus(QueryStatus.RUNNING); + } + } + + outputs += spm.getOutputRows(); + child.putStatistics(OUTPUT_ROWS, (float) spm.getOutputRows() + ""); + if (spm.getWorkareaMaxMem() != 0) { + child.putStatistics(WORKAREA_MAX_MEN, spm.getWorkareaMaxMem() + ""); + maxMem = Math.max(spm.getWorkareaMaxMem(), maxMem); + } + if (spm.getWorkareaMaxTempSeg() != 0) { + child.putStatistics(WORKAREA_MAX_TEMPSEG, spm.getWorkareaMaxTempSeg() + ""); + maxDisk = Math.max(spm.getWorkareaMaxTempSeg(), maxDisk); + } + if (spm.getStarts() != 0) { + child.putStatistics(START_TIMES, spm.getStarts() + ""); + starts += spm.getStarts(); + } + Map otherstats = spm.getOtherstats(); + if (MapUtils.isNotEmpty(otherstats)) { + child.putAttribute(OTHER_STATS, otherstats.entrySet() + .stream().map(entry -> String.format("%s : %s", entry.getKey(), entry.getValue())) + .collect(Collectors.toList())); + } + if (otherstats.containsKey(IO_READ_BYTES)) { + child.putStatistics(IO_READ_BYTES, otherstats.get(IO_READ_BYTES)); + ioBytes += Long.parseLong(otherstats.get(IO_READ_BYTES)); + } + if (otherstats.containsKey(BYTES_IN_TOTAL)) { + child.putStatistics(BYTES_IN_TOTAL, otherstats.get(BYTES_IN_TOTAL)); + ssstoreBytes += Long.parseLong(otherstats.get(BYTES_IN_TOTAL)); + } + if (otherstats.containsKey(SSSTORE_ROWS_IN_TOTAL)) { + child.putStatistics(SSSTORE_ROWS_IN_TOTAL, otherstats.get(SSSTORE_ROWS_IN_TOTAL)); + ssstoreRows += Long.parseLong(otherstats.get(SSSTORE_ROWS_IN_TOTAL)); + } + if (otherstats.containsKey(MEMSTORE_ROWS_IN_TOTAL)) { + child.putStatistics(MEMSTORE_ROWS_IN_TOTAL, otherstats.get(MEMSTORE_ROWS_IN_TOTAL)); + memstoreRows += Long.parseLong(otherstats.get(MEMSTORE_ROWS_IN_TOTAL)); + } + subOperators.add(child); + } + + if (ioBytes > 0 || ssstoreBytes > 0 || ssstoreRows > 0 || memstoreRows > 0) { + operator.putStatistics(IO_READ_BYTES, ioBytes + ""); + operator.putStatistics(BYTES_IN_TOTAL, ssstoreBytes + ""); + operator.putStatistics(SSSTORE_ROWS_IN_TOTAL, ssstoreRows + ""); + operator.putStatistics(MEMSTORE_ROWS_IN_TOTAL, memstoreRows + ""); + graph.addStatistics(IO_READ_BYTES, ioBytes + ""); + graph.addStatistics(BYTES_IN_TOTAL, ssstoreBytes + ""); + graph.addStatistics(SSSTORE_ROWS_IN_TOTAL, ssstoreRows + ""); + graph.addStatistics(MEMSTORE_ROWS_IN_TOTAL, memstoreRows + ""); + } + if (maxMem > 0 || maxDisk > 0) { + operator.putStatistics(WORKAREA_MAX_MEN, maxMem + ""); + operator.putStatistics(WORKAREA_MAX_TEMPSEG, maxDisk + ""); + } + if (starts > 0) { + operator.putStatistics(START_TIMES, starts + ""); + } + operator.putStatistics(OUTPUT_ROWS, outputs + ""); + + operator.setDuration(totalCPUTs); + operator.putOverview(CPU_TIME, totalCPUTs); + operator.putOverview(IO_WAIT_TIME, totalIOWaitTs); + long totalCpuTime = (long) graph.getOverview().getOrDefault(CPU_TIME, 0L) + totalCPUTs; + graph.putOverview(CPU_TIME, totalCpuTime); + graph.setDuration(totalCpuTime); + long totalIOWaitTime = (long) graph.getOverview().getOrDefault(IO_WAIT_TIME, 0L) + totalIOWaitTs; + graph.putOverview(IO_WAIT_TIME, totalIOWaitTime); + + operator.putOverview(PARALLEL, spms.size()); + if (maxChangeTs > 0) { + operator.putOverview(SKEWNESS, String.format("%.2f", (maxChangeTs - minChangeTs) * 1f / maxChangeTs)); + } + + // output rows + List inEdges = operator.getInEdges(); + if (inEdges != null && inEdges.size() == 1) { + inEdges.get(0).setWeight(outputs); + } + + subOperators.sort(Comparator.comparingLong(PlanGraphOperator::getDuration).reversed()); + Map map = new LinkedHashMap<>(); + for (PlanGraphOperator o : subOperators) { + map.put(o.getTitle(), o); + } + operator.setSubNodes(map); + } + + private static void setRuntimeStatistics(SqlPlanMonitor stats, PlanGraphOperator operator, PlanGraph graph) { + // overview + Long dbTime = stats.getDbTime(); + Long userIOWaitTime = stats.getUserIOWaitTime(); + Long cpuTime = dbTime - userIOWaitTime; + operator.setDuration(cpuTime); + operator.putOverview(CPU_TIME, cpuTime); + long totalCpuTime = (long) graph.getOverview().getOrDefault(CPU_TIME, 0L) + cpuTime; + graph.putOverview(CPU_TIME, totalCpuTime); + graph.setDuration(totalCpuTime); + operator.putOverview(IO_WAIT_TIME, userIOWaitTime); + long totalIOWaitTime = (long) graph.getOverview().getOrDefault(IO_WAIT_TIME, 0L) + userIOWaitTime; + graph.putOverview(IO_WAIT_TIME, totalIOWaitTime); + if (stats.getLastChangeTime() != null) { + operator.setStatus(QueryStatus.FINISHED); + if (stats.getFirstChangeTime() != null) { + Duration changeTime = Duration.between(stats.getFirstChangeTime().toInstant(), + stats.getLastChangeTime().toInstant()); + operator.putOverview(CHANGE_TIME, TimespanFormatUtil.formatTimespan(changeTime)); + } + } else if (stats.getLastRefreshTime() != null) { + operator.setStatus(QueryStatus.FINISHED); + } else { + if (stats.getFirstChangeTime() == null && stats.getDbTime() < 100) { + operator.setStatus(QueryStatus.PREPARING); + } else { + operator.setStatus(QueryStatus.RUNNING); + } + } + // statistics + if (stats.getWorkareaMaxMem() != 0) { + operator.putStatistics(WORKAREA_MAX_MEN, stats.getWorkareaMaxMem() + ""); + } + if (stats.getWorkareaMaxTempSeg() != 0) { + operator.putStatistics(WORKAREA_MAX_TEMPSEG, stats.getWorkareaMaxTempSeg() + ""); + } + if (stats.getStarts() != 0) { + operator.putStatistics(START_TIMES, stats.getStarts() + ""); + } + operator.putStatistics(IO_READ_BYTES, stats.getOtherstats().get(IO_READ_BYTES)); + operator.putStatistics(BYTES_IN_TOTAL, stats.getOtherstats().get(BYTES_IN_TOTAL)); + operator.putStatistics(SSSTORE_ROWS_IN_TOTAL, stats.getOtherstats().get(SSSTORE_ROWS_IN_TOTAL)); + operator.putStatistics(MEMSTORE_ROWS_IN_TOTAL, stats.getOtherstats().get(MEMSTORE_ROWS_IN_TOTAL)); + graph.addStatistics(IO_READ_BYTES, stats.getOtherstats().get(IO_READ_BYTES)); + graph.addStatistics(BYTES_IN_TOTAL, stats.getOtherstats().get(BYTES_IN_TOTAL)); + graph.addStatistics(SSSTORE_ROWS_IN_TOTAL, stats.getOtherstats().get(SSSTORE_ROWS_IN_TOTAL)); + graph.addStatistics(MEMSTORE_ROWS_IN_TOTAL, stats.getOtherstats().get(MEMSTORE_ROWS_IN_TOTAL)); + // attributes + if (MapUtils.isNotEmpty(stats.getOtherstats())) { + operator.putAttribute(OTHER_STATS, stats.getOtherstats().entrySet() + .stream().map(entry -> String.format("%s : %s", entry.getKey(), entry.getValue())) + .collect(Collectors.toList())); + } + // output rows + List inEdges = operator.getInEdges(); + if (inEdges != null && inEdges.size() == 1) { + inEdges.get(0).setWeight(Float.valueOf(stats.getOutputRows())); + } + } +} diff --git a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/initializer/EnablePlanMonitorInitializer.java b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/initializer/EnablePlanMonitorInitializer.java new file mode 100644 index 0000000000..c4c9a428d7 --- /dev/null +++ b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/initializer/EnablePlanMonitorInitializer.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.plugin.connect.obmysql.initializer; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + +import com.oceanbase.odc.common.util.VersionUtils; +import com.oceanbase.odc.core.datasource.ConnectionInitializer; +import com.oceanbase.odc.core.sql.util.OBUtils; + +import lombok.extern.slf4j.Slf4j; + +/** + * @author: liuyizhuo.lyz + * @date: 2024/5/20 + */ +@Slf4j +public class EnablePlanMonitorInitializer implements ConnectionInitializer { + @Override + public void init(Connection connection) throws SQLException { + String version = OBUtils.getObVersion(connection); + if (VersionUtils.isLessThan(version, "4.2.4")) { + return; + } + try (Statement statement = connection.createStatement()) { + statement.execute("set enable_sql_plan_monitor='ON'"); + } catch (Exception e) { + log.warn("Enable enable_sql_plan_monitor failed, errMsg={}", e.getMessage()); + } + } +} diff --git a/server/plugins/connect-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLExtensionTest.java b/server/plugins/connect-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLExtensionTest.java index 5308e3d010..a4c16f7edd 100644 --- a/server/plugins/connect-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLExtensionTest.java +++ b/server/plugins/connect-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLExtensionTest.java @@ -34,7 +34,6 @@ import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.constant.OdcConstants; import com.oceanbase.odc.core.shared.model.SqlExecDetail; -import com.oceanbase.odc.core.shared.model.SqlExplain; import com.oceanbase.odc.plugin.connect.api.ConnectionExtensionPoint; import com.oceanbase.odc.plugin.connect.api.InformationExtensionPoint; import com.oceanbase.odc.plugin.connect.api.SessionExtensionPoint; @@ -42,6 +41,7 @@ import com.oceanbase.odc.plugin.connect.api.TestResult; import com.oceanbase.odc.plugin.connect.model.ConnectionPropertiesBuilder; import com.oceanbase.odc.plugin.connect.model.JdbcUrlProperty; +import com.oceanbase.odc.plugin.connect.model.diagnose.SqlExplain; import com.oceanbase.odc.test.database.TestDBConfiguration; import com.oceanbase.odc.test.database.TestDBConfigurations; import com.oceanbase.odc.test.util.FileUtil; diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/queryprofile/PlanGraphBuilderTest.java b/server/plugins/connect-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/connect/obmysql/queryprofile/PlanGraphBuilderTest.java similarity index 52% rename from server/odc-service/src/test/java/com/oceanbase/odc/service/queryprofile/PlanGraphBuilderTest.java rename to server/plugins/connect-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/connect/obmysql/queryprofile/PlanGraphBuilderTest.java index bc82f9c24e..81753a400b 100644 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/queryprofile/PlanGraphBuilderTest.java +++ b/server/plugins/connect-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/connect/obmysql/queryprofile/PlanGraphBuilderTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.queryprofile; +package com.oceanbase.odc.plugin.connect.obmysql.queryprofile; import java.util.Arrays; import java.util.LinkedHashMap; @@ -24,11 +24,12 @@ import org.junit.BeforeClass; import org.junit.Test; -import com.oceanbase.odc.common.graph.GraphVertex; -import com.oceanbase.odc.service.queryprofile.helper.PlanGraphBuilder; -import com.oceanbase.odc.service.queryprofile.model.OBSqlPlan; -import com.oceanbase.odc.service.queryprofile.model.PredicateKey; -import com.oceanbase.odc.service.queryprofile.model.SqlPlanGraph; +import com.google.common.collect.ImmutableMap; +import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.core.shared.model.OBSqlPlan; +import com.oceanbase.odc.plugin.connect.model.diagnose.PlanGraph; +import com.oceanbase.odc.plugin.connect.model.diagnose.PlanGraphOperator; +import com.oceanbase.odc.plugin.connect.obmysql.diagnose.PlanGraphBuilder; /** * @author liuyizhuo.lyz @@ -48,8 +49,8 @@ public static void setUp() { @Test public void test_BuildPlanGraph_BuildSuccess() { List records = getPlanRecords(); - SqlPlanGraph graph = PlanGraphBuilder.buildPlanGraph(records); - List vertices = graph.getTopoOrderedVertices(); + PlanGraph graph = PlanGraphBuilder.buildPlanGraph(records); + List vertices = graph.getVertexes(); Assert.assertEquals(vertices.size(), records.size()); } @@ -57,7 +58,7 @@ public void test_BuildPlanGraph_BuildSuccess() { public void test_ParseAccessPredicates() { String accessPredicates = "access([ORDERS.O_ORDERKEY], [ORDERS.O_CUSTKEY])"; Map> access = PlanGraphBuilder.parsePredicates(accessPredicates, PARAMS); - List columns = access.get(PredicateKey.getLabel("access")); + List columns = access.get("access"); Assert.assertEquals(Arrays.asList("ORDERS.O_ORDERKEY", "ORDERS.O_CUSTKEY"), columns); } @@ -65,12 +66,47 @@ public void test_ParseAccessPredicates() { public void test_ParseFilterPredicates() { String filterPredicates = "filter([ORDERS.O_ORDERDATE >= :3], [ORDERS.O_ORDERDATE <= :4])"; Map> filter = PlanGraphBuilder.parsePredicates(filterPredicates, PARAMS); - List colums = filter.get(PredicateKey.getLabel("filter")); + List colums = filter.get("filter"); Assert.assertEquals(Arrays.asList( "ORDERS.O_ORDERDATE >= '1995-01-01 00:00:00'", "ORDERS.O_ORDERDATE <= '1996-12-30 00:00:00'"), colums); } + @Test + public void test_BuildPlanGraphByJsonMap() { + String json = "{\n" + + " \"ID\": 0,\n" + + " \"OPERATOR\": \"SORT\",\n" + + " \"NAME\": \"\",\n" + + " \"EST.ROWS\": 697,\n" + + " \"EST.TIME(us)\": 817249,\n" + + " \"CHILD_1\": {\n" + + " \"ID\": 1,\n" + + " \"OPERATOR\": \"HASH GROUP BY\",\n" + + " \"NAME\": \"\",\n" + + " \"EST.ROWS\": 697,\n" + + " \"EST.TIME(us)\": 817050\n" + + " },\n" + + " \"CHILD_2\": {\n" + + " \"ID\": 2,\n" + + " \"OPERATOR\": \"MATERIAL\",\n" + + " \"NAME\": \"\",\n" + + " \"EST.ROWS\": 25,\n" + + " \"EST.TIME(us)\": 5,\n" + + " \"output\": \"output([N2.N_NAME])\"\n" + + " }\n" + + "}"; + Map outputFilters = ImmutableMap.of("0", + "output([ORDERS.O_ORDERKEY], [ORDERS.O_ORDERDATE], [ORDERS.O_CUSTKEY]), filter([ORDERS.O_ORDERDATE >= '1995-01-01 00:00:00'], [ORDERS.O_ORDERDATE <= \n" + + " '1996-12-30 00:00:00']), rowset=256\n" + + " access([ORDERS.O_ORDERKEY], [ORDERS.O_ORDERDATE], [ORDERS.O_CUSTKEY]), partitions(p0)\n" + + " is_index_back=false, is_global_index=false, filter_before_indexback[false,false], \n" + + " range_key([ORDERS.O_ORDERKEY], [ORDERS.O_ORDERDATE], [ORDERS.O_CUSTKEY]), range(MIN,MIN,MIN ; MAX,MAX,MAX)always true"); + PlanGraph graph = PlanGraphBuilder.buildPlanGraph( + JsonUtils.fromJsonMap(json, String.class, Object.class), outputFilters); + Assert.assertEquals(3, graph.getVertexes().size()); + } + private List getPlanRecords() { OBSqlPlan plan1 = new OBSqlPlan(); plan1.setId("0"); diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/queryprofile/PlanParameterSubstitutorTest.java b/server/plugins/connect-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/connect/obmysql/queryprofile/PlanParameterSubstitutorTest.java similarity index 91% rename from server/odc-service/src/test/java/com/oceanbase/odc/service/queryprofile/PlanParameterSubstitutorTest.java rename to server/plugins/connect-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/connect/obmysql/queryprofile/PlanParameterSubstitutorTest.java index b881f53dad..c8cf418b50 100644 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/queryprofile/PlanParameterSubstitutorTest.java +++ b/server/plugins/connect-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/connect/obmysql/queryprofile/PlanParameterSubstitutorTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.queryprofile; +package com.oceanbase.odc.plugin.connect.obmysql.queryprofile; import java.util.Map; @@ -21,7 +21,7 @@ import org.junit.Test; import com.google.common.collect.ImmutableMap; -import com.oceanbase.odc.service.queryprofile.helper.PlanParameterSubstitutor; +import com.oceanbase.odc.plugin.connect.obmysql.diagnose.PlanParameterSubstitutor; /** * @author liuyizhuo.lyz diff --git a/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleConnectionExtension.java b/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleConnectionExtension.java index 99d728dce0..117badd996 100644 --- a/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleConnectionExtension.java +++ b/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleConnectionExtension.java @@ -24,6 +24,7 @@ import com.oceanbase.odc.core.datasource.ConnectionInitializer; import com.oceanbase.odc.plugin.connect.obmysql.OBMySQLConnectionExtension; +import com.oceanbase.odc.plugin.connect.obmysql.initializer.EnablePlanMonitorInitializer; import com.oceanbase.odc.plugin.connect.obmysql.initializer.EnableTraceInitializer; import com.oceanbase.odc.plugin.connect.oboracle.initializer.OracleDBMSOutputInitializer; @@ -43,6 +44,7 @@ public List getConnectionInitializers() { List initializers = new ArrayList<>(); initializers.add(new EnableTraceInitializer()); initializers.add(new OracleDBMSOutputInitializer()); + initializers.add(new EnablePlanMonitorInitializer()); return initializers; } diff --git a/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleDiagnoseExtension.java b/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleDiagnoseExtension.java index 01e6be6e64..cbd43392f6 100644 --- a/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleDiagnoseExtension.java +++ b/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleDiagnoseExtension.java @@ -19,18 +19,27 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.List; +import java.util.Map; import org.pf4j.Extension; import com.alibaba.fastjson.JSON; +import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.common.util.VersionUtils; +import com.oceanbase.odc.core.shared.constant.ConnectType; import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.exception.OBException; +import com.oceanbase.odc.core.shared.model.OBSqlPlan; import com.oceanbase.odc.core.shared.model.PlanNode; import com.oceanbase.odc.core.shared.model.SqlExecDetail; -import com.oceanbase.odc.core.shared.model.SqlExplain; +import com.oceanbase.odc.core.shared.model.SqlPlanMonitor; +import com.oceanbase.odc.core.sql.util.OBUtils; +import com.oceanbase.odc.plugin.connect.model.diagnose.PlanGraph; +import com.oceanbase.odc.plugin.connect.model.diagnose.SqlExplain; import com.oceanbase.odc.plugin.connect.obmysql.OBMySQLDiagnoseExtension; import com.oceanbase.odc.plugin.connect.obmysql.diagnose.DiagnoseUtil; +import com.oceanbase.odc.plugin.connect.obmysql.diagnose.PlanGraphBuilder; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -137,7 +146,7 @@ protected SqlExplain innerGetPhysicalPlan(Connection connection, String querySql @Override public SqlExecDetail getExecutionDetailById(Connection connection, @NonNull String id) throws SQLException { // v$sql_audit中对于分区表会有多条记录，需要过滤 - String appendSql = "TRACE_ID = '" + id + "' AND LENGTH(QUERY_SQL) > 0;"; + String appendSql = "TRACE_ID = '" + id + "' AND LENGTH(QUERY_SQL) > 0 AND IS_INNER_SQL = 0;"; return innerGetExecutionDetail(connection, appendSql, id); } @@ -172,4 +181,43 @@ protected SqlExecDetail innerGetExecutionDetail(Connection connection, String ap } } + @Override + protected String getPlanIdByTraceId(Statement stmt, String traceId) throws SQLException { + try { + return OBUtils.queryPlanIdByTraceIdFromASH(stmt, traceId, ConnectType.OB_ORACLE); + } catch (SQLException e) { + return OBUtils.queryPlanIdByTraceIdFromAudit(stmt, traceId, ConnectType.OB_ORACLE); + } + } + + @Override + protected String getPhysicalPlanByDbmsXplan(Statement stmt, String planId) throws SQLException { + try (ResultSet rs = + stmt.executeQuery("select VALUE from V$NLS_PARAMETERS where PARAMETER='NLS_CHARACTERSET'")) { + if (rs.next() && StringUtils.containsIgnoreCase(rs.getString(1), "GBK")) { + stmt.execute("set names gbk"); + } + } + StringBuilder builder = new StringBuilder(); + try (ResultSet rs = stmt.executeQuery("select * from TABLE(DBMS_XPLAN.DISPLAY_CURSOR(" + planId + "))")) { + while (rs.next()) { + builder.append(rs.getString(1)).append("\n"); + } + } + if (builder.length() == 0) { + throw new SQLException("Failed to query plan by dbms_xplan.display_cursor"); + } + return builder.toString(); + } + + protected PlanGraph getPlanGraph(Statement stmt, String planId) throws SQLException { + List planRecords = OBUtils.queryOBSqlPlanByPlanId(stmt, planId, ConnectType.OB_ORACLE); + return PlanGraphBuilder.buildPlanGraph(planRecords); + } + + protected List getSqlPlanMonitorRecords(Statement stmt, String traceId) throws SQLException { + Map statId2Name = OBUtils.querySPMStatNames(stmt, ConnectType.OB_ORACLE); + return OBUtils.querySqlPlanMonitorStats( + stmt, traceId, ConnectType.OB_ORACLE, statId2Name); + } } diff --git a/server/plugins/connect-plugin-ob-oracle/src/test/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleExtensionTest.java b/server/plugins/connect-plugin-ob-oracle/src/test/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleExtensionTest.java index c7a4493f53..426f0c4cdd 100644 --- a/server/plugins/connect-plugin-ob-oracle/src/test/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleExtensionTest.java +++ b/server/plugins/connect-plugin-ob-oracle/src/test/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleExtensionTest.java @@ -36,7 +36,6 @@ import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.constant.OdcConstants; import com.oceanbase.odc.core.shared.model.SqlExecDetail; -import com.oceanbase.odc.core.shared.model.SqlExplain; import com.oceanbase.odc.plugin.connect.api.ConnectionExtensionPoint; import com.oceanbase.odc.plugin.connect.api.InformationExtensionPoint; import com.oceanbase.odc.plugin.connect.api.SessionExtensionPoint; @@ -45,6 +44,7 @@ import com.oceanbase.odc.plugin.connect.api.TraceExtensionPoint; import com.oceanbase.odc.plugin.connect.model.ConnectionPropertiesBuilder; import com.oceanbase.odc.plugin.connect.model.JdbcUrlProperty; +import com.oceanbase.odc.plugin.connect.model.diagnose.SqlExplain; import com.oceanbase.odc.test.database.TestDBConfiguration; import com.oceanbase.odc.test.database.TestDBConfigurations; import com.oceanbase.odc.test.util.FileUtil; From cfbf04e4b542faafa4057e56b00a0cb53721366b Mon Sep 17 00:00:00 2001 From: XiaoYang Date: Thu, 20 Jun 2024 14:00:01 +0800 Subject: [PATCH 13/64] builds: update code owner configuration (#2785) --- .github/CODEOWNERS | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index be6a131311..5ae66039e9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -20,7 +20,7 @@ /server/odc-core/ @yhilmare @yizhouxw /server/odc-core/**/alarm/ @LuckyPickleZZ /server/odc-core/**/authority/ @yhilmare -/server/odc-core/**/datamasking/ @smallsheeeep +/server/odc-core/**/datamasking/ @LuckyPickleZZ /server/odc-core/**/datasource/ @yhilmare /server/odc-core/**/flow/ @yhilmare /server/odc-core/**/migrate/ @yhilmare @yizhouxw @@ -38,35 +38,35 @@ # service business /server/odc-service/**/service/audit/ @MarkPotato777 @yizhouxw /server/odc-service/**/service/automation/ @LuckyPickleZZ @ungreat -/server/odc-service/**/service/captcha/ @MarkPotato777 @smallsheeeep -/server/odc-service/**/service/collaboration/ @MarkPotato777 @smallsheeeep +/server/odc-service/**/service/captcha/ @MarkPotato777 @yizhouxw +/server/odc-service/**/service/collaboration/ @MarkPotato777 @yizhouxw /server/odc-service/**/service/config/ @MarkPotato777 @yizhouxw -/server/odc-service/**/service/datasecurity/ @smallsheeeep @yhilmare +/server/odc-service/**/service/datasecurity/ @LuckyPickleZZ @yhilmare /server/odc-service/**/service/datatransfer/ @LuckyPickleZZ @yhilmare /server/odc-service/**/service/db/ @PeachThinking @yhilmare /server/odc-service/**/service/diagnose/ @LuckyPickleZZ @yizhouxw /server/odc-service/**/service/dispatch/ @yhilmare @yizhouxw -/server/odc-service/**/service/dlm/ @guowl3 @smallsheeeep +/server/odc-service/**/service/dlm/ @guowl3 @kiko-art /server/odc-service/**/service/dml/ @LuckyPickleZZ @PeachThinking -/server/odc-service/**/service/encryption/ @smallsheeeep @yizhouxw +/server/odc-service/**/service/encryption/ @PeachThinking @yizhouxw /server/odc-service/**/service/feature/ @MarkPotato777 @yizhouxw /server/odc-service/**/service/flow/ @yhilmare @krihy /server/odc-service/**/service/i18n/ @LuckyPickleZZ -/server/odc-service/**/service/iam/ @MarkPotato777 @smallsheeeep +/server/odc-service/**/service/iam/ @MarkPotato777 @PeachThinking /server/odc-service/**/service/info/ @yhilmare @yizhouxw -/server/odc-service/**/service/integration/ @smallsheeeep @ungreat +/server/odc-service/**/service/integration/ @yiminpeng @ungreat /server/odc-service/**/service/lab/ @LuckyPickleZZ @ungreat /server/odc-service/**/service/monitor/ @ungreat @yizhouxw /server/odc-service/**/service/notification/ @LuckyPickleZZ @MarkPotato777 /server/odc-service/**/service/objectstorage/ @MarkPotato777 @yizhouxw /server/odc-service/**/service/onlineschemachange/ @krihy @LuckyPickleZZ /server/odc-service/**/service/partitionplan/ @guowl3 @yhilmare -/server/odc-service/**/service/permission/ @smallsheeeep @yhilmare +/server/odc-service/**/service/permission/ @MarkPotato777 @yhilmare /server/odc-service/**/service/pldebug/ @yhilmare @krihy /server/odc-service/**/service/plugin/ @yhilmare @krihy /server/odc-service/**/service/quartz/ @guowl3 @yhilmare -/server/odc-service/**/service/requlation/ @MarkPotato777 @smallsheeeep -/server/odc-service/**/service/resourcegroup/ @MarkPotato777 @smallsheeeep +/server/odc-service/**/service/requlation/ @MarkPotato777 @yhilmare +/server/odc-service/**/service/resourcegroup/ @MarkPotato777 @yhilmare /server/odc-service/**/service/resultset/ @LuckyPickleZZ @PeachThinking /server/odc-service/**/service/rollbackplan/ @PeachThinking @MarkPotato777 /server/odc-service/**/service/schedule/ @guowl3 @yhilmare @@ -111,13 +111,13 @@ # CI/CD -/.github/ @smallsheeeep @yhilmare @yizhouxw -/builds/ @smallsheeeep @yhilmare @yizhouxw -/script/ @smallsheeeep @yhilmare @yizhouxw -/distribution/ @smallsheeeep @yhilmare @yizhouxw -/server/odc-test/ @smallsheeeep @yhilmare @yizhouxw -/server/integration-test/ @smallsheeeep @yhilmare @yizhouxw -/server/test-script/ @smallsheeeep @yhilmare @yizhouxw +/.github/ @MarkPotato777 @yhilmare @yizhouxw +/builds/ @MarkPotato777 @yhilmare @yizhouxw +/script/ @MarkPotato777 @yhilmare @yizhouxw +/distribution/ @MarkPotato777 @yhilmare @yizhouxw +/server/odc-test/ @MarkPotato777 @yhilmare @yizhouxw +/server/integration-test/ @MarkPotato777 @yhilmare @yizhouxw +/server/test-script/ @MarkPotato777 @yhilmare @yizhouxw # i18n /server/odc-core/src/main/resources/i18n/ @Jane201510 @JessieWuJiexi @yizhouxw From dc733da024be77c5023a643ae5ae37036b10f1bf Mon Sep 17 00:00:00 2001 From: XiaoYang Date: Thu, 20 Jun 2024 14:32:39 +0800 Subject: [PATCH 14/64] fix(database-permission): mistake caused by code merge (#2786) * fix code merge mistake * remove useless dependency inject --- .../connection/database/DatabaseService.java | 4 ---- .../permission/DBResourcePermissionHelper.java | 13 ++++++++++--- .../resultset/ResultSetExportFlowableTask.java | 7 ------- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java index 99e0f33b50..abd8797b83 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java @@ -115,7 +115,6 @@ import com.oceanbase.odc.service.onlineschemachange.ddl.OscDBAccessorFactory; import com.oceanbase.odc.service.onlineschemachange.rename.OscDBUserUtil; import com.oceanbase.odc.service.permission.DBResourcePermissionHelper; -import com.oceanbase.odc.service.permission.common.PermissionCheckWhitelist; import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; import com.oceanbase.odc.service.plugin.SchemaPluginUtil; import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; @@ -209,9 +208,6 @@ public class DatabaseService { @Autowired private DBSchemaSyncProperties dbSchemaSyncProperties; - @Autowired - private PermissionCheckWhitelist permissionCheckWhitelist; - @Transactional(rollbackFor = Exception.class) @SkipAuthorize("internal authenticated") public Database detail(@NonNull Long id) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java index 3222853675..21aab9aec4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java @@ -50,6 +50,7 @@ import com.oceanbase.odc.service.connection.database.model.DBResource; import com.oceanbase.odc.service.connection.database.model.UnauthorizedDBResource; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; +import com.oceanbase.odc.service.permission.common.PermissionCheckWhitelist; import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; import com.oceanbase.tools.dbbrowser.model.DBObjectType; @@ -79,6 +80,9 @@ public class DBResourcePermissionHelper { @Autowired private DBObjectRepository dbObjectRepository; + @Autowired + private PermissionCheckWhitelist permissionCheckWhitelist; + private static final Set ORACLE_DATA_DICTIONARY = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); private static final Set MYSQL_DATA_DICTIONARY = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); @@ -108,9 +112,11 @@ public void checkDBPermissions(Collection databaseIds, Collection> id2PermissionTypes = getInnerDBPermissionTypes(databaseIds); for (Long databaseId : toCheckDatabaseIds) { @@ -293,7 +299,8 @@ public List filterUnauthorizedDBResources( for (Map.Entry> entry : resource2Types.entrySet()) { DBResource resource = entry.getKey(); Set needs = entry.getValue(); - if (CollectionUtils.isEmpty(needs)) { + if (CollectionUtils.isEmpty(needs) || permissionCheckWhitelist.containsDatabase(resource.getDatabaseName(), + resource.getDialectType())) { continue; } if (resource.getType() == ResourceType.ODC_DATABASE) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/resultset/ResultSetExportFlowableTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/resultset/ResultSetExportFlowableTask.java index a992caf451..3700581b51 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/resultset/ResultSetExportFlowableTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/resultset/ResultSetExportFlowableTask.java @@ -25,26 +25,19 @@ import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.metadb.task.TaskEntity; import com.oceanbase.odc.service.connection.ConnectionService; -import com.oceanbase.odc.service.datasecurity.DataMaskingService; import com.oceanbase.odc.service.flow.task.BaseODCFlowTaskDelegate; import com.oceanbase.odc.service.flow.task.model.ResultSetExportResult; import com.oceanbase.odc.service.flow.util.FlowTaskUtil; -import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.task.TaskService; import lombok.extern.slf4j.Slf4j; @Slf4j public class ResultSetExportFlowableTask extends BaseODCFlowTaskDelegate { - public static final String TASK_WORKSPACE = "task.workspace"; @Autowired private DumperResultSetExportTaskManager taskManager; @Autowired - private AuthenticationFacade authenticationFacade; - @Autowired - private DataMaskingService maskingService; - @Autowired private ConnectionService connectionService; private ResultSetExportTaskContext context; private volatile TaskStatus status; From 3c33aedd1270e2334b320a32e158c9a29a100350 Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Mon, 24 Jun 2024 12:00:09 +0800 Subject: [PATCH 15/64] feat(server): allows some beans to be loaded only in server mode (#2757) * add conditional on server * add conditional on server * add conditional on server * add conditional on server --- .../odc/service/common/ConditionOnServer.java | 31 ++++++++++++ .../common/ExcludeProfileCondition.java | 40 +++++++++++++++ .../connection/ConnectionSchedules.java | 3 ++ .../connection/DatabaseSyncSchedules.java | 2 + ...eChangeChangingOrderTemplateSchedules.java | 2 + .../db/schema/DBSchemaSyncScheduler.java | 2 + .../odc/service/dlm/DlmLimiterService.java | 4 -- .../odc/service/flow/FlowSchedules.java | 2 + .../odc/service/lab/LabSchedules.java | 2 + .../NotificationScheduleConfiguration.java | 2 + .../odc/service/quartz/QuartzJobService.java | 2 +- .../quartz/config/QuartzConfiguration.java | 2 + .../config/TaskFrameworkConfiguration.java | 49 ++++++++++++++++++- .../DefaultJobProcessUpdateListener.java | 4 +- 14 files changed, 139 insertions(+), 8 deletions(-) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/common/ConditionOnServer.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/common/ExcludeProfileCondition.java diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/common/ConditionOnServer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/common/ConditionOnServer.java new file mode 100644 index 0000000000..d32f8bfbcf --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/common/ConditionOnServer.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.common; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.springframework.context.annotation.Conditional; + +@Conditional(ExcludeProfileCondition.class) +@Target({ElementType.TYPE, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface ConditionOnServer { +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/common/ExcludeProfileCondition.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/common/ExcludeProfileCondition.java new file mode 100644 index 0000000000..99d1c86f71 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/common/ExcludeProfileCondition.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.common; + +import java.util.Arrays; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotatedTypeMetadata; + +public class ExcludeProfileCondition implements Condition { + + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + Environment environment = context.getEnvironment(); + String excludeProfilesProperty = environment.getProperty("odc.server.condition.exclude-profile", ""); + String[] excludeProfiles = excludeProfilesProperty.split(","); + String[] activeProfiles = environment.getActiveProfiles(); + for (String activeProfile : activeProfiles) { + if (Arrays.asList(excludeProfiles).contains(activeProfile.trim())) { + return false; + } + } + return true; + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionSchedules.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionSchedules.java index 8cc7ae77b0..b0238163a8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionSchedules.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionSchedules.java @@ -19,10 +19,13 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import com.oceanbase.odc.service.common.ConditionOnServer; + import lombok.extern.slf4j.Slf4j; @Slf4j @Component +@ConditionOnServer public class ConnectionSchedules { @Autowired diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/DatabaseSyncSchedules.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/DatabaseSyncSchedules.java index 0f65453816..2fe813a53c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/DatabaseSyncSchedules.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/DatabaseSyncSchedules.java @@ -23,6 +23,7 @@ import org.springframework.util.CollectionUtils; import com.oceanbase.odc.core.shared.constant.ConnectionVisibleScope; +import com.oceanbase.odc.service.common.ConditionOnServer; import com.oceanbase.odc.service.connection.database.DatabaseSyncManager; import com.oceanbase.odc.service.connection.model.ConnectionConfig; @@ -35,6 +36,7 @@ */ @Slf4j @Component +@ConditionOnServer public class DatabaseSyncSchedules { @Autowired private DatabaseSyncManager databaseSyncManager; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/databasechange/DatabaseChangeChangingOrderTemplateSchedules.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/databasechange/DatabaseChangeChangingOrderTemplateSchedules.java index 4219faf1d7..043d0be1ca 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/databasechange/DatabaseChangeChangingOrderTemplateSchedules.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/databasechange/DatabaseChangeChangingOrderTemplateSchedules.java @@ -32,6 +32,7 @@ import com.oceanbase.odc.metadb.databasechange.DatabaseChangeChangingOrderTemplateEntity; import com.oceanbase.odc.metadb.databasechange.DatabaseChangeChangingOrderTemplateRepository; import com.oceanbase.odc.metadb.databasechange.DatabaseChangeChangingOrderTemplateSpecs; +import com.oceanbase.odc.service.common.ConditionOnServer; import lombok.extern.slf4j.Slf4j; @@ -41,6 +42,7 @@ */ @Slf4j @Component +@ConditionOnServer public class DatabaseChangeChangingOrderTemplateSchedules { private static final int PAGE_SIZE = 100; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncScheduler.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncScheduler.java index af215b72f6..134598fdc7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncScheduler.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncScheduler.java @@ -33,6 +33,7 @@ import com.oceanbase.odc.core.shared.constant.OrganizationType; import com.oceanbase.odc.metadb.iam.OrganizationEntity; import com.oceanbase.odc.metadb.iam.OrganizationRepository; +import com.oceanbase.odc.service.common.ConditionOnServer; import com.oceanbase.odc.service.config.UserConfigKeys; import com.oceanbase.odc.service.config.UserConfigService; import com.oceanbase.odc.service.config.model.Configuration; @@ -47,6 +48,7 @@ */ @Slf4j @Component +@ConditionOnServer public class DBSchemaSyncScheduler { @Autowired diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DlmLimiterService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DlmLimiterService.java index ca2f683e7d..36691eeb89 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DlmLimiterService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DlmLimiterService.java @@ -31,7 +31,6 @@ import com.oceanbase.odc.metadb.dlm.DlmLimiterConfigEntity; import com.oceanbase.odc.metadb.dlm.DlmLimiterConfigRepository; import com.oceanbase.odc.service.dlm.model.RateLimitConfiguration; -import com.oceanbase.odc.service.schedule.ScheduleService; /** * @Author：tinker @@ -62,9 +61,6 @@ public class DlmLimiterService { @Autowired private DlmLimiterConfigRepository limiterConfigRepository; - @Autowired - private ScheduleService scheduleService; - public DlmLimiterConfigEntity create(RateLimitConfiguration config) { checkLimiterConfig(config); DlmLimiterConfigEntity entity = mapper.modelToEntity(config); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowSchedules.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowSchedules.java index 889cae6a43..84b3de204e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowSchedules.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowSchedules.java @@ -33,6 +33,7 @@ import com.oceanbase.odc.metadb.flow.ServiceTaskInstanceRepository; import com.oceanbase.odc.metadb.task.TaskEntity; import com.oceanbase.odc.metadb.task.TaskRepository; +import com.oceanbase.odc.service.common.ConditionOnServer; import com.oceanbase.odc.service.flow.model.FlowNodeStatus; import com.oceanbase.odc.service.flow.task.model.FlowTaskProperties; import com.oceanbase.odc.service.flow.task.model.RuntimeTaskConstants; @@ -49,6 +50,7 @@ */ @Slf4j @Component +@ConditionOnServer public class FlowSchedules { private static final Integer OB_MAX_IN_SIZE = 2000; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/lab/LabSchedules.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/lab/LabSchedules.java index de3e1bfffd..dacb766eee 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/lab/LabSchedules.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/lab/LabSchedules.java @@ -26,6 +26,7 @@ import com.oceanbase.odc.core.shared.Verify; import com.oceanbase.odc.metadb.connection.ConnectionEntity; +import com.oceanbase.odc.service.common.ConditionOnServer; import com.oceanbase.odc.service.config.SystemConfigService; import com.oceanbase.odc.service.config.model.Configuration; import com.oceanbase.odc.service.connection.ConnectionSessionHistoryService; @@ -42,6 +43,7 @@ @Slf4j @Component @ConditionalOnProperty(value = "odc.lab.enabled", havingValue = "true") +@ConditionOnServer public class LabSchedules { @Autowired private ResourceService resourceService; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/NotificationScheduleConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/NotificationScheduleConfiguration.java index c0e34d5725..ad1baed0e4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/NotificationScheduleConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/NotificationScheduleConfiguration.java @@ -30,12 +30,14 @@ import org.springframework.scheduling.annotation.SchedulingConfigurer; import org.springframework.scheduling.config.ScheduledTaskRegistrar; +import com.oceanbase.odc.service.common.ConditionOnServer; import com.oceanbase.odc.service.notification.model.EventStatus; import com.oceanbase.odc.service.notification.model.MessageSendingStatus; @EnableScheduling @Configuration @Profile({"!clientMode"}) +@ConditionOnServer public class NotificationScheduleConfiguration implements SchedulingConfigurer { @Autowired private NotificationProperties notificationProperties; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/QuartzJobService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/QuartzJobService.java index fa6682cae6..9437c5e06d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/QuartzJobService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/QuartzJobService.java @@ -55,7 +55,7 @@ @SkipAuthorize("odc internal usage") public class QuartzJobService { - @Autowired + @Autowired(required = false) @Qualifier(value = ("defaultScheduler")) private Scheduler scheduler; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/config/QuartzConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/config/QuartzConfiguration.java index e4b48430ba..c0788720fa 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/config/QuartzConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/config/QuartzConfiguration.java @@ -28,6 +28,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.quartz.SchedulerFactoryBean; +import com.oceanbase.odc.service.common.ConditionOnServer; import com.oceanbase.odc.service.quartz.OdcJobListener; import com.oceanbase.odc.service.quartz.OdcTriggerListener; @@ -38,6 +39,7 @@ */ @Configuration +@ConditionOnServer public class QuartzConfiguration { @Autowired diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkConfiguration.java index afe3564e7d..e72e87b6c6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkConfiguration.java @@ -16,8 +16,12 @@ package com.oceanbase.odc.service.task.config; +import java.util.Map; +import java.util.concurrent.TimeUnit; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -25,10 +29,15 @@ import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.scheduling.quartz.SchedulerFactoryBean; +import com.oceanbase.odc.common.event.EventPublisher; +import com.oceanbase.odc.service.common.ConditionOnServer; import com.oceanbase.odc.service.task.caller.K8sJobClient; import com.oceanbase.odc.service.task.caller.NativeK8sJobClient; +import com.oceanbase.odc.service.task.exception.JobException; import com.oceanbase.odc.service.task.jasypt.DefaultJasyptEncryptorConfigProperties; import com.oceanbase.odc.service.task.jasypt.JasyptEncryptorConfigProperties; +import com.oceanbase.odc.service.task.schedule.JobDefinition; +import com.oceanbase.odc.service.task.schedule.JobScheduler; import com.oceanbase.odc.service.task.schedule.MonitorProcessRateLimiter; import com.oceanbase.odc.service.task.schedule.StartJobRateLimiter; import com.oceanbase.odc.service.task.service.TaskFrameworkService; @@ -48,7 +57,8 @@ public class TaskFrameworkConfiguration { @Lazy @Bean - @ConditionalOnMissingBean(K8sJobClient.class) + @ConditionalOnMissingBean({K8sJobClient.class}) + @ConditionalOnBean(TaskFrameworkProperties.class) public K8sJobClient k8sJobClient(@Autowired TaskFrameworkProperties taskFrameworkProperties) { try { log.info("k8s url is {}", taskFrameworkProperties.getK8sProperties().getKubeUrl()); @@ -61,6 +71,7 @@ public K8sJobClient k8sJobClient(@Autowired TaskFrameworkProperties taskFramewor } @Bean + @ConditionalOnBean(TaskFrameworkService.class) public StartJobRateLimiter monitorProcessRateLimiter(@Autowired TaskFrameworkService taskFrameworkService) { return new MonitorProcessRateLimiter(TaskFrameworkPropertiesSupplier.getSupplier(), taskFrameworkService); } @@ -73,6 +84,8 @@ public JasyptEncryptorConfigProperties JasyptEncryptorConfigProperties( @Lazy @Bean("taskFrameworkSchedulerFactoryBean") + @ConditionalOnBean(TaskFrameworkProperties.class) + @ConditionOnServer public SchedulerFactoryBean taskFrameworkSchedulerFactoryBean( TaskFrameworkProperties taskFrameworkProperties, @Qualifier("taskFrameworkMonitorExecutor") ThreadPoolTaskExecutor executor) { @@ -85,6 +98,7 @@ public SchedulerFactoryBean taskFrameworkSchedulerFactoryBean( } @Bean + @ConditionalOnBean(TaskFrameworkProperties.class) public TaskFrameworkEnabledProperties taskFrameworkEnabledProperties( @Autowired TaskFrameworkProperties taskFrameworkProperties) { TaskFrameworkEnabledProperties properties = new TaskFrameworkEnabledProperties(); @@ -95,15 +109,48 @@ public TaskFrameworkEnabledProperties taskFrameworkEnabledProperties( } @Bean + @ConditionOnServer public JobConfiguration jobConfiguration() { return new DefaultSpringJobConfiguration(); } @Bean + @ConditionalOnBean(JobConfiguration.class) public JobSchedulerFactoryBean jobSchedulerFactoryBean(@Autowired JobConfiguration jobConfiguration) { JobSchedulerFactoryBean factoryBean = new JobSchedulerFactoryBean(); factoryBean.setJobConfiguration(jobConfiguration); return factoryBean; } + @Bean + @ConditionalOnMissingBean + public JobScheduler jobScheduler() { + return new JobScheduler() { + @Override + public Long scheduleJobNow(JobDefinition jd) { + throw new UnsupportedOperationException(); + } + + @Override + public void cancelJob(Long jobId) throws JobException { + throw new UnsupportedOperationException(); + } + + @Override + public void modifyJobParameters(Long jobId, Map jobParameters) throws JobException { + throw new UnsupportedOperationException(); + } + + @Override + public void await(Long jobId, Integer timeout, TimeUnit timeUnit) throws InterruptedException { + throw new UnsupportedOperationException(); + } + + @Override + public EventPublisher getEventPublisher() { + throw new UnsupportedOperationException(); + } + }; + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateListener.java index 1d71c0f3f7..daf951b1e8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateListener.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateListener.java @@ -34,7 +34,7 @@ import com.oceanbase.odc.service.task.executor.task.TaskResult; import com.oceanbase.odc.service.task.model.ExecutorInfo; import com.oceanbase.odc.service.task.schedule.JobIdentity; -import com.oceanbase.odc.service.task.service.StdTaskFrameworkService; +import com.oceanbase.odc.service.task.service.TaskFrameworkService; import lombok.extern.slf4j.Slf4j; @@ -54,7 +54,7 @@ public class DefaultJobProcessUpdateListener extends AbstractEventListener Date: Wed, 26 Jun 2024 20:00:03 +0800 Subject: [PATCH 16/64] refactor(flow): add organizationId to flow instance detail (#2841) --- .../odc/service/flow/model/FlowInstanceDetailResp.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowInstanceDetailResp.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowInstanceDetailResp.java index f24bdda812..04497a9690 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowInstanceDetailResp.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowInstanceDetailResp.java @@ -77,6 +77,7 @@ public class FlowInstanceDetailResp { private Long id; + private Long organizationId; private TaskType type; private List subTypes; private Integer maxRiskLevel; @@ -119,6 +120,7 @@ public FlowInstanceDetailResp map(@NonNull FlowInstanceEntity entity) { resp.setCreateTime(entity.getCreateTime()); resp.setStatus(entity.getStatus()); resp.setProjectId(entity.getProjectId()); + resp.setOrganizationId(entity.getOrganizationId()); Set candidates = getCandidatesByFlowInstanceId.apply(entity.getId()); if (candidates != null) { resp.setCandidateApprovers(candidates.stream().map(InnerUser::new).collect(Collectors.toSet())); @@ -163,6 +165,7 @@ public FlowInstanceDetailResp map(@NonNull FlowInstance flowInstance, @NonNull F if (this.ifRollbackable != null) { resp.setRollbackable(ifRollbackable.test(flowInstance.getId())); } + resp.setOrganizationId(flowInstance.getOrganizationId()); resp.setCreator(InnerUser.of(flowInstance.getCreatorId(), getUserById, getRolesByUserId)); resp.setCompleteTime(flowInstance.getUpdateTime()); resp.setCreateTime(flowInstance.getCreateTime()); From 32e5f85b5da09c01b13ae226a7e0fbc4359e7fb8 Mon Sep 17 00:00:00 2001 From: yiminpeng <101048604+yiminpeng@users.noreply.github.com> Date: Mon, 1 Jul 2024 17:14:18 +0800 Subject: [PATCH 17/64] enhancement(migrate): delete useless yaml/sql file (#2858) --- .../common/V_3_4_0_12__add_data_masking.sql | 76 --- .../common/V_3_4_0_13__data_masking_rule.yaml | 171 ------- ...V_3_4_0_14__data_masking_rule_segment.yaml | 450 ------------------ .../common/V_3_4_0_2__iam_permission.yaml | 103 ---- .../rbac/V_3_4_0_3__iam_role_permission.yaml | 24 - 5 files changed, 824 deletions(-) delete mode 100644 server/odc-migrate/src/main/resources/migrate/common/V_3_4_0_12__add_data_masking.sql delete mode 100644 server/odc-migrate/src/main/resources/migrate/common/V_3_4_0_13__data_masking_rule.yaml delete mode 100644 server/odc-migrate/src/main/resources/migrate/common/V_3_4_0_14__data_masking_rule_segment.yaml delete mode 100644 server/odc-migrate/src/main/resources/migrate/common/V_3_4_0_2__iam_permission.yaml delete mode 100644 server/odc-migrate/src/main/resources/migrate/rbac/V_3_4_0_3__iam_role_permission.yaml diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_3_4_0_12__add_data_masking.sql b/server/odc-migrate/src/main/resources/migrate/common/V_3_4_0_12__add_data_masking.sql deleted file mode 100644 index d3d3faf001..0000000000 --- a/server/odc-migrate/src/main/resources/migrate/common/V_3_4_0_12__add_data_masking.sql +++ /dev/null @@ -1,76 +0,0 @@ ---- ---- v4.0.0 ---- - ---- data masking rule -CREATE TABLE IF NOT EXISTS `data_masking_rule` ( - `id` bigint NOT NULL AUTO_INCREMENT, - `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', - `user_update_time` datetime DEFAULT NULL COMMENT '用户修改时间', - `creator_id` bigint NOT NULL COMMENT '创建用户 ID, references iam_user(id)', - `organization_id` bigint NOT NULL COMMENT '所属组织 ID, references iam_organization(id)', - `name` varchar(256) NOT NULL COMMENT '脱敏规则名称', - `is_enabled` tinyint(1) NOT NULL COMMENT '是否启用', - `is_builtin` tinyint(1) NOT NULL DEFAULT '0' COMMENT '是否为内置脱敏规则', - `type` varchar(32) NOT NULL COMMENT '脱敏规则类型，optional values: HASH, MASK, NULL, PSEUDO, ROUNDING, SUBSTITUTION', - CONSTRAINT pk_data_masking_rule_id PRIMARY KEY (`id`), - CONSTRAINT uk_data_masking_rule_organization_id_creator_id_name UNIQUE KEY (`organization_id`, `creator_id`, `name`) -) COMMENT = '脱敏规则记录表'; - -CREATE TABLE IF NOT EXISTS `data_masking_rule_property` ( - `id` bigint NOT NULL AUTO_INCREMENT, - `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', - `creator_id` bigint NOT NULL COMMENT '创建用户 ID, references iam_user(id)', - `organization_id` bigint NOT NULL COMMENT '所属组织 ID, references iam_organization(id)', - `key_string` VARCHAR(64) NOT NULL COMMENT 'property的key', - `value_string` VARCHAR(512) DEFAULT NULL COMMENT 'property的value', - `rule_id` bigint NOT NULL COMMENT '关联的规则 ID, references data_masking_rule(id)', - CONSTRAINT pk_data_masking_rule_property_id PRIMARY KEY (`id`), - CONSTRAINT uk_data_masking_rule_property_organization_id_rule_id_key UNIQUE KEY (`organization_id`, `rule_id`, `key_string`) -) COMMENT = '脱敏规则属性表'; - -CREATE TABLE IF NOT EXISTS `data_masking_rule_segment` ( - `id` bigint NOT NULL AUTO_INCREMENT, - `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', - `creator_id` bigint NOT NULL COMMENT '创建用户 ID, references iam_user(id)', - `organization_id` bigint NOT NULL COMMENT '所属组织 ID, references iam_organization(id)', - `is_mask` tinyint(1) DEFAULT NULL COMMENT '此段是否遮掩 / 替换', - `type` varchar(32) DEFAULT NULL COMMENT '分段类型，optional values: DIGIT, DIGIT_PERCENTAGE, LEFT_OVER, DELIMITER', - `segments_type` varchar(32) DEFAULT NULL COMMENT '整体分段类型，optional values: CUSTOM, PRE_1_POST_1, PRE_3_POST_2, PRE_3_POST_4, ALL, PRE_3, POST_4', - `replaced_characters` varchar(32) DEFAULT NULL COMMENT '替换字符', - `delimiter` varchar(16) DEFAULT NULL COMMENT '分段使用的分隔符', - `digit_number` int DEFAULT NULL COMMENT '分段位数', - `digit_percentage` int DEFAULT NULL COMMENT '分段位数比例', - `ordinal` int DEFAULT NULL COMMENT '此段在所有分段中的顺序，从0开始', - `rule_id` bigint NOT NULL COMMENT '关联的规则 ID, references data_masking_rule(id)', - CONSTRAINT pk_data_masking_rule_segment_id PRIMARY KEY (`id`) -) COMMENT = '脱敏分段表'; - ---- data masking policy -CREATE TABLE IF NOT EXISTS `data_masking_policy` ( - `id` bigint NOT NULL AUTO_INCREMENT, - `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', - `creator_id` bigint NOT NULL COMMENT '创建用户 ID, references iam_user(id)', - `organization_id` bigint NOT NULL COMMENT '所属组织 ID, references iam_organization(id)', - `name` varchar(256) NOT NULL COMMENT '脱敏策略名称', - CONSTRAINT pk_data_masking_policy_id PRIMARY KEY (`id`), - CONSTRAINT uk_data_masking_policy_organization_id_creator_id_name UNIQUE KEY (`organization_id`, `creator_id`, `name`) -) COMMENT = '脱敏策略表'; - -CREATE TABLE IF NOT EXISTS `data_masking_rule_applying` ( - `id` bigint NOT NULL AUTO_INCREMENT, - `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', - `creator_id` bigint NOT NULL COMMENT '创建用户 ID, references iam_user(id)', - `organization_id` bigint NOT NULL COMMENT '所属组织 ID, references iam_organization(id)', - `policy_id` bigint NOT NULL COMMENT '脱敏策略 ID, references data_masking_policy(id)', - `rule_id` bigint DEFAULT NULL COMMENT '脱敏规则 ID, references data_masking_rule(id)', - `includes` text NOT NULL COMMENT '脱敏包含列表达式', - `excludes` text DEFAULT NULL COMMENT '脱敏排除列表达式', - `priority` int NOT NULL COMMENT '脱敏规则应用在所属脱敏策略中的排序，从 0 开始，数值越小表示顺位靠前', - CONSTRAINT pk_data_masking_rule_applying_id PRIMARY KEY (`id`) -) COMMENT = '脱敏规则应用表'; \ No newline at end of file diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_3_4_0_13__data_masking_rule.yaml b/server/odc-migrate/src/main/resources/migrate/common/V_3_4_0_13__data_masking_rule.yaml deleted file mode 100644 index 0a22a393d1..0000000000 --- a/server/odc-migrate/src/main/resources/migrate/common/V_3_4_0_13__data_masking_rule.yaml +++ /dev/null @@ -1,171 +0,0 @@ -kind: resource -version: v2 -templates: - - metadata: - allow_duplicate: false - table_name: data_masking_rule - unique_keys: [ "organization_id", "name" ] - specs: - - column_name: id - default_value: 1 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: name - value: ${com.oceanbase.odc.builtin-resource.MaskRule.NAME.Name} - - column_name: type - value: "SUBSTITUTION" - - column_name: is_enabled - value: true - data_type: java.lang.Boolean - - column_name: is_builtin - value: true - data_type: java.lang.Boolean - - metadata: - allow_duplicate: false - table_name: data_masking_rule - unique_keys: [ "organization_id", "name" ] - specs: - - column_name: id - default_value: 2 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: name - value: ${com.oceanbase.odc.builtin-resource.MaskRule.ID.Name} - - column_name: type - value: "MASK" - - column_name: is_enabled - value: true - data_type: java.lang.Boolean - - column_name: is_builtin - value: true - data_type: java.lang.Boolean - - metadata: - allow_duplicate: false - table_name: data_masking_rule - unique_keys: [ "organization_id", "name" ] - specs: - - column_name: id - default_value: 3 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: name - value: ${com.oceanbase.odc.builtin-resource.MaskRule.MOBILE_PHONE.Name} - - column_name: type - value: "MASK" - - column_name: is_enabled - value: true - data_type: java.lang.Boolean - - column_name: is_builtin - value: true - data_type: java.lang.Boolean - - metadata: - allow_duplicate: false - table_name: data_masking_rule - unique_keys: [ "organization_id", "name" ] - specs: - - column_name: id - default_value: 4 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: name - value: ${com.oceanbase.odc.builtin-resource.MaskRule.FIXED_PHONE.Name} - - column_name: type - value: "MASK" - - column_name: is_enabled - value: true - data_type: java.lang.Boolean - - column_name: is_builtin - value: true - data_type: java.lang.Boolean - - metadata: - allow_duplicate: false - table_name: data_masking_rule - unique_keys: [ "organization_id", "name" ] - specs: - - column_name: id - default_value: 5 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: name - value: ${com.oceanbase.odc.builtin-resource.MaskRule.BANKCARD.Name} - - column_name: type - value: "MASK" - - column_name: is_enabled - value: true - data_type: java.lang.Boolean - - column_name: is_builtin - value: true - data_type: java.lang.Boolean - - metadata: - allow_duplicate: false - table_name: data_masking_rule - unique_keys: [ "organization_id", "name" ] - specs: - - column_name: id - default_value: 6 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: name - value: ${com.oceanbase.odc.builtin-resource.MaskRule.MAIL.Name} - - column_name: type - value: "SUBSTITUTION" - - column_name: is_enabled - value: true - data_type: java.lang.Boolean - - column_name: is_builtin - value: true - data_type: java.lang.Boolean - - metadata: - allow_duplicate: false - table_name: data_masking_rule - unique_keys: [ "organization_id", "name" ] - specs: - - column_name: id - default_value: 7 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: name - value: ${com.oceanbase.odc.builtin-resource.MaskRule.DEFAULT.Name} - - column_name: type - value: "MASK" - - column_name: is_enabled - value: true - data_type: java.lang.Boolean - - column_name: is_builtin - value: true - data_type: java.lang.Boolean \ No newline at end of file diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_3_4_0_14__data_masking_rule_segment.yaml b/server/odc-migrate/src/main/resources/migrate/common/V_3_4_0_14__data_masking_rule_segment.yaml deleted file mode 100644 index e4f71b380b..0000000000 --- a/server/odc-migrate/src/main/resources/migrate/common/V_3_4_0_14__data_masking_rule_segment.yaml +++ /dev/null @@ -1,450 +0,0 @@ -kind: resource -version: v2 -templates: - # 内置姓名规则分段 - - metadata: - allow_duplicate: false - table_name: data_masking_rule_segment - specs: - - column_name: id - default_value: 1 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: rule_id - value_from: - field_ref: - ref_file: migrate/common/V_3_4_0_13__data_masking_rule.yaml - field_path: templates.0.specs.0.value - - column_name: is_mask - value: true - data_type: java.lang.Boolean - - column_name: type - value: "LEFT_OVER" - - column_name: replaced_characters - value: "****" - - column_name: ordinal - value: 0 - data_type: java.lang.Integer - - metadata: - allow_duplicate: false - table_name: data_masking_rule_segment - specs: - - column_name: id - default_value: 2 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: rule_id - value_from: - field_ref: - ref_file: migrate/common/V_3_4_0_13__data_masking_rule.yaml - field_path: templates.0.specs.0.value - - column_name: is_mask - value: false - data_type: java.lang.Boolean - - column_name: type - value: "DIGIT" - - column_name: digit_number - value: 1 - data_type: java.lang.Integer - - column_name: ordinal - value: 1 - data_type: java.lang.Integer - # 身份证件号码 - - metadata: - allow_duplicate: false - table_name: data_masking_rule_segment - specs: - - column_name: id - default_value: 3 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: rule_id - value_from: - field_ref: - ref_file: migrate/common/V_3_4_0_13__data_masking_rule.yaml - field_path: templates.1.specs.0.value - - column_name: is_mask - value: false - data_type: java.lang.Boolean - - column_name: type - value: "DIGIT" - - column_name: digit_number - value: 1 - data_type: java.lang.Integer - - column_name: ordinal - value: 0 - data_type: java.lang.Integer - - metadata: - allow_duplicate: false - table_name: data_masking_rule_segment - specs: - - column_name: id - default_value: 4 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: rule_id - value_from: - field_ref: - ref_file: migrate/common/V_3_4_0_13__data_masking_rule.yaml - field_path: templates.1.specs.0.value - - column_name: is_mask - value: true - data_type: java.lang.Boolean - - column_name: type - value: "LEFT_OVER" - - column_name: ordinal - value: 1 - data_type: java.lang.Integer - - metadata: - allow_duplicate: false - table_name: data_masking_rule_segment - specs: - - column_name: id - default_value: 5 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: rule_id - value_from: - field_ref: - ref_file: migrate/common/V_3_4_0_13__data_masking_rule.yaml - field_path: templates.1.specs.0.value - - column_name: is_mask - value: false - data_type: java.lang.Boolean - - column_name: type - value: "DIGIT" - - column_name: digit_number - value: 1 - data_type: java.lang.Integer - - column_name: ordinal - value: 2 - data_type: java.lang.Integer - # 手机号 - - metadata: - allow_duplicate: false - table_name: data_masking_rule_segment - specs: - - column_name: id - default_value: 6 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: rule_id - value_from: - field_ref: - ref_file: migrate/common/V_3_4_0_13__data_masking_rule.yaml - field_path: templates.2.specs.0.value - - column_name: segments_type - value: "PRE_3_POST_2" - # 固定电话 - - metadata: - allow_duplicate: false - table_name: data_masking_rule_segment - specs: - - column_name: id - default_value: 7 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: rule_id - value_from: - field_ref: - ref_file: migrate/common/V_3_4_0_13__data_masking_rule.yaml - field_path: templates.3.specs.0.value - - column_name: is_mask - value: true - data_type: java.lang.Boolean - - column_name: type - value: "LEFT_OVER" - - column_name: ordinal - value: 0 - data_type: java.lang.Integer - - metadata: - allow_duplicate: false - table_name: data_masking_rule_segment - specs: - - column_name: id - default_value: 8 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: rule_id - value_from: - field_ref: - ref_file: migrate/common/V_3_4_0_13__data_masking_rule.yaml - field_path: templates.3.specs.0.value - - column_name: is_mask - value: false - data_type: java.lang.Boolean - - column_name: type - value: "DIGIT" - - column_name: digit_number - value: 2 - data_type: java.lang.Integer - - column_name: ordinal - value: 1 - data_type: java.lang.Integer - # 银行卡号 - - metadata: - allow_duplicate: false - table_name: data_masking_rule_segment - specs: - - column_name: id - default_value: 9 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: rule_id - value_from: - field_ref: - ref_file: migrate/common/V_3_4_0_13__data_masking_rule.yaml - field_path: templates.4.specs.0.value - - column_name: is_mask - value: true - data_type: java.lang.Boolean - - column_name: type - value: "LEFT_OVER" - - column_name: ordinal - value: 0 - data_type: java.lang.Integer - - metadata: - allow_duplicate: false - table_name: data_masking_rule_segment - specs: - - column_name: id - default_value: 10 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: rule_id - value_from: - field_ref: - ref_file: migrate/common/V_3_4_0_13__data_masking_rule.yaml - field_path: templates.4.specs.0.value - - column_name: is_mask - value: false - data_type: java.lang.Boolean - - column_name: type - value: "DIGIT" - - column_name: digit_number - value: 4 - data_type: java.lang.Integer - - column_name: ordinal - value: 1 - data_type: java.lang.Integer - # 邮箱 - - metadata: - allow_duplicate: false - table_name: data_masking_rule_segment - specs: - - column_name: id - default_value: 11 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: rule_id - value_from: - field_ref: - ref_file: migrate/common/V_3_4_0_13__data_masking_rule.yaml - field_path: templates.5.specs.0.value - - column_name: is_mask - value: false - data_type: java.lang.Boolean - - column_name: type - value: "DIGIT" - - column_name: digit_number - value: 3 - data_type: java.lang.Integer - - column_name: ordinal - value: 0 - data_type: java.lang.Integer - - metadata: - allow_duplicate: false - table_name: data_masking_rule_segment - specs: - - column_name: id - default_value: 12 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: rule_id - value_from: - field_ref: - ref_file: migrate/common/V_3_4_0_13__data_masking_rule.yaml - field_path: templates.5.specs.0.value - - column_name: is_mask - value: true - data_type: java.lang.Boolean - - column_name: type - value: "DELIMITER" - - column_name: delimiter - value: "@" - - column_name: replaced_characters - value: "***" - - column_name: ordinal - value: 1 - data_type: java.lang.Integer - - metadata: - allow_duplicate: false - table_name: data_masking_rule_segment - specs: - - column_name: id - default_value: 13 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: rule_id - value_from: - field_ref: - ref_file: migrate/common/V_3_4_0_13__data_masking_rule.yaml - field_path: templates.5.specs.0.value - - column_name: is_mask - value: false - data_type: java.lang.Boolean - - column_name: type - value: "LEFT_OVER" - - column_name: ordinal - value: 2 - data_type: java.lang.Integer - # 缺省规则 - - metadata: - allow_duplicate: false - table_name: data_masking_rule_segment - specs: - - column_name: id - default_value: 14 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: rule_id - value_from: - field_ref: - ref_file: migrate/common/V_3_4_0_13__data_masking_rule.yaml - field_path: templates.6.specs.0.value - - column_name: is_mask - value: false - data_type: java.lang.Boolean - - column_name: type - value: "DIGIT_PERCENTAGE" - - column_name: digit_percentage - value: 34 - data_type: java.lang.Integer - - column_name: ordinal - value: 0 - data_type: java.lang.Integer - - metadata: - allow_duplicate: false - table_name: data_masking_rule_segment - specs: - - column_name: id - default_value: 15 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: rule_id - value_from: - field_ref: - ref_file: migrate/common/V_3_4_0_13__data_masking_rule.yaml - field_path: templates.6.specs.0.value - - column_name: is_mask - value: true - data_type: java.lang.Boolean - - column_name: type - value: "LEFT_OVER" - - column_name: ordinal - value: 1 - data_type: java.lang.Integer - - metadata: - allow_duplicate: false - table_name: data_masking_rule_segment - specs: - - column_name: id - default_value: 16 - data_type: java.lang.Long - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: rule_id - value_from: - field_ref: - ref_file: migrate/common/V_3_4_0_13__data_masking_rule.yaml - field_path: templates.6.specs.0.value - - column_name: is_mask - value: false - data_type: java.lang.Boolean - - column_name: type - value: "DIGIT_PERCENTAGE" - - column_name: digit_percentage - value: 34 - data_type: java.lang.Integer - - column_name: ordinal - value: 2 - data_type: java.lang.Integer \ No newline at end of file diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_3_4_0_2__iam_permission.yaml b/server/odc-migrate/src/main/resources/migrate/common/V_3_4_0_2__iam_permission.yaml deleted file mode 100644 index e260638561..0000000000 --- a/server/odc-migrate/src/main/resources/migrate/common/V_3_4_0_2__iam_permission.yaml +++ /dev/null @@ -1,103 +0,0 @@ -kind: resource -version: v2 -templates: - - metadata: - allow_duplicate: false - table_name: iam_permission - unique_keys: ["action", "organization_id", "resource_identifier", "type" ] - specs: - - column_name: id - default_value: 30 - data_type: java.lang.Long - - column_name: action - value: "read" - - column_name: resource_identifier - value: "ODC_DATA_MASKING_RULE:*" - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: type - value: "SYSTEM" - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: is_builtin - value: true - data_type: java.lang.Boolean - - column_name: description - value: "builtin data masking rule read permission" - - metadata: - allow_duplicate: false - table_name: iam_permission - unique_keys: [ "action", "organization_id", "resource_identifier", "type" ] - specs: - - column_name: id - default_value: 31 - data_type: java.lang.Long - - column_name: action - value: "create" - - column_name: resource_identifier - value: "ODC_DATA_MASKING_RULE:*" - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: type - value: "SYSTEM" - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: is_builtin - value: true - data_type: java.lang.Boolean - - column_name: description - value: "builtin data masking rule create permission" - - metadata: - allow_duplicate: false - table_name: iam_permission - unique_keys: [ "action", "organization_id", "resource_identifier", "type" ] - specs: - - column_name: id - default_value: 32 - data_type: java.lang.Long - - column_name: action - value: "update" - - column_name: resource_identifier - value: "ODC_DATA_MASKING_RULE:*" - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: type - value: "SYSTEM" - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: is_builtin - value: true - data_type: java.lang.Boolean - - column_name: description - value: "builtin data masking rule update permission" - - metadata: - allow_duplicate: false - table_name: iam_permission - unique_keys: [ "action", "organization_id", "resource_identifier", "type" ] - specs: - - column_name: id - default_value: 33 - data_type: java.lang.Long - - column_name: action - value: "delete" - - column_name: resource_identifier - value: "ODC_DATA_MASKING_RULE:*" - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: type - value: "SYSTEM" - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long - - column_name: is_builtin - value: true - data_type: java.lang.Boolean - - column_name: description - value: "builtin data masking rule delete permission" diff --git a/server/odc-migrate/src/main/resources/migrate/rbac/V_3_4_0_3__iam_role_permission.yaml b/server/odc-migrate/src/main/resources/migrate/rbac/V_3_4_0_3__iam_role_permission.yaml deleted file mode 100644 index c7a398caa7..0000000000 --- a/server/odc-migrate/src/main/resources/migrate/rbac/V_3_4_0_3__iam_role_permission.yaml +++ /dev/null @@ -1,24 +0,0 @@ -kind: resource -version: v2 -templates: - - metadata: - allow_duplicate: false - table_name: iam_role_permission - unique_keys: [ "role_id", "permission_id" ] - specs: - - column_name: role_id - value_from: - field_ref: - ref_file: migrate/rbac/V_3_2_0_5__iam_role.yaml - field_path: templates.0.specs.0.value - - column_name: permission_id - value_from: - field_ref: - ref_file: migrate/common/V_3_4_0_2__iam_permission.yaml - field_path: templates.*.specs.0.value - - column_name: organization_id - value: ${ORGANIZATION_ID} - data_type: java.lang.Long - - column_name: creator_id - value: ${CREATOR_ID} - data_type: java.lang.Long \ No newline at end of file From 3eb1d4453d9293274565189352f16d4868f9f0d9 Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Tue, 2 Jul 2024 14:02:13 +0800 Subject: [PATCH 18/64] fix(data viewing): get result-set timeout (#2848) * fix get result-set timeout * Fixed the timeout value * Amend the variable name from the timeout to gettingResultTimeoutSeconds --- .../oceanbase/odc/service/session/ConnectConsoleService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java index 9f1ae6ae53..c6e3ff673e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java @@ -336,10 +336,11 @@ public AsyncExecuteResultResp getMoreResults(@NotNull String sessionId, String r ConnectionSession connectionSession = sessionService.nullSafeGet(sessionId); AsyncExecuteContext context = (AsyncExecuteContext) ConnectionSessionUtil.getExecuteContext(connectionSession, requestId); - int timeout = Objects.isNull(timeoutSeconds) ? DEFAULT_GET_RESULT_TIMEOUT_SECONDS : timeoutSeconds; + int gettingResultTimeoutSeconds = + Objects.isNull(timeoutSeconds) ? DEFAULT_GET_RESULT_TIMEOUT_SECONDS : timeoutSeconds; boolean shouldRemoveContext = context.isFinished(); try { - List resultList = context.getMoreSqlExecutionResults(timeout); + List resultList = context.getMoreSqlExecutionResults(gettingResultTimeoutSeconds * 1000); List results = resultList.stream().map(jdbcGeneralResult -> { SqlExecuteResult result = generateResult(connectionSession, jdbcGeneralResult, context.getContextMap()); try (TraceStage stage = result.getSqlTuple().getSqlWatch().start(SqlExecuteStages.SQL_AFTER_CHECK)) { From bbea22d07742a57e1dcee3517feac29d45d1127a Mon Sep 17 00:00:00 2001 From: IL MARE Date: Tue, 2 Jul 2024 20:19:41 +0800 Subject: [PATCH 19/64] fix(ob-sql-parser): failed to recognize interval expression in ob-oracle mode (#2873) --- .../src/main/resources/oboracle/sql/OBLexer.g4 | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/libs/ob-sql-parser/src/main/resources/oboracle/sql/OBLexer.g4 b/libs/ob-sql-parser/src/main/resources/oboracle/sql/OBLexer.g4 index 9f744561bb..0a74030a57 100644 --- a/libs/ob-sql-parser/src/main/resources/oboracle/sql/OBLexer.g4 +++ b/libs/ob-sql-parser/src/main/resources/oboracle/sql/OBLexer.g4 @@ -4874,11 +4874,11 @@ DATE_VALUE ; INTERVAL_VALUE - : I N T E R V A L ([ \t\n\r\f]+|('--'(~[\n\r])*))?'\''(~['])*'\''[ \t\n\r\f]*( Y E A R | M O N T H )[ \t\n\r\f]*('('|[\uff08])[ \t\n\r\f]*[0-9]+[ \t\n\r\f]*(')'|[\uff09])? - | I N T E R V A L ([ \t\n\r\f]+|('--'(~[\n\r])*))?'\''(~['])*'\''[ \t\n\r\f]*( Y E A R | M O N T H )([ \t\n\r\f]*('('|[\uff08])[ \t\n\r\f]*[0-9]+[ \t\n\r\f]*(')'|[\uff09])[ \t\n\r\f]*|[ \t\n\r\f]+) T O [ \t\n\r\f]+( Y E A R | M O N T H ) - | I N T E R V A L ([ \t\n\r\f]+|('--'(~[\n\r])*))?'\''(~['])*'\''[ \t\n\r\f]*( D A Y | H O U R | M I N U T E | S E C O N D )[ \t\n\r\f]*('('|[\uff08])[ \t\n\r\f]*[0-9]+[ \t\n\r\f]*(')'|[\uff09])? - | I N T E R V A L ([ \t\n\r\f]+|('--'(~[\n\r])*))?'\''(~['])*'\''[ \t\n\r\f]* S E C O N D [ \t\n\r\f]*('('|[\uff08])[ \t\n\r\f]*[0-9]+[ \t\n\r\f]*','[ \t\n\r\f]*[0-9]+[ \t\n\r\f]*(')'|[\uff09]) - | I N T E R V A L ([ \t\n\r\f]+|('--'(~[\n\r])*))?'\''(~['])*'\''[ \t\n\r\f]*( D A Y | H O U R | M I N U T E | S E C O N D )([ \t\n\r\f]*('('|[\uff08])[ \t\n\r\f]*[0-9]+[ \t\n\r\f]*(')'|[\uff09])[ \t\n\r\f]*|[ \t\n\r\f]+) T O [ \t\n\r\f]+( D A Y | H O U R | M I N U T E | S E C O N D [ \t\n\r\f]*('('|[\uff08])[ \t\n\r\f]*[0-9]+[ \t\n\r\f]*(')'|[\uff09])?) + : I N T E R V A L WHITESPACE? '\''(~['])*'\'' SPACE_* ( Y E A R | M O N T H ) INTERVAL_PRICISION? + | I N T E R V A L WHITESPACE? '\''(~['])*'\'' SPACE_* ( Y E A R | M O N T H ) (INTERVAL_PRICISION SPACE_* | SPACE_+) T O SPACE_+ ( Y E A R | M O N T H ) + | I N T E R V A L WHITESPACE? '\''(~['])*'\'' SPACE_* ( D A Y | H O U R | M I N U T E | S E C O N D ) INTERVAL_PRICISION? + | I N T E R V A L WHITESPACE? '\''(~['])*'\'' SPACE_* S E C O N D INTERVAL_SEC_PRICISION + | I N T E R V A L WHITESPACE? '\''(~['])*'\'' SPACE_* ( D A Y | H O U R | M I N U T E | S E C O N D )(INTERVAL_PRICISION SPACE_* | SPACE_+) T O SPACE_+( D A Y | H O U R | M I N U T E | S E C O N D INTERVAL_PRICISION?) ; HINT_VALUE @@ -5094,3 +5094,7 @@ fragment QS_EXCLAM : '!' .*? '!'; fragment QS_SHARP : '#' .*? '#'; fragment QS_QUOTE : '\'' .*? '\''; fragment QS_DQUOTE : '"' .*? '"'; +fragment SPACE_ : [ \t\n\r\f]; +fragment INTERVAL_PRICISION : SPACE_*('('|[\uff08])SPACE_*[0-9]+SPACE_*(')'|[\uff09]); +fragment INTERVAL_SEC_PRICISION : SPACE_*('('|[\uff08])SPACE_*[0-9]+SPACE_*','SPACE_*[0-9]+SPACE_*(')'|[\uff09]); +fragment WHITESPACE : (SPACE_+|('--'(~[\n\r])*)); From 7f806c98d48e87006d3cbd5ce6f7621095f9d2f2 Mon Sep 17 00:00:00 2001 From: CHLK <30882682+CHLK@users.noreply.github.com> Date: Wed, 3 Jul 2024 10:41:43 +0800 Subject: [PATCH 20/64] fix(audit): client ip length more langer then audit column client_ip_address (#2863) * fix(aduit): client ip length more langer then audit column client_ip_address * fix(aduit): rename getTrulyClientIp to getFirstIpFromRemoteAddress,and move it from WebRequestUtilsTest to AuditUtils * style: change the position of class comments in NotificationChannelRelationEntity * style: format AuditUtilsTest * style: remove /.github/hooks/pre-push from .gitignore --- .../NotificationChannelRelationEntity.java | 15 ++--- .../odc/service/audit/AuditEventAspect.java | 4 +- .../odc/service/audit/util/AuditUtils.java | 26 +++++++++ .../service/audit/util/AuditUtilsTest.java | 55 +++++++++++++++++++ 4 files changed, 89 insertions(+), 11 deletions(-) create mode 100644 server/odc-service/src/test/java/com/oceanbase/odc/service/audit/util/AuditUtilsTest.java diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/notification/NotificationChannelRelationEntity.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/notification/NotificationChannelRelationEntity.java index a3160c92c8..55dc986ed3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/notification/NotificationChannelRelationEntity.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/notification/NotificationChannelRelationEntity.java @@ -15,16 +15,6 @@ */ package com.oceanbase.odc.metadb.notification; -/** - * @Author: Lebie - * @Date: 2023/3/20 21:36 - * @Description: [] - */ -/** - * @Author: Lebie - * @Date: 2023/3/20 21:36 - * @Description: [] - */ import java.util.Date; import javax.persistence.Column; @@ -39,6 +29,11 @@ import lombok.Data; +/** + * @Author: Lebie + * @Date: 2023/3/20 21:36 + * @Description: [] + */ @Data @Entity @Table(name = "notification_policy_channel_relation") diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/audit/AuditEventAspect.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/audit/AuditEventAspect.java index 4789187478..28b053abc2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/audit/AuditEventAspect.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/audit/AuditEventAspect.java @@ -294,7 +294,9 @@ private AuditEvent createAuditEvent(Method method, Object[] args) { .type(auditEventMeta.getType()) .startTime(new Date()) .serverIpAddress(SystemUtils.getLocalIpAddress()) - .clientIpAddress(WebRequestUtils.getClientAddress(servletRequest)) + .clientIpAddress( + AuditUtils.getFirstIpFromRemoteAddress( + WebRequestUtils.getClientAddress(servletRequest))) .organizationId(authenticationFacade.currentOrganizationId()) .userId(authenticationFacade.currentUserId()) .username(authenticationFacade.currentUsername()) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/audit/util/AuditUtils.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/audit/util/AuditUtils.java index c63127d2ac..fddfeb2325 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/audit/util/AuditUtils.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/audit/util/AuditUtils.java @@ -323,4 +323,30 @@ public static AuditEventAction getActualActionForTask(AuditEventType type, Audit return action; } + /** + * 
+     * Get the first ip of {@param remoteAddress}.
+     * The X-Forwarded-For header may contain multiple IP addresses, separated
+     * by commas, and typically, the first non-unknown IP is considered to be the client's IP address.
+     *
+ * + * @author keyang.lk + * @date 2024-07-02 + * @param remoteAddress + * @return The first ip of remoteAddress + */ + public static String getFirstIpFromRemoteAddress(String remoteAddress) { + if (remoteAddress == null || remoteAddress.isEmpty() || "unknown".equalsIgnoreCase(remoteAddress)) { + return "N/A"; + } + // 处理X-Forwarded-For可能包含多个IP地址的情况（由逗号分隔），通常第一个非unknown的IP是客户端的IP + String[] ips = remoteAddress.split(","); + for (String ip : ips) { + if (ip != null && !ip.isEmpty() && + !"unknown".equalsIgnoreCase(ip.trim())) { + return ip.trim(); + } + } + return remoteAddress; + } } diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/audit/util/AuditUtilsTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/audit/util/AuditUtilsTest.java new file mode 100644 index 0000000000..c84624de99 --- /dev/null +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/audit/util/AuditUtilsTest.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.audit.util; + +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; +import java.util.Collection; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class AuditUtilsTest { + @Parameter(0) + public String input; + @Parameter(1) + public String except; + + @Parameters(name = "{index}: getFirstIpFromRemoteAddress({0})={1}") + public static Collection data() { + return Arrays.asList(new Object[][] { + {"", "N/A"}, + {null, "N/A"}, + {"unknown", "N/A"}, + {"UNKNOWN", "N/A"}, + {"123", "123"}, + {"192.168.1.1", "192.168.1.1"}, + {",192.168.1.1", "192.168.1.1"}, + {"192.168.1.1,122.122.1.1,127.0.0.1", "192.168.1.1"}, + {"unknown,192.168.1.1,122.122.1.1,127.0.0.1", "192.168.1.1"} + }); + } + + @Test + public void getFirstIpFromRemoteAddress() { + assertEquals(except, AuditUtils.getFirstIpFromRemoteAddress(input)); + } +} From 2c385aca84cb5b7443b930b5cf0bee45b0fbad4c Mon Sep 17 00:00:00 2001 From: IL MARE Date: Wed, 3 Jul 2024 11:06:51 +0800 Subject: [PATCH 21/64] fix(i18n): fix i18n docs encoding error (#2874) * update i18n * update i18n docs --- .../i18n/BusinessMessages_zh_CN.properties | 1452 ++++++++--------- 1 file changed, 726 insertions(+), 726 deletions(-) diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties index 8ba8959900..a4078edcda 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties @@ -1,176 +1,176 @@ -# business_messages_zh_CN.properties \u7B80\u4F53\u4E2D\u6587 +# business_messages_zh_CN.properties 简体中文 # # ResourceType # -com.oceanbase.odc.ResourceType.ODC_PRIVILEGE=\u6743\u9650 -com.oceanbase.odc.ResourceType.ODC_ORGANIZATION=\u7EC4\u7EC7 -com.oceanbase.odc.ResourceType.ODC_ROLE=\u89D2\u8272 -com.oceanbase.odc.ResourceType.ODC_RESOURCE_ROLE=\u8D44\u6E90\u89D2\u8272 -com.oceanbase.odc.ResourceType.ODC_USER=\u7528\u6237 -com.oceanbase.odc.ResourceType.ODC_CONNECTION=\u6570\u636E\u6E90 -com.oceanbase.odc.ResourceType.ODC_PRIVATE_CONNECTION=\u79C1\u6709\u8FDE\u63A5\u914D\u7F6E -com.oceanbase.odc.ResourceType.ODC_CONNECT_LABEL=\u8FDE\u63A5\u6807\u7B7E -com.oceanbase.odc.ResourceType.ODC_SESSION=\u8FDE\u63A5\u4F1A\u8BDD -com.oceanbase.odc.ResourceType.ODC_TASK=\u4EFB\u52A1 -com.oceanbase.odc.ResourceType.ODC_SCRIPT=\u811A\u672C -com.oceanbase.odc.ResourceType.ODC_SNIPPET=\u4EE3\u7801\u7247\u6BB5 -com.oceanbase.odc.ResourceType.ODC_SOURCE_FILE=\u6E90\u4EE3\u7801\u6587\u4EF6 -com.oceanbase.odc.ResourceType.ODC_FILE=\u6587\u4EF6 -com.oceanbase.odc.ResourceType.ODC_RESOURCE_GROUP=\u8D44\u6E90\u7EC4 -com.oceanbase.odc.ResourceType.ODC_FLOW_INSTANCE=\u6D41\u7A0B\u5B9E\u4F8B -com.oceanbase.odc.ResourceType.ODC_FLOW_CONFIG=\u6D41\u7A0B\u914D\u7F6E -com.oceanbase.odc.ResourceType.ODC_FLOW_APPROVAL_INSTANCE=\u5BA1\u6279\u8282\u70B9 -com.oceanbase.odc.ResourceType.ODC_FLOW_TASK_INSTANCE=\u4EFB\u52A1\u8282\u70B9 -com.oceanbase.odc.ResourceType.ODC_FLOW_GATEWAY_INSTANCE=\u7F51\u5173\u8282\u70B9 -com.oceanbase.odc.ResourceType.ODC_ASYNC_SQL_RESULT=\u5F02\u6B65 SQL \u6267\u884C\u7ED3\u679C -com.oceanbase.odc.ResourceType.ODC_SYSTEM_CONFIG=\u7CFB\u7EDF\u914D\u7F6E -com.oceanbase.odc.ResourceType.ODC_AUDIT_EVENT=\u5BA1\u8BA1\u4E8B\u4EF6 +com.oceanbase.odc.ResourceType.ODC_PRIVILEGE=权限 +com.oceanbase.odc.ResourceType.ODC_ORGANIZATION=组织 +com.oceanbase.odc.ResourceType.ODC_ROLE=角色 +com.oceanbase.odc.ResourceType.ODC_RESOURCE_ROLE=资源角色 +com.oceanbase.odc.ResourceType.ODC_USER=用户 +com.oceanbase.odc.ResourceType.ODC_CONNECTION=数据源 +com.oceanbase.odc.ResourceType.ODC_PRIVATE_CONNECTION=私有连接配置 +com.oceanbase.odc.ResourceType.ODC_CONNECT_LABEL=连接标签 +com.oceanbase.odc.ResourceType.ODC_SESSION=连接会话 +com.oceanbase.odc.ResourceType.ODC_TASK=任务 +com.oceanbase.odc.ResourceType.ODC_SCRIPT=脚本 +com.oceanbase.odc.ResourceType.ODC_SNIPPET=代码片段 +com.oceanbase.odc.ResourceType.ODC_SOURCE_FILE=源代码文件 +com.oceanbase.odc.ResourceType.ODC_FILE=文件 +com.oceanbase.odc.ResourceType.ODC_RESOURCE_GROUP=资源组 +com.oceanbase.odc.ResourceType.ODC_FLOW_INSTANCE=流程实例 +com.oceanbase.odc.ResourceType.ODC_FLOW_CONFIG=流程配置 +com.oceanbase.odc.ResourceType.ODC_FLOW_APPROVAL_INSTANCE=审批节点 +com.oceanbase.odc.ResourceType.ODC_FLOW_TASK_INSTANCE=任务节点 +com.oceanbase.odc.ResourceType.ODC_FLOW_GATEWAY_INSTANCE=网关节点 +com.oceanbase.odc.ResourceType.ODC_ASYNC_SQL_RESULT=异步 SQL 执行结果 +com.oceanbase.odc.ResourceType.ODC_SYSTEM_CONFIG=系统配置 +com.oceanbase.odc.ResourceType.ODC_AUDIT_EVENT=审计事件 com.oceanbase.odc.ResourceType.ODC_BUCKET=OSS Bucket -com.oceanbase.odc.ResourceType.ODC_STORAGE_OBJECT_METADATA=\u5BF9\u8C61\u5143\u6570\u636E -com.oceanbase.odc.ResourceType.ODC_EXTERNAL_APPROVAL=\u5916\u90E8\u5BA1\u6279\u5355 -com.oceanbase.odc.ResourceType.ODC_TUTORIAL=\u6559\u7A0B -com.oceanbase.odc.ResourceType.OB_CLUSTER=OB \u96C6\u7FA4 -com.oceanbase.odc.ResourceType.OB_TENANT=OB \u79DF\u6237 -com.oceanbase.odc.ResourceType.OB_USER=\u6570\u636E\u5E93\u7528\u6237 -com.oceanbase.odc.ResourceType.OB_DATABASE=\u6570\u636E\u5E93 -com.oceanbase.odc.ResourceType.OB_TABLE=\u8868 -com.oceanbase.odc.ResourceType.OB_INDEX=\u7D22\u5F15 -com.oceanbase.odc.ResourceType.OB_COLUMN=\u5217 -com.oceanbase.odc.ResourceType.OB_CONSTRAINT=\u7EA6\u675F -com.oceanbase.odc.ResourceType.OB_VIEW=\u89C6\u56FE -com.oceanbase.odc.ResourceType.OB_SEQUENCE=\u5E8F\u5217 -com.oceanbase.odc.ResourceType.OB_TRIGGER=\u89E6\u53D1\u5668 -com.oceanbase.odc.ResourceType.OB_FUNCTION=\u51FD\u6570 -com.oceanbase.odc.ResourceType.OB_PROCEDURE=\u5B58\u50A8\u8FC7\u7A0B -com.oceanbase.odc.ResourceType.OB_PACKAGE=\u7A0B\u5E8F\u5305 -com.oceanbase.odc.ResourceType.OB_TYPE=\u7C7B\u578B -com.oceanbase.odc.ResourceType.OB_SYNONYM=\u540C\u4E49\u8BCD -com.oceanbase.odc.ResourceType.OB_SESSION=\u6570\u636E\u5E93\u4F1A\u8BDD -com.oceanbase.odc.ResourceType.ALIYUN_ACCOUNT=\u963F\u91CC\u4E91\u4E3B\u8D26\u53F7 -com.oceanbase.odc.ResourceType.ALIYUN_SUB_ACCOUNT=\u963F\u91CC\u4E91\u5B50\u8D26\u53F7 -com.oceanbase.odc.ResourceType.ODC_DATA_MASKING_RULE=\u8131\u654F\u89C4\u5219 -com.oceanbase.odc.ResourceType.ODC_DATA_MASKING_POLICY=\u8131\u654F\u7B56\u7565 -com.oceanbase.odc.ResourceType.ODC_SHADOWTABLE_COMPARING_TASK=\u5F71\u5B50\u8868\u7ED3\u6784\u5BF9\u6BD4\u4EFB\u52A1 -com.oceanbase.odc.ResourceType.ODC_SCHEDULE=\u5B9A\u65F6\u6267\u884C -com.oceanbase.odc.ResourceType.ODC_SCHEDULE_TRIGGER=\u4F5C\u4E1A\u89E6\u53D1\u5668 -com.oceanbase.odc.ResourceType.ODC_SCHEDULE_TASK=\u8C03\u5EA6\u4EFB\u52A1 -com.oceanbase.odc.ResourceType.ODC_DLM_LIMITER_CONFIG=\u9650\u6D41\u914D\u7F6E -com.oceanbase.odc.ResourceType.ODC_PL_DEBUG_SESSION=PL \u8C03\u8BD5\u4F1A\u8BDD -com.oceanbase.odc.ResourceType.ODC_JOB=\u4EFB\u52A1 -com.oceanbase.odc.ResourceType.ODC_AUTOMATION_RULE=\u81EA\u52A8\u5316\u89C4\u5219 -com.oceanbase.odc.ResourceType.ODC_EXTERNAL_SQL_INTERCEPTOR=\u5916\u90E8 SQL \u62E6\u622A\u5668 -com.oceanbase.odc.ResourceType.ODC_INTEGRATION=\u96C6\u6210 -com.oceanbase.odc.ResourceType.ODC_PROJECT=\u9879\u76EE -com.oceanbase.odc.ResourceType.ODC_ENVIRONMENT=\u73AF\u5883 -com.oceanbase.odc.ResourceType.ODC_DATASOURCE=\u6570\u636E\u6E90 -com.oceanbase.odc.ResourceType.ODC_DATABASE=\u6570\u636E\u5E93 -com.oceanbase.odc.ResourceType.ODC_LOGICAL_TABLE=\u903B\u8F91\u8868 -com.oceanbase.odc.ResourceType.ODC_TABLE=\u6570\u636E\u8868 -com.oceanbase.odc.ResourceType.ODC_RULESET=\u89C4\u5219\u96C6 -com.oceanbase.odc.ResourceType.ODC_RULE=\u89C4\u5219 -com.oceanbase.odc.ResourceType.ODC_SENSITIVE_COLUMN=\u654F\u611F\u5217 -com.oceanbase.odc.ResourceType.ODC_SENSITIVE_RULE=\u8BC6\u522B\u89C4\u5219 -com.oceanbase.odc.ResourceType.ODC_MASKING_ALGORITHM=\u8131\u654F\u7B97\u6CD5 -com.oceanbase.odc.ResourceType.ODC_SENSITIVE_COLUMN_SCANNING_TASK=\u654F\u611F\u5217\u626B\u63CF\u4EFB\u52A1 -com.oceanbase.odc.ResourceType.ODC_APPROVAL_FLOW_CONFIG=\u5BA1\u6279\u6D41\u914D\u7F6E -com.oceanbase.odc.ResourceType.ODC_RISK_LEVEL=\u98CE\u9669\u7B49\u7EA7 -com.oceanbase.odc.ResourceType.ODC_RISK_DETECT_RULE=\u98CE\u9669\u7B49\u7EA7\u8BC6\u522B\u89C4\u5219 -com.oceanbase.odc.ResourceType.ODC_INDIVIDUAL_ORGANIZATION=\u4E2A\u4EBA\u7A7A\u95F4 -com.oceanbase.odc.ResourceType.ODC_TEAM_ORGANIZATION=\u56E2\u961F\u7A7A\u95F4 +com.oceanbase.odc.ResourceType.ODC_STORAGE_OBJECT_METADATA=对象元数据 +com.oceanbase.odc.ResourceType.ODC_EXTERNAL_APPROVAL=外部审批单 +com.oceanbase.odc.ResourceType.ODC_TUTORIAL=教程 +com.oceanbase.odc.ResourceType.OB_CLUSTER=OB 集群 +com.oceanbase.odc.ResourceType.OB_TENANT=OB 租户 +com.oceanbase.odc.ResourceType.OB_USER=数据库用户 +com.oceanbase.odc.ResourceType.OB_DATABASE=数据库 +com.oceanbase.odc.ResourceType.OB_TABLE=表 +com.oceanbase.odc.ResourceType.OB_INDEX=索引 +com.oceanbase.odc.ResourceType.OB_COLUMN=列 +com.oceanbase.odc.ResourceType.OB_CONSTRAINT=约束 +com.oceanbase.odc.ResourceType.OB_VIEW=视图 +com.oceanbase.odc.ResourceType.OB_SEQUENCE=序列 +com.oceanbase.odc.ResourceType.OB_TRIGGER=触发器 +com.oceanbase.odc.ResourceType.OB_FUNCTION=函数 +com.oceanbase.odc.ResourceType.OB_PROCEDURE=存储过程 +com.oceanbase.odc.ResourceType.OB_PACKAGE=程序包 +com.oceanbase.odc.ResourceType.OB_TYPE=类型 +com.oceanbase.odc.ResourceType.OB_SYNONYM=同义词 +com.oceanbase.odc.ResourceType.OB_SESSION=数据库会话 +com.oceanbase.odc.ResourceType.ALIYUN_ACCOUNT=阿里云主账号 +com.oceanbase.odc.ResourceType.ALIYUN_SUB_ACCOUNT=阿里云子账号 +com.oceanbase.odc.ResourceType.ODC_DATA_MASKING_RULE=脱敏规则 +com.oceanbase.odc.ResourceType.ODC_DATA_MASKING_POLICY=脱敏策略 +com.oceanbase.odc.ResourceType.ODC_SHADOWTABLE_COMPARING_TASK=影子表结构对比任务 +com.oceanbase.odc.ResourceType.ODC_SCHEDULE=定时执行 +com.oceanbase.odc.ResourceType.ODC_SCHEDULE_TRIGGER=作业触发器 +com.oceanbase.odc.ResourceType.ODC_SCHEDULE_TASK=调度任务 +com.oceanbase.odc.ResourceType.ODC_DLM_LIMITER_CONFIG=限流配置 +com.oceanbase.odc.ResourceType.ODC_PL_DEBUG_SESSION=PL 调试会话 +com.oceanbase.odc.ResourceType.ODC_JOB=任务 +com.oceanbase.odc.ResourceType.ODC_AUTOMATION_RULE=自动化规则 +com.oceanbase.odc.ResourceType.ODC_EXTERNAL_SQL_INTERCEPTOR=外部 SQL 拦截器 +com.oceanbase.odc.ResourceType.ODC_INTEGRATION=集成 +com.oceanbase.odc.ResourceType.ODC_PROJECT=项目 +com.oceanbase.odc.ResourceType.ODC_ENVIRONMENT=环境 +com.oceanbase.odc.ResourceType.ODC_DATASOURCE=数据源 +com.oceanbase.odc.ResourceType.ODC_DATABASE=数据库 +com.oceanbase.odc.ResourceType.ODC_LOGICAL_TABLE=逻辑表 +com.oceanbase.odc.ResourceType.ODC_TABLE=数据表 +com.oceanbase.odc.ResourceType.ODC_RULESET=规则集 +com.oceanbase.odc.ResourceType.ODC_RULE=规则 +com.oceanbase.odc.ResourceType.ODC_SENSITIVE_COLUMN=敏感列 +com.oceanbase.odc.ResourceType.ODC_SENSITIVE_RULE=识别规则 +com.oceanbase.odc.ResourceType.ODC_MASKING_ALGORITHM=脱敏算法 +com.oceanbase.odc.ResourceType.ODC_SENSITIVE_COLUMN_SCANNING_TASK=敏感列扫描任务 +com.oceanbase.odc.ResourceType.ODC_APPROVAL_FLOW_CONFIG=审批流配置 +com.oceanbase.odc.ResourceType.ODC_RISK_LEVEL=风险等级 +com.oceanbase.odc.ResourceType.ODC_RISK_DETECT_RULE=风险等级识别规则 +com.oceanbase.odc.ResourceType.ODC_INDIVIDUAL_ORGANIZATION=个人空间 +com.oceanbase.odc.ResourceType.ODC_TEAM_ORGANIZATION=团队空间 -com.oceanbase.odc.ResourceType.ODC_NOTIFICATION_CHANNEL=\u901A\u77E5\u901A\u9053 -com.oceanbase.odc.ResourceType.ODC_NOTIFICATION_POLICY=\u901A\u77E5\u89C4\u5219 -com.oceanbase.odc.ResourceType.ODC_NOTIFICATION_MESSAGE=\u901A\u77E5\u6D88\u606F -com.oceanbase.odc.ResourceType.ODC_STRUCTURE_COMPARISON_TASK=\u7ED3\u6784\u5BF9\u6BD4\u4EFB\u52A1 -com.oceanbase.odc.ResourceType.ODC_DATABASE_CHANGE_ORDER_TEMPLATE=\u6570\u636E\u5E93\u53D8\u66F4\u987A\u5E8F\u6A21\u677F +com.oceanbase.odc.ResourceType.ODC_NOTIFICATION_CHANNEL=通知通道 +com.oceanbase.odc.ResourceType.ODC_NOTIFICATION_POLICY=通知规则 +com.oceanbase.odc.ResourceType.ODC_NOTIFICATION_MESSAGE=通知消息 +com.oceanbase.odc.ResourceType.ODC_STRUCTURE_COMPARISON_TASK=结构对比任务 +com.oceanbase.odc.ResourceType.ODC_DATABASE_CHANGE_ORDER_TEMPLATE=数据库变更顺序模板 # # Batch Import # -com.oceanbase.odc.FieldName.DATASOURCE_NAME=\u6570\u636E\u6E90\u540D\u79F0 -com.oceanbase.odc.FieldName.DATASOURCE_TYPE=\u6570\u636E\u6E90\u7C7B\u578B -com.oceanbase.odc.FieldName.DATASOURCE_HOST=\u4E3B\u673A IP -com.oceanbase.odc.FieldName.DATASOURCE_PORT=\u7AEF\u53E3 -com.oceanbase.odc.FieldName.DATASOURCE_CLUSTERNAME=\u96C6\u7FA4\u540D -com.oceanbase.odc.FieldName.DATASOURCE_TENANTNAME=\u79DF\u6237\u540D -com.oceanbase.odc.FieldName.DATASOURCE_USERNAME=\u6570\u636E\u5E93\u7528\u6237\u540D -com.oceanbase.odc.FieldName.DATASOURCE_PASSWORD=\u6570\u636E\u5E93\u5BC6\u7801 -com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT=\u73AF\u5883 -com.oceanbase.odc.FieldName.DATASOURCE_SYSTENANTUSERNAME=sys \u79DF\u6237\u8D26\u53F7 -com.oceanbase.odc.FieldName.DATASOURCE_SYSTENANTPASSWORD=sys \u79DF\u6237\u5BC6\u7801 -com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT_DEFAULT=\u9ED8\u8BA4 -com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT_DEV=\u5F00\u53D1 -com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT_PROD=\u751F\u4EA7 -com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT_SIT=\u6D4B\u8BD5 -com.oceanbase.odc.FieldName.USER_ACCOUNTNAME=\u8D26\u53F7 -com.oceanbase.odc.FieldName.USER_NAME=\u59D3\u540D -com.oceanbase.odc.FieldName.USER_PASSWORD=\u5BC6\u7801 -com.oceanbase.odc.FieldName.USER_ENABLED=\u8D26\u53F7\u72B6\u6001 -com.oceanbase.odc.FieldName.USER_ROLEIDS=\u89D2\u8272 -com.oceanbase.odc.FieldName.USER_DESCRIPTION=\u5907\u6CE8 +com.oceanbase.odc.FieldName.DATASOURCE_NAME=数据源名称 +com.oceanbase.odc.FieldName.DATASOURCE_TYPE=数据源类型 +com.oceanbase.odc.FieldName.DATASOURCE_HOST=主机 IP +com.oceanbase.odc.FieldName.DATASOURCE_PORT=端口 +com.oceanbase.odc.FieldName.DATASOURCE_CLUSTERNAME=集群名 +com.oceanbase.odc.FieldName.DATASOURCE_TENANTNAME=租户名 +com.oceanbase.odc.FieldName.DATASOURCE_USERNAME=数据库用户名 +com.oceanbase.odc.FieldName.DATASOURCE_PASSWORD=数据库密码 +com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT=环境 +com.oceanbase.odc.FieldName.DATASOURCE_SYSTENANTUSERNAME=sys 租户账号 +com.oceanbase.odc.FieldName.DATASOURCE_SYSTENANTPASSWORD=sys 租户密码 +com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT_DEFAULT=默认 +com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT_DEV=开发 +com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT_PROD=生产 +com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT_SIT=测试 +com.oceanbase.odc.FieldName.USER_ACCOUNTNAME=账号 +com.oceanbase.odc.FieldName.USER_NAME=姓名 +com.oceanbase.odc.FieldName.USER_PASSWORD=密码 +com.oceanbase.odc.FieldName.USER_ENABLED=账号状态 +com.oceanbase.odc.FieldName.USER_ROLEIDS=角色 +com.oceanbase.odc.FieldName.USER_DESCRIPTION=备注 -com.oceanbase.odc.Symbols.LEFT_BRACKET=\u3010 -com.oceanbase.odc.Symbols.RIGHT_BRACKET=\u3011 -com.oceanbase.odc.Symbols.COMMA=\uFF0C +com.oceanbase.odc.Symbols.LEFT_BRACKET=【 +com.oceanbase.odc.Symbols.RIGHT_BRACKET=】 +com.oceanbase.odc.Symbols.COMMA=， # # LimitMetric # -com.oceanbase.odc.LimitMetric.TRANSFER_TASK_COUNT=\u5BFC\u5165\u5BFC\u51FA\u4EFB\u52A1\u6570\u91CF -com.oceanbase.odc.LimitMetric.MOCK_TASK_COUNT=\u6A21\u62DF\u6570\u636E\u4EFB\u52A1\u6570\u91CF -com.oceanbase.odc.LimitMetric.OBCLIENT_INSTANCE_COUNT=\u547D\u4EE4\u884C\u7A97\u53E3\u6570\u91CF -com.oceanbase.odc.LimitMetric.FAILED_LOGIN_ATTEMPT_COUNT=\u767B\u5F55\u5931\u8D25\u5C1D\u8BD5\u6B21\u6570 -com.oceanbase.odc.LimitMetric.SQL_LENGTH=SQL \u8BED\u53E5\u5B57\u7B26\u6570 -com.oceanbase.odc.LimitMetric.SQL_SIZE=SQL \u8BED\u53E5\u5B57\u8282\u6570 -com.oceanbase.odc.LimitMetric.SQL_STATEMENT_COUNT=SQL \u8BED\u53E5\u6570\u91CF -com.oceanbase.odc.LimitMetric.FILE_SIZE=\u6587\u4EF6\u5B57\u8282\u6570 -com.oceanbase.odc.LimitMetric.FILE_COUNT=\u6587\u4EF6\u6570\u91CF -com.oceanbase.odc.LimitMetric.TRANSACTION_QUERY_LIMIT=\u67E5\u8BE2\u7ED3\u679C\u96C6\u5927\u5C0F -com.oceanbase.odc.LimitMetric.SESSION_COUNT=\u6570\u636E\u5E93 SESSION \u6570\u91CF -com.oceanbase.odc.LimitMetric.USER_COUNT=\u6570\u636E\u5E93\u8FDE\u63A5\u7528\u6237\u6570 -com.oceanbase.odc.LimitMetric.EXPORT_OBJECT_COUNT=\u5BFC\u51FA\u5BF9\u8C61\u6570\u91CF -com.oceanbase.odc.LimitMetric.TABLE_NAME_LENGTH=\u8868\u540D\u957F\u5EA6 +com.oceanbase.odc.LimitMetric.TRANSFER_TASK_COUNT=导入导出任务数量 +com.oceanbase.odc.LimitMetric.MOCK_TASK_COUNT=模拟数据任务数量 +com.oceanbase.odc.LimitMetric.OBCLIENT_INSTANCE_COUNT=命令行窗口数量 +com.oceanbase.odc.LimitMetric.FAILED_LOGIN_ATTEMPT_COUNT=登录失败尝试次数 +com.oceanbase.odc.LimitMetric.SQL_LENGTH=SQL 语句字符数 +com.oceanbase.odc.LimitMetric.SQL_SIZE=SQL 语句字节数 +com.oceanbase.odc.LimitMetric.SQL_STATEMENT_COUNT=SQL 语句数量 +com.oceanbase.odc.LimitMetric.FILE_SIZE=文件字节数 +com.oceanbase.odc.LimitMetric.FILE_COUNT=文件数量 +com.oceanbase.odc.LimitMetric.TRANSACTION_QUERY_LIMIT=查询结果集大小 +com.oceanbase.odc.LimitMetric.SESSION_COUNT=数据库 SESSION 数量 +com.oceanbase.odc.LimitMetric.USER_COUNT=数据库连接用户数 +com.oceanbase.odc.LimitMetric.EXPORT_OBJECT_COUNT=导出对象数量 +com.oceanbase.odc.LimitMetric.TABLE_NAME_LENGTH=表名长度 # # ConnectionAccountType # -com.oceanbase.odc.ConnectionAccountType.MAIN=\u8BFB\u5199\u8D26\u53F7 -com.oceanbase.odc.ConnectionAccountType.READONLY=\u53EA\u8BFB\u8D26\u53F7 -com.oceanbase.odc.ConnectionAccountType.SYS_READ=SYS\u79DF\u6237\u8D26\u53F7 +com.oceanbase.odc.ConnectionAccountType.MAIN=读写账号 +com.oceanbase.odc.ConnectionAccountType.READONLY=只读账号 +com.oceanbase.odc.ConnectionAccountType.SYS_READ=SYS租户账号 # # AuditEventAction # -com.oceanbase.odc.AuditEventAction.DISABLE_ROLE=\u7981\u7528\u89D2\u8272 -com.oceanbase.odc.AuditEventAction.DELETE_ROLE=\u5220\u9664\u89D2\u8272 -com.oceanbase.odc.AuditEventAction.DELETE_DATA_MASKING_RULE=\u5220\u9664\u8131\u654F\u89C4\u5219 -com.oceanbase.odc.AuditEventAction.UPDATE_PERSONAL_CONFIGURATION=\u66F4\u65B0\u4E2A\u4EBA\u914D\u7F6E -com.oceanbase.odc.AuditEventAction.SET_PASSWORD=\u8BBE\u7F6E\u5BC6\u7801 -com.oceanbase.odc.AuditEventAction.CHANGE_PASSWORD=\u4FEE\u6539\u5BC6\u7801 -com.oceanbase.odc.AuditEventAction.RESET_PASSWORD=\u91CD\u7F6E\u5BC6\u7801 -com.oceanbase.odc.AuditEventAction.CREATE_CONNECTION=\u521B\u5EFA\u8FDE\u63A5 -com.oceanbase.odc.AuditEventAction.DELETE_CONNECTION=\u5220\u9664\u8FDE\u63A5 -com.oceanbase.odc.AuditEventAction.UPDATE_CONNECTION=\u66F4\u65B0\u8FDE\u63A5 -com.oceanbase.odc.AuditEventAction.CREATE_SESSION=\u521B\u5EFA\u4F1A\u8BDD -com.oceanbase.odc.AuditEventAction.CLOSE_SESSION=\u5173\u95ED\u4F1A\u8BDD -com.oceanbase.odc.AuditEventAction.ENABLE_CONNECTION=\u542F\u7528\u8FDE\u63A5 -com.oceanbase.odc.AuditEventAction.DISABLE_CONNECTION=\u7981\u7528\u8FDE\u63A5 -com.oceanbase.odc.AuditEventAction.DOWNLOAD_SCRIPT=\u4E0B\u8F7D\u811A\u672C -com.oceanbase.odc.AuditEventAction.UPDATE_SCRIPT=\u66F4\u65B0\u811A\u672C -com.oceanbase.odc.AuditEventAction.DELETE_SCRIPT=\u5220\u9664\u811A\u672C -com.oceanbase.odc.AuditEventAction.UPLOAD_SCRIPT=\u4E0A\u4F20\u811A\u672C -com.oceanbase.odc.AuditEventAction.UPDATE_ORGANIZATION_CONFIGURATION=\u66F4\u65B0\u7EC4\u7EC7\u914D\u7F6E -com.oceanbase.odc.AuditEventAction.ADD_USER=\u589E\u52A0\u7528\u6237 -com.oceanbase.odc.AuditEventAction.UPDATE_USER=\u66F4\u65B0\u7528\u6237 -com.oceanbase.odc.AuditEventAction.DELETE_USER=\u5220\u9664\u7528\u6237 -com.oceanbase.odc.AuditEventAction.ENABLE_USER=\u542F\u7528\u7528\u6237 -com.oceanbase.odc.AuditEventAction.DISABLE_USER=\u7981\u7528\u7528\u6237 -com.oceanbase.odc.AuditEventAction.ADD_ROLE=\u589E\u52A0\u89D2\u8272 -com.oceanbase.odc.AuditEventAction.UPDATE_ROLE=\u66F4\u65B0\u89D2\u8272 -com.oceanbase.odc.AuditEventAction.ENABLE_ROLE=\u542F\u7528\u89D2\u8272 -com.oceanbase.odc.AuditEventAction.ADD_RESOURCE_GROUP=\u589E\u52A0\u8D44\u6E90\u7EC4 -com.oceanbase.odc.AuditEventAction.UPDATE_RESOURCE_GROUP=\u66F4\u65B0\u8D44\u6E90\u7EC4 -com.oceanbase.odc.AuditEventAction.DELETE_RESOURCE_GROUP=\u5220\u9664\u8D44\u6E90\u7EC4 -com.oceanbase.odc.AuditEventAction.ENABLE_RESOURCE_GROUP=\u542F\u7528\u8D44\u6E90\u7EC4 -com.oceanbase.odc.AuditEventAction.DISABLE_RESOURCE_GROUP=\u7981\u7528\u8D44\u6E90\u7EC4 +com.oceanbase.odc.AuditEventAction.DISABLE_ROLE=禁用角色 +com.oceanbase.odc.AuditEventAction.DELETE_ROLE=删除角色 +com.oceanbase.odc.AuditEventAction.DELETE_DATA_MASKING_RULE=删除脱敏规则 +com.oceanbase.odc.AuditEventAction.UPDATE_PERSONAL_CONFIGURATION=更新个人配置 +com.oceanbase.odc.AuditEventAction.SET_PASSWORD=设置密码 +com.oceanbase.odc.AuditEventAction.CHANGE_PASSWORD=修改密码 +com.oceanbase.odc.AuditEventAction.RESET_PASSWORD=重置密码 +com.oceanbase.odc.AuditEventAction.CREATE_CONNECTION=创建连接 +com.oceanbase.odc.AuditEventAction.DELETE_CONNECTION=删除连接 +com.oceanbase.odc.AuditEventAction.UPDATE_CONNECTION=更新连接 +com.oceanbase.odc.AuditEventAction.CREATE_SESSION=创建会话 +com.oceanbase.odc.AuditEventAction.CLOSE_SESSION=关闭会话 +com.oceanbase.odc.AuditEventAction.ENABLE_CONNECTION=启用连接 +com.oceanbase.odc.AuditEventAction.DISABLE_CONNECTION=禁用连接 +com.oceanbase.odc.AuditEventAction.DOWNLOAD_SCRIPT=下载脚本 +com.oceanbase.odc.AuditEventAction.UPDATE_SCRIPT=更新脚本 +com.oceanbase.odc.AuditEventAction.DELETE_SCRIPT=删除脚本 +com.oceanbase.odc.AuditEventAction.UPLOAD_SCRIPT=上传脚本 +com.oceanbase.odc.AuditEventAction.UPDATE_ORGANIZATION_CONFIGURATION=更新组织配置 +com.oceanbase.odc.AuditEventAction.ADD_USER=增加用户 +com.oceanbase.odc.AuditEventAction.UPDATE_USER=更新用户 +com.oceanbase.odc.AuditEventAction.DELETE_USER=删除用户 +com.oceanbase.odc.AuditEventAction.ENABLE_USER=启用用户 +com.oceanbase.odc.AuditEventAction.DISABLE_USER=禁用用户 +com.oceanbase.odc.AuditEventAction.ADD_ROLE=增加角色 +com.oceanbase.odc.AuditEventAction.UPDATE_ROLE=更新角色 +com.oceanbase.odc.AuditEventAction.ENABLE_ROLE=启用角色 +com.oceanbase.odc.AuditEventAction.ADD_RESOURCE_GROUP=增加资源组 +com.oceanbase.odc.AuditEventAction.UPDATE_RESOURCE_GROUP=更新资源组 +com.oceanbase.odc.AuditEventAction.DELETE_RESOURCE_GROUP=删除资源组 +com.oceanbase.odc.AuditEventAction.ENABLE_RESOURCE_GROUP=启用资源组 +com.oceanbase.odc.AuditEventAction.DISABLE_RESOURCE_GROUP=禁用资源组 com.oceanbase.odc.AuditEventAction.SELECT=SELECT com.oceanbase.odc.AuditEventAction.DELETE=DELETE com.oceanbase.odc.AuditEventAction.INSERT=INSERT @@ -182,652 +182,652 @@ com.oceanbase.odc.AuditEventAction.ALTER=ALTER com.oceanbase.odc.AuditEventAction.TRUNCATE=TRUNCATE com.oceanbase.odc.AuditEventAction.CREATE=CREATE com.oceanbase.odc.AuditEventAction.OTHERS=OTHERS -com.oceanbase.odc.AuditEventAction.EXPORT_AUDIT_EVENT=\u5BFC\u51FA\u5BA1\u8BA1\u4E8B\u4EF6 -com.oceanbase.odc.AuditEventAction.CREATE_FLOW_CONFIG=\u521B\u5EFA\u6D41\u7A0B\u914D\u7F6E -com.oceanbase.odc.AuditEventAction.UPDATE_FLOW_CONFIG=\u66F4\u65B0\u6D41\u7A0B\u914D\u7F6E -com.oceanbase.odc.AuditEventAction.ENABLE_FLOW_CONFIG=\u542F\u7528\u6D41\u7A0B\u914D\u7F6E -com.oceanbase.odc.AuditEventAction.DISABLE_FLOW_CONFIG=\u7981\u7528\u6D41\u7A0B\u914D\u7F6E -com.oceanbase.odc.AuditEventAction.DELETE_FLOW_CONFIG=\u5220\u9664\u6D41\u7A0B\u914D\u7F6E -com.oceanbase.odc.AuditEventAction.BATCH_DELETE_FLOW_CONFIG=\u6279\u91CF\u5220\u9664\u6D41\u7A0B\u914D\u7F6E -com.oceanbase.odc.AuditEventAction.CREATE_TASK=\u521B\u5EFA\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.STOP_TASK=\u505C\u6B62\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.ROLLBACK_TASK=\u56DE\u6EDA\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.EXECUTE_TASK=\u6267\u884C\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.APPROVE=\u540C\u610F -com.oceanbase.odc.AuditEventAction.REJECT=\u62D2\u7EDD -com.oceanbase.odc.AuditEventAction.CREATE_ASYNC_TASK=\u521B\u5EFA\u6570\u636E\u5E93\u53D8\u66F4\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.CREATE_MULTIPLE_ASYNC_TASK=\u521B\u5EFA\u591A\u5E93\u53D8\u66F4\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.CREATE_MOCKDATA_TASK=\u521B\u5EFA\u6A21\u62DF\u6570\u636E\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.CREATE_IMPORT_TASK=\u521B\u5EFA\u5BFC\u5165\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.CREATE_EXPORT_TASK=\u521B\u5EFA\u5BFC\u51FA\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.CREATE_EXPORT_RESULT_SET_TASK=\u521B\u5EFA\u5BFC\u51FA\u7ED3\u679C\u96C6\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.CREATE_PERMISSION_APPLY_TASK=\u521B\u5EFA\u6743\u9650\u7533\u8BF7\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.CREATE_SHADOWTABLE_SYNC_TASK=\u521B\u5EFA\u5F71\u5B50\u8868\u540C\u6B65\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.CREATE_STRUCTURE_COMPARISON_TASK=\u521B\u5EFA\u7ED3\u6784\u6BD4\u5BF9\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.CREATE_PARTITION_PLAN_TASK=\u521B\u5EFA\u5206\u533A\u8BA1\u5212\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.CREATE_ALTER_SCHEDULE_TASK=\u521B\u5EFA\u4FEE\u6539\u8C03\u5EA6\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.CREATE_ONLINE_SCHEMA_CHANGE_TASK=\u521B\u5EFA\u65E0\u9501\u7ED3\u6784\u53D8\u66F4\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.CREATE_APPLY_PROJECT_PERMISSION_TASK=\u521B\u5EFA\u7533\u8BF7\u9879\u76EE\u6743\u9650\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.CREATE_APPLY_DATABASE_PERMISSION_TASK=\u521B\u5EFA\u7533\u8BF7\u6570\u636E\u5E93\u6743\u9650\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.CREATE_APPLY_TABLE_PERMISSION_TASK=\u521B\u5EFA\u7533\u8BF7\u6570\u636E\u8868\u6743\u9650\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.STOP_ASYNC_TASK=\u505C\u6B62\u6570\u636E\u5E93\u53D8\u66F4\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.STOP_MULTIPLE_ASYNC_TASK=\u505C\u6B62\u591A\u5E93\u53D8\u66F4\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.STOP_MOCKDATA_TASK=\u505C\u6B62\u6A21\u62DF\u6570\u636E\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.STOP_IMPORT_TASK=\u505C\u6B62\u5BFC\u5165\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.STOP_EXPORT_TASK=\u505C\u6B62\u5BFC\u51FA\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.STOP_EXPORT_RESULT_SET_TASK=\u505C\u6B62\u5BFC\u51FA\u7ED3\u679C\u96C6\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.STOP_SHADOWTABLE_SYNC_TASK=\u505C\u6B62\u5F71\u5B50\u8868\u540C\u6B65\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.STOP_STRUCTURE_COMPARISON_TASK=\u505C\u6B62\u7ED3\u6784\u6BD4\u5BF9\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.STOP_PARTITION_PLAN_TASK=\u505C\u6B62\u5206\u533A\u8BA1\u5212\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.STOP_ALTER_SCHEDULE_TASK=\u505C\u6B62\u4FEE\u6539\u8C03\u5EA6\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.STOP_ONLINE_SCHEMA_CHANGE_TASK=\u505C\u6B62\u65E0\u9501\u7ED3\u6784\u53D8\u66F4\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.STOP_APPLY_PROJECT_PERMISSION_TASK=\u505C\u6B62\u7533\u8BF7\u9879\u76EE\u6743\u9650\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.STOP_APPLY_DATABASE_PERMISSION_TASK=\u505C\u6B62\u7533\u8BF7\u6570\u636E\u5E93\u6743\u9650\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.STOP_APPLY_TABLE_PERMISSION_TASK=\u505C\u6B62\u7533\u8BF7\u6570\u636E\u8868\u6743\u9650\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.EXECUTE_ASYNC_TASK=\u6267\u884C\u6570\u636E\u5E93\u53D8\u66F4\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.EXECUTE_MULTIPLE_ASYNC_TASK=\u6267\u884C\u591A\u5E93\u53D8\u66F4\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.EXECUTE_MOCKDATA_TASK=\u6267\u884C\u6A21\u62DF\u6570\u636E\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.EXECUTE_IMPORT_TASK=\u6267\u884C\u5BFC\u5165\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.EXECUTE_EXPORT_TASK=\u6267\u884C\u5BFC\u51FA\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.EXECUTE_EXPORT_RESULT_SET_TASK=\u6267\u884C\u5BFC\u51FA\u7ED3\u679C\u96C6\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.EXECUTE_SHADOWTABLE_SYNC_TASK=\u6267\u884C\u5F71\u5B50\u8868\u540C\u6B65\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.EXECUTE_STRUCTURE_COMPARISON_TASK=\u6267\u884C\u7ED3\u6784\u6BD4\u5BF9\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.EXECUTE_PARTITION_PLAN_TASK=\u6267\u884C\u5206\u533A\u8BA1\u5212\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.EXECUTE_ALTER_SCHEDULE_TASK=\u6267\u884C\u4FEE\u6539\u8C03\u5EA6\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.EXECUTE_ONLINE_SCHEMA_CHANGE_TASK=\u6267\u884C\u65E0\u9501\u7ED3\u6784\u53D8\u66F4\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.APPROVE_ASYNC_TASK=\u540C\u610F\u6570\u636E\u5E93\u53D8\u66F4\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.APPROVE_MULTIPLE_ASYNC_TASK=\u540C\u610F\u591A\u5E93\u53D8\u66F4\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.APPROVE_MOCKDATA_TASK=\u540C\u610F\u6A21\u62DF\u6570\u636E\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.APPROVE_IMPORT_TASK=\u540C\u610F\u5BFC\u5165\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.APPROVE_EXPORT_TASK=\u540C\u610F\u5BFC\u51FA\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.APPROVE_EXPORT_RESULT_SET_TASK=\u540C\u610F\u5BFC\u51FA\u7ED3\u679C\u96C6\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.APPROVE_PERMISSION_APPLY_TASK=\u540C\u610F\u6743\u9650\u7533\u8BF7\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.APPROVE_SHADOWTABLE_SYNC_TASK=\u540C\u610F\u5F71\u5B50\u8868\u540C\u6B65\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.APPROVE_STRUCTURE_COMPARISON_TASK=\u540C\u610F\u7ED3\u6784\u6BD4\u5BF9\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.APPROVE_PARTITION_PLAN_TASK=\u540C\u610F\u5206\u533A\u8BA1\u5212\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.APPROVE_ALTER_SCHEDULE_TASK=\u540C\u610F\u4FEE\u6539\u8C03\u5EA6\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.APPROVE_ONLINE_SCHEMA_CHANGE_TASK=\u540C\u610F\u65E0\u9501\u7ED3\u6784\u53D8\u66F4\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_PROJECT_PERMISSION_TASK=\u540C\u610F\u7533\u8BF7\u9879\u76EE\u6743\u9650\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_DATABASE_PERMISSION_TASK=\u540C\u610F\u7533\u8BF7\u6570\u636E\u5E93\u6743\u9650\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_TABLE_PERMISSION_TASK=\u540C\u610F\u7533\u8BF7\u6570\u636E\u8868\u6743\u9650\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.REJECT_ASYNC_TASK=\u62D2\u7EDD\u6570\u636E\u5E93\u53D8\u66F4\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.REJECT_MULTIPLE_ASYNC_TASK=\u62D2\u7EDD\u591A\u5E93\u53D8\u66F4\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.REJECT_MOCKDATA_TASK=\u62D2\u7EDD\u6A21\u62DF\u6570\u636E\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.REJECT_IMPORT_TASK=\u62D2\u7EDD\u5BFC\u5165\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.REJECT_EXPORT_TASK=\u62D2\u7EDD\u5BFC\u51FA\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.REJECT_EXPORT_RESULT_SET_TASK=\u62D2\u7EDD\u5BFC\u51FA\u7ED3\u679C\u96C6\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.REJECT_PERMISSION_APPLY_TASK=\u62D2\u7EDD\u6743\u9650\u7533\u8BF7\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.REJECT_SHADOWTABLE_SYNC_TASK=\u62D2\u7EDD\u5F71\u5B50\u8868\u540C\u6B65\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.REJECT_STRUCTURE_COMPARISON_TASK=\u62D2\u7EDD\u7ED3\u6784\u6BD4\u5BF9\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.REJECT_PARTITION_PLAN_TASK=\u62D2\u7EDD\u5206\u533A\u8BA1\u5212\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.REJECT_ALTER_SCHEDULE_TASK=\u62D2\u7EDD\u4FEE\u6539\u8C03\u5EA6\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.REJECT_ONLINE_SCHEMA_CHANGE_TASK=\u62D2\u7EDD\u65E0\u9501\u7ED3\u6784\u53D8\u66F4\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.REJECT_APPLY_PROJECT_PERMISSION_TASK=\u62D2\u7EDD\u7533\u8BF7\u9879\u76EE\u6743\u9650\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.REJECT_APPLY_DATABASE_PERMISSION_TASK=\u62D2\u7EDD\u7533\u8BF7\u6570\u636E\u5E93\u6743\u9650\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.REJECT_APPLY_TABLE_PERMISSION_TASK=\u62D2\u7EDD\u7533\u8BF7\u6570\u636E\u8868\u6743\u9650\u4EFB\u52A1 -com.oceanbase.odc.AuditEventAction.CREATE_DATA_MASKING_RULE=\u521B\u5EFA\u8131\u654F\u89C4\u5219 -com.oceanbase.odc.AuditEventAction.UPDATE_DATA_MASKING_RULE=\u66F4\u65B0\u8131\u654F\u89C4\u5219 -com.oceanbase.odc.AuditEventAction.ENABLE_DATA_MASKING_RULE=\u542F\u7528\u8131\u654F\u89C4\u5219 -com.oceanbase.odc.AuditEventAction.DISABLE_DATA_MASKING_RULE=\u7981\u7528\u8131\u654F\u89C4\u5219 -com.oceanbase.odc.AuditEventAction.CREATE_DATA_MASKING_POLICY=\u521B\u5EFA\u8131\u654F\u7B56\u7565 -com.oceanbase.odc.AuditEventAction.UPDATE_DATA_MASKING_POLICY=\u66F4\u65B0\u8131\u654F\u7B56\u7565 -com.oceanbase.odc.AuditEventAction.DELETE_DATA_MASKING_POLICY=\u5220\u9664\u8131\u654F\u7B56\u7565 -com.oceanbase.odc.AuditEventAction.CREATE_INTEGRATION=\u521B\u5EFA\u96C6\u6210 -com.oceanbase.odc.AuditEventAction.ENABLE_INTEGRATION=\u542F\u7528\u96C6\u6210 -com.oceanbase.odc.AuditEventAction.DISABLE_INTEGRATION=\u7981\u7528\u96C6\u6210 -com.oceanbase.odc.AuditEventAction.DELETE_INTEGRATION=\u5220\u9664\u96C6\u6210 -com.oceanbase.odc.AuditEventAction.ADD_DATABASE=\u589E\u52A0\u6570\u636E\u5E93 -com.oceanbase.odc.AuditEventAction.DELETE_DATABASE=\u5220\u9664\u6570\u636E\u5E93 -com.oceanbase.odc.AuditEventAction.TRANSFER_DATABASE_TO_PROJECT=\u8F6C\u79FB\u9879\u76EE -com.oceanbase.odc.AuditEventAction.CREATE_DATASOURCE=\u521B\u5EFA\u6570\u636E\u6E90 -com.oceanbase.odc.AuditEventAction.DELETE_DATASOURCE=\u5220\u9664\u6570\u636E\u6E90 -com.oceanbase.odc.AuditEventAction.UPDATE_DATASOURCE=\u66F4\u65B0\u6570\u636E\u6E90 -com.oceanbase.odc.AuditEventAction.CREATE_PROJECT=\u521B\u5EFA\u9879\u76EE -com.oceanbase.odc.AuditEventAction.UPDATE_SQL_SECURITY_RULE=\u4FEE\u6539 SQL \u5B89\u5168\u89C4\u5219 -com.oceanbase.odc.AuditEventAction.GRANT_DATABASE_PERMISSION=\u65B0\u589E\u5E93\u6743\u9650 -com.oceanbase.odc.AuditEventAction.REVOKE_DATABASE_PERMISSION=\u56DE\u6536\u5E93\u6743\u9650 -com.oceanbase.odc.AuditEventAction.GRANT_TABLE_PERMISSION=\u65B0\u589E\u8868\u6743\u9650 -com.oceanbase.odc.AuditEventAction.REVOKE_TABLE_PERMISSION=\u56DE\u6536\u8868\u6743\u9650 +com.oceanbase.odc.AuditEventAction.EXPORT_AUDIT_EVENT=导出审计事件 +com.oceanbase.odc.AuditEventAction.CREATE_FLOW_CONFIG=创建流程配置 +com.oceanbase.odc.AuditEventAction.UPDATE_FLOW_CONFIG=更新流程配置 +com.oceanbase.odc.AuditEventAction.ENABLE_FLOW_CONFIG=启用流程配置 +com.oceanbase.odc.AuditEventAction.DISABLE_FLOW_CONFIG=禁用流程配置 +com.oceanbase.odc.AuditEventAction.DELETE_FLOW_CONFIG=删除流程配置 +com.oceanbase.odc.AuditEventAction.BATCH_DELETE_FLOW_CONFIG=批量删除流程配置 +com.oceanbase.odc.AuditEventAction.CREATE_TASK=创建任务 +com.oceanbase.odc.AuditEventAction.STOP_TASK=停止任务 +com.oceanbase.odc.AuditEventAction.ROLLBACK_TASK=回滚任务 +com.oceanbase.odc.AuditEventAction.EXECUTE_TASK=执行任务 +com.oceanbase.odc.AuditEventAction.APPROVE=同意 +com.oceanbase.odc.AuditEventAction.REJECT=拒绝 +com.oceanbase.odc.AuditEventAction.CREATE_ASYNC_TASK=创建数据库变更任务 +com.oceanbase.odc.AuditEventAction.CREATE_MULTIPLE_ASYNC_TASK=创建多库变更任务 +com.oceanbase.odc.AuditEventAction.CREATE_MOCKDATA_TASK=创建模拟数据任务 +com.oceanbase.odc.AuditEventAction.CREATE_IMPORT_TASK=创建导入任务 +com.oceanbase.odc.AuditEventAction.CREATE_EXPORT_TASK=创建导出任务 +com.oceanbase.odc.AuditEventAction.CREATE_EXPORT_RESULT_SET_TASK=创建导出结果集任务 +com.oceanbase.odc.AuditEventAction.CREATE_PERMISSION_APPLY_TASK=创建权限申请任务 +com.oceanbase.odc.AuditEventAction.CREATE_SHADOWTABLE_SYNC_TASK=创建影子表同步任务 +com.oceanbase.odc.AuditEventAction.CREATE_STRUCTURE_COMPARISON_TASK=创建结构比对任务 +com.oceanbase.odc.AuditEventAction.CREATE_PARTITION_PLAN_TASK=创建分区计划任务 +com.oceanbase.odc.AuditEventAction.CREATE_ALTER_SCHEDULE_TASK=创建修改调度任务 +com.oceanbase.odc.AuditEventAction.CREATE_ONLINE_SCHEMA_CHANGE_TASK=创建无锁结构变更任务 +com.oceanbase.odc.AuditEventAction.CREATE_APPLY_PROJECT_PERMISSION_TASK=创建申请项目权限任务 +com.oceanbase.odc.AuditEventAction.CREATE_APPLY_DATABASE_PERMISSION_TASK=创建申请数据库权限任务 +com.oceanbase.odc.AuditEventAction.CREATE_APPLY_TABLE_PERMISSION_TASK=创建申请数据表权限任务 +com.oceanbase.odc.AuditEventAction.STOP_ASYNC_TASK=停止数据库变更任务 +com.oceanbase.odc.AuditEventAction.STOP_MULTIPLE_ASYNC_TASK=停止多库变更任务 +com.oceanbase.odc.AuditEventAction.STOP_MOCKDATA_TASK=停止模拟数据任务 +com.oceanbase.odc.AuditEventAction.STOP_IMPORT_TASK=停止导入任务 +com.oceanbase.odc.AuditEventAction.STOP_EXPORT_TASK=停止导出任务 +com.oceanbase.odc.AuditEventAction.STOP_EXPORT_RESULT_SET_TASK=停止导出结果集任务 +com.oceanbase.odc.AuditEventAction.STOP_SHADOWTABLE_SYNC_TASK=停止影子表同步任务 +com.oceanbase.odc.AuditEventAction.STOP_STRUCTURE_COMPARISON_TASK=停止结构比对任务 +com.oceanbase.odc.AuditEventAction.STOP_PARTITION_PLAN_TASK=停止分区计划任务 +com.oceanbase.odc.AuditEventAction.STOP_ALTER_SCHEDULE_TASK=停止修改调度任务 +com.oceanbase.odc.AuditEventAction.STOP_ONLINE_SCHEMA_CHANGE_TASK=停止无锁结构变更任务 +com.oceanbase.odc.AuditEventAction.STOP_APPLY_PROJECT_PERMISSION_TASK=停止申请项目权限任务 +com.oceanbase.odc.AuditEventAction.STOP_APPLY_DATABASE_PERMISSION_TASK=停止申请数据库权限任务 +com.oceanbase.odc.AuditEventAction.STOP_APPLY_TABLE_PERMISSION_TASK=停止申请数据表权限任务 +com.oceanbase.odc.AuditEventAction.EXECUTE_ASYNC_TASK=执行数据库变更任务 +com.oceanbase.odc.AuditEventAction.EXECUTE_MULTIPLE_ASYNC_TASK=执行多库变更任务 +com.oceanbase.odc.AuditEventAction.EXECUTE_MOCKDATA_TASK=执行模拟数据任务 +com.oceanbase.odc.AuditEventAction.EXECUTE_IMPORT_TASK=执行导入任务 +com.oceanbase.odc.AuditEventAction.EXECUTE_EXPORT_TASK=执行导出任务 +com.oceanbase.odc.AuditEventAction.EXECUTE_EXPORT_RESULT_SET_TASK=执行导出结果集任务 +com.oceanbase.odc.AuditEventAction.EXECUTE_SHADOWTABLE_SYNC_TASK=执行影子表同步任务 +com.oceanbase.odc.AuditEventAction.EXECUTE_STRUCTURE_COMPARISON_TASK=执行结构比对任务 +com.oceanbase.odc.AuditEventAction.EXECUTE_PARTITION_PLAN_TASK=执行分区计划任务 +com.oceanbase.odc.AuditEventAction.EXECUTE_ALTER_SCHEDULE_TASK=执行修改调度任务 +com.oceanbase.odc.AuditEventAction.EXECUTE_ONLINE_SCHEMA_CHANGE_TASK=执行无锁结构变更任务 +com.oceanbase.odc.AuditEventAction.APPROVE_ASYNC_TASK=同意数据库变更任务 +com.oceanbase.odc.AuditEventAction.APPROVE_MULTIPLE_ASYNC_TASK=同意多库变更任务 +com.oceanbase.odc.AuditEventAction.APPROVE_MOCKDATA_TASK=同意模拟数据任务 +com.oceanbase.odc.AuditEventAction.APPROVE_IMPORT_TASK=同意导入任务 +com.oceanbase.odc.AuditEventAction.APPROVE_EXPORT_TASK=同意导出任务 +com.oceanbase.odc.AuditEventAction.APPROVE_EXPORT_RESULT_SET_TASK=同意导出结果集任务 +com.oceanbase.odc.AuditEventAction.APPROVE_PERMISSION_APPLY_TASK=同意权限申请任务 +com.oceanbase.odc.AuditEventAction.APPROVE_SHADOWTABLE_SYNC_TASK=同意影子表同步任务 +com.oceanbase.odc.AuditEventAction.APPROVE_STRUCTURE_COMPARISON_TASK=同意结构比对任务 +com.oceanbase.odc.AuditEventAction.APPROVE_PARTITION_PLAN_TASK=同意分区计划任务 +com.oceanbase.odc.AuditEventAction.APPROVE_ALTER_SCHEDULE_TASK=同意修改调度任务 +com.oceanbase.odc.AuditEventAction.APPROVE_ONLINE_SCHEMA_CHANGE_TASK=同意无锁结构变更任务 +com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_PROJECT_PERMISSION_TASK=同意申请项目权限任务 +com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_DATABASE_PERMISSION_TASK=同意申请数据库权限任务 +com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_TABLE_PERMISSION_TASK=同意申请数据表权限任务 +com.oceanbase.odc.AuditEventAction.REJECT_ASYNC_TASK=拒绝数据库变更任务 +com.oceanbase.odc.AuditEventAction.REJECT_MULTIPLE_ASYNC_TASK=拒绝多库变更任务 +com.oceanbase.odc.AuditEventAction.REJECT_MOCKDATA_TASK=拒绝模拟数据任务 +com.oceanbase.odc.AuditEventAction.REJECT_IMPORT_TASK=拒绝导入任务 +com.oceanbase.odc.AuditEventAction.REJECT_EXPORT_TASK=拒绝导出任务 +com.oceanbase.odc.AuditEventAction.REJECT_EXPORT_RESULT_SET_TASK=拒绝导出结果集任务 +com.oceanbase.odc.AuditEventAction.REJECT_PERMISSION_APPLY_TASK=拒绝权限申请任务 +com.oceanbase.odc.AuditEventAction.REJECT_SHADOWTABLE_SYNC_TASK=拒绝影子表同步任务 +com.oceanbase.odc.AuditEventAction.REJECT_STRUCTURE_COMPARISON_TASK=拒绝结构比对任务 +com.oceanbase.odc.AuditEventAction.REJECT_PARTITION_PLAN_TASK=拒绝分区计划任务 +com.oceanbase.odc.AuditEventAction.REJECT_ALTER_SCHEDULE_TASK=拒绝修改调度任务 +com.oceanbase.odc.AuditEventAction.REJECT_ONLINE_SCHEMA_CHANGE_TASK=拒绝无锁结构变更任务 +com.oceanbase.odc.AuditEventAction.REJECT_APPLY_PROJECT_PERMISSION_TASK=拒绝申请项目权限任务 +com.oceanbase.odc.AuditEventAction.REJECT_APPLY_DATABASE_PERMISSION_TASK=拒绝申请数据库权限任务 +com.oceanbase.odc.AuditEventAction.REJECT_APPLY_TABLE_PERMISSION_TASK=拒绝申请数据表权限任务 +com.oceanbase.odc.AuditEventAction.CREATE_DATA_MASKING_RULE=创建脱敏规则 +com.oceanbase.odc.AuditEventAction.UPDATE_DATA_MASKING_RULE=更新脱敏规则 +com.oceanbase.odc.AuditEventAction.ENABLE_DATA_MASKING_RULE=启用脱敏规则 +com.oceanbase.odc.AuditEventAction.DISABLE_DATA_MASKING_RULE=禁用脱敏规则 +com.oceanbase.odc.AuditEventAction.CREATE_DATA_MASKING_POLICY=创建脱敏策略 +com.oceanbase.odc.AuditEventAction.UPDATE_DATA_MASKING_POLICY=更新脱敏策略 +com.oceanbase.odc.AuditEventAction.DELETE_DATA_MASKING_POLICY=删除脱敏策略 +com.oceanbase.odc.AuditEventAction.CREATE_INTEGRATION=创建集成 +com.oceanbase.odc.AuditEventAction.ENABLE_INTEGRATION=启用集成 +com.oceanbase.odc.AuditEventAction.DISABLE_INTEGRATION=禁用集成 +com.oceanbase.odc.AuditEventAction.DELETE_INTEGRATION=删除集成 +com.oceanbase.odc.AuditEventAction.ADD_DATABASE=增加数据库 +com.oceanbase.odc.AuditEventAction.DELETE_DATABASE=删除数据库 +com.oceanbase.odc.AuditEventAction.TRANSFER_DATABASE_TO_PROJECT=转移项目 +com.oceanbase.odc.AuditEventAction.CREATE_DATASOURCE=创建数据源 +com.oceanbase.odc.AuditEventAction.DELETE_DATASOURCE=删除数据源 +com.oceanbase.odc.AuditEventAction.UPDATE_DATASOURCE=更新数据源 +com.oceanbase.odc.AuditEventAction.CREATE_PROJECT=创建项目 +com.oceanbase.odc.AuditEventAction.UPDATE_SQL_SECURITY_RULE=修改 SQL 安全规则 +com.oceanbase.odc.AuditEventAction.GRANT_DATABASE_PERMISSION=新增库权限 +com.oceanbase.odc.AuditEventAction.REVOKE_DATABASE_PERMISSION=回收库权限 +com.oceanbase.odc.AuditEventAction.GRANT_TABLE_PERMISSION=新增表权限 +com.oceanbase.odc.AuditEventAction.REVOKE_TABLE_PERMISSION=回收表权限 -com.oceanbase.odc.AuditEventAction.CREATE_ENVIRONMENT=\u521B\u5EFA\u73AF\u5883 -com.oceanbase.odc.AuditEventAction.UPDATE_ENVIRONMENT=\u4FEE\u6539\u73AF\u5883 -com.oceanbase.odc.AuditEventAction.ENABLE_ENVIRONMENT=\u542F\u7528\u73AF\u5883 -com.oceanbase.odc.AuditEventAction.DISABLE_ENVIRONMENT=\u7981\u7528\u73AF\u5883 -com.oceanbase.odc.AuditEventAction.DELETE_ENVIRONMENT=\u5220\u9664\u73AF\u5883 +com.oceanbase.odc.AuditEventAction.CREATE_ENVIRONMENT=创建环境 +com.oceanbase.odc.AuditEventAction.UPDATE_ENVIRONMENT=修改环境 +com.oceanbase.odc.AuditEventAction.ENABLE_ENVIRONMENT=启用环境 +com.oceanbase.odc.AuditEventAction.DISABLE_ENVIRONMENT=禁用环境 +com.oceanbase.odc.AuditEventAction.DELETE_ENVIRONMENT=删除环境 -com.oceanbase.odc.AuditEventAction.CREATE_AUTOMATION_RULE=\u521B\u5EFA\u81EA\u52A8\u6388\u6743\u89C4\u5219 -com.oceanbase.odc.AuditEventAction.ENABLE_AUTOMATION_RULE=\u542F\u7528\u81EA\u52A8\u6388\u6743\u89C4\u5219 -com.oceanbase.odc.AuditEventAction.DISABLE_AUTOMATION_RULE=\u7981\u7528\u81EA\u52A8\u6388\u6743\u89C4\u5219 -com.oceanbase.odc.AuditEventAction.UPDATE_AUTOMATION_RULE=\u4FEE\u6539\u81EA\u52A8\u6388\u6743\u89C4\u5219 -com.oceanbase.odc.AuditEventAction.DELETE_AUTOMATION_RULE=\u5220\u9664\u81EA\u52A8\u6388\u6743\u89C4\u5219 +com.oceanbase.odc.AuditEventAction.CREATE_AUTOMATION_RULE=创建自动授权规则 +com.oceanbase.odc.AuditEventAction.ENABLE_AUTOMATION_RULE=启用自动授权规则 +com.oceanbase.odc.AuditEventAction.DISABLE_AUTOMATION_RULE=禁用自动授权规则 +com.oceanbase.odc.AuditEventAction.UPDATE_AUTOMATION_RULE=修改自动授权规则 +com.oceanbase.odc.AuditEventAction.DELETE_AUTOMATION_RULE=删除自动授权规则 -com.oceanbase.odc.AuditEventAction.CREATE_NOTIFICATION_CHANNEL=\u521B\u5EFA\u63A8\u9001\u901A\u9053 -com.oceanbase.odc.AuditEventAction.UPDATE_NOTIFICATION_CHANNEL=\u4FEE\u6539\u63A8\u9001\u901A\u9053 -com.oceanbase.odc.AuditEventAction.DELETE_NOTIFICATION_CHANNEL=\u5220\u9664\u63A8\u9001\u901A\u9053 -com.oceanbase.odc.AuditEventAction.BATCH_UPDATE_NOTIFICATION_POLICIES=\u66F4\u65B0\u63A8\u9001\u89C4\u5219 +com.oceanbase.odc.AuditEventAction.CREATE_NOTIFICATION_CHANNEL=创建推送通道 +com.oceanbase.odc.AuditEventAction.UPDATE_NOTIFICATION_CHANNEL=修改推送通道 +com.oceanbase.odc.AuditEventAction.DELETE_NOTIFICATION_CHANNEL=删除推送通道 +com.oceanbase.odc.AuditEventAction.BATCH_UPDATE_NOTIFICATION_POLICIES=更新推送规则 -com.oceanbase.odc.AuditEventAction.BATCH_CREATE_SENSITIVE_COLUMNS=\u6279\u91CF\u521B\u5EFA\u654F\u611F\u5217 -com.oceanbase.odc.AuditEventAction.BATCH_UPDATE_SENSITIVE_COLUMNS=\u6279\u91CF\u66F4\u65B0\u654F\u611F\u5217 -com.oceanbase.odc.AuditEventAction.BATCH_DELETE_SENSITIVE_COLUMNS=\u6279\u91CF\u5220\u9664\u654F\u611F\u5217 -com.oceanbase.odc.AuditEventAction.ENABLE_SENSITIVE_COLUMN=\u542F\u7528\u654F\u611F\u5217 -com.oceanbase.odc.AuditEventAction.DISABLE_SENSITIVE_COLUMN=\u7981\u7528\u654F\u611F\u5217 +com.oceanbase.odc.AuditEventAction.BATCH_CREATE_SENSITIVE_COLUMNS=批量创建敏感列 +com.oceanbase.odc.AuditEventAction.BATCH_UPDATE_SENSITIVE_COLUMNS=批量更新敏感列 +com.oceanbase.odc.AuditEventAction.BATCH_DELETE_SENSITIVE_COLUMNS=批量删除敏感列 +com.oceanbase.odc.AuditEventAction.ENABLE_SENSITIVE_COLUMN=启用敏感列 +com.oceanbase.odc.AuditEventAction.DISABLE_SENSITIVE_COLUMN=禁用敏感列 -com.oceanbase.odc.AuditEventAction.CREATE_DATABASE_CHANGE_CHANGING_ORDER_TEMPLATE=\u521B\u5EFA\u6570\u636E\u5E93\u53D8\u66F4\u987A\u5E8F\u6A21\u677F -com.oceanbase.odc.AuditEventAction.UPDATE_DATABASE_CHANGE_CHANGING_ORDER_TEMPLATE=\u66F4\u65B0\u6570\u636E\u5E93\u53D8\u66F4\u987A\u5E8F\u6A21\u677F -com.oceanbase.odc.AuditEventAction.DELETE_DATABASE_CHANGE_CHANGING_ORDER_TEMPLATE=\u5220\u9664\u6570\u636E\u5E93\u53D8\u66F4\u987A\u5E8F\u6A21\u677F +com.oceanbase.odc.AuditEventAction.CREATE_DATABASE_CHANGE_CHANGING_ORDER_TEMPLATE=创建数据库变更顺序模板 +com.oceanbase.odc.AuditEventAction.UPDATE_DATABASE_CHANGE_CHANGING_ORDER_TEMPLATE=更新数据库变更顺序模板 +com.oceanbase.odc.AuditEventAction.DELETE_DATABASE_CHANGE_CHANGING_ORDER_TEMPLATE=删除数据库变更顺序模板 # # AuditEventType # -com.oceanbase.odc.AuditEventType.PERSONAL_CONFIGURATION=\u4E2A\u4EBA\u914D\u7F6E -com.oceanbase.odc.AuditEventType.PASSWORD_MANAGEMENT=\u5BC6\u7801\u7BA1\u7406 -com.oceanbase.odc.AuditEventType.CONNECTION_MANAGEMENT=\u8FDE\u63A5\u7BA1\u7406 -com.oceanbase.odc.AuditEventType.SCRIPT_MANAGEMENT=\u811A\u672C\u7BA1\u7406 -com.oceanbase.odc.AuditEventType.DATABASE_OPERATION=\u6570\u636E\u5E93\u64CD\u4F5C -com.oceanbase.odc.AuditEventType.ORGANIZATION_CONFIGURATION=\u7EC4\u7EC7\u914D\u7F6E -com.oceanbase.odc.AuditEventType.RESOURCE_GROUP_MANAGEMENT=\u8D44\u6E90\u7EC4\u7BA1\u7406 -com.oceanbase.odc.AuditEventType.MEMBER_MANAGEMENT=\u4EBA\u5458\u7BA1\u7406 -com.oceanbase.odc.AuditEventType.AUDIT_EVENT=\u5BA1\u8BA1\u4E8B\u4EF6 -com.oceanbase.odc.AuditEventType.FLOW_CONFIG=\u6D41\u7A0B\u914D\u7F6E -com.oceanbase.odc.AuditEventType.MULTIPLE_ASYNC=\u591A\u5E93\u53D8\u66F4 -com.oceanbase.odc.AuditEventType.ASYNC=\u6570\u636E\u5E93\u53D8\u66F4 -com.oceanbase.odc.AuditEventType.MOCKDATA=\u6A21\u62DF\u6570\u636E -com.oceanbase.odc.AuditEventType.IMPORT=\u5BFC\u5165 -com.oceanbase.odc.AuditEventType.EXPORT=\u5BFC\u51FA -com.oceanbase.odc.AuditEventType.EXPORT_RESULT_SET=\u5BFC\u51FA\u7ED3\u679C\u96C6 -com.oceanbase.odc.AuditEventType.SHADOWTABLE_SYNC=\u5F71\u5B50\u8868\u540C\u6B65 -com.oceanbase.odc.AuditEventType.STRUCTURE_COMPARISON=\u7ED3\u6784\u6BD4\u5BF9 -com.oceanbase.odc.AuditEventType.PARTITION_PLAN=\u5206\u533A\u8BA1\u5212 -com.oceanbase.odc.AuditEventType.ALTER_SCHEDULE=\u4FEE\u6539\u8C03\u5EA6 -com.oceanbase.odc.AuditEventType.ONLINE_SCHEMA_CHANGE=\u65E0\u9501\u7ED3\u6784\u53D8\u66F4 -com.oceanbase.odc.AuditEventType.APPLY_PROJECT_PERMISSION=\u7533\u8BF7\u9879\u76EE\u6743\u9650 -com.oceanbase.odc.AuditEventType.APPLY_DATABASE_PERMISSION=\u7533\u8BF7\u6570\u636E\u5E93\u6743\u9650 -com.oceanbase.odc.AuditEventType.APPLY_TABLE_PERMISSION=\u7533\u8BF7\u6570\u636E\u8868\u6743\u9650 -com.oceanbase.odc.AuditEventType.DATA_MASKING_RULE=\u8131\u654F\u89C4\u5219 -com.oceanbase.odc.AuditEventType.DATA_MASKING_POLICY=\u8131\u654F\u7B56\u7565 -com.oceanbase.odc.AuditEventType.PERMISSION_APPLY=\u6743\u9650\u7533\u8BF7 -com.oceanbase.odc.AuditEventType.UNKNOWN_TASK_TYPE=\u672A\u77E5\u4EFB\u52A1\u7C7B\u578B -com.oceanbase.odc.AuditEventType.DATABASE_MANAGEMENT=\u6570\u636E\u5E93\u7BA1\u7406 -com.oceanbase.odc.AuditEventType.DATASOURCE_MANAGEMENT=\u6570\u636E\u6E90\u7BA1\u7406 -com.oceanbase.odc.AuditEventType.PROJECT_MANAGEMENT=\u9879\u76EE\u7BA1\u7406 -com.oceanbase.odc.AuditEventType.SQL_SECURITY_RULE_MANAGEMENT=SQL \u5B89\u5168\u89C4\u5219\u7BA1\u7406 -com.oceanbase.odc.AuditEventType.DATABASE_PERMISSION_MANAGEMENT=\u5E93\u6743\u9650\u7BA1\u7406 -com.oceanbase.odc.AuditEventType.TABLE_PERMISSION_MANAGEMENT=\u8868\u6743\u9650\u7BA1\u7406 -com.oceanbase.odc.AuditEventType.ENVIRONMENT_MANAGEMENT=\u73AF\u5883\u7BA1\u7406 -com.oceanbase.odc.AuditEventType.AUTOMATION_RULE_MANAGEMENT=\u81EA\u52A8\u6388\u6743\u89C4\u5219\u7BA1\u7406 -com.oceanbase.odc.AuditEventType.NOTIFICATION_MANAGEMENT=\u6D88\u606F\u63A8\u9001\u7BA1\u7406 -com.oceanbase.odc.AuditEventType.SENSITIVE_COLUMN_MANAGEMENT=\u654F\u611F\u5217\u7BA1\u7406 -com.oceanbase.odc.AuditEventType.DATABASE_CHANGE_CHANGING_ORDER_TEMPLATE_MANAGEMENT=\u6570\u636E\u5E93\u53D8\u66F4\u987A\u5E8F\u6A21\u677F\u7BA1\u7406 +com.oceanbase.odc.AuditEventType.PERSONAL_CONFIGURATION=个人配置 +com.oceanbase.odc.AuditEventType.PASSWORD_MANAGEMENT=密码管理 +com.oceanbase.odc.AuditEventType.CONNECTION_MANAGEMENT=连接管理 +com.oceanbase.odc.AuditEventType.SCRIPT_MANAGEMENT=脚本管理 +com.oceanbase.odc.AuditEventType.DATABASE_OPERATION=数据库操作 +com.oceanbase.odc.AuditEventType.ORGANIZATION_CONFIGURATION=组织配置 +com.oceanbase.odc.AuditEventType.RESOURCE_GROUP_MANAGEMENT=资源组管理 +com.oceanbase.odc.AuditEventType.MEMBER_MANAGEMENT=人员管理 +com.oceanbase.odc.AuditEventType.AUDIT_EVENT=审计事件 +com.oceanbase.odc.AuditEventType.FLOW_CONFIG=流程配置 +com.oceanbase.odc.AuditEventType.MULTIPLE_ASYNC=多库变更 +com.oceanbase.odc.AuditEventType.ASYNC=数据库变更 +com.oceanbase.odc.AuditEventType.MOCKDATA=模拟数据 +com.oceanbase.odc.AuditEventType.IMPORT=导入 +com.oceanbase.odc.AuditEventType.EXPORT=导出 +com.oceanbase.odc.AuditEventType.EXPORT_RESULT_SET=导出结果集 +com.oceanbase.odc.AuditEventType.SHADOWTABLE_SYNC=影子表同步 +com.oceanbase.odc.AuditEventType.STRUCTURE_COMPARISON=结构比对 +com.oceanbase.odc.AuditEventType.PARTITION_PLAN=分区计划 +com.oceanbase.odc.AuditEventType.ALTER_SCHEDULE=修改调度 +com.oceanbase.odc.AuditEventType.ONLINE_SCHEMA_CHANGE=无锁结构变更 +com.oceanbase.odc.AuditEventType.APPLY_PROJECT_PERMISSION=申请项目权限 +com.oceanbase.odc.AuditEventType.APPLY_DATABASE_PERMISSION=申请数据库权限 +com.oceanbase.odc.AuditEventType.APPLY_TABLE_PERMISSION=申请数据表权限 +com.oceanbase.odc.AuditEventType.DATA_MASKING_RULE=脱敏规则 +com.oceanbase.odc.AuditEventType.DATA_MASKING_POLICY=脱敏策略 +com.oceanbase.odc.AuditEventType.PERMISSION_APPLY=权限申请 +com.oceanbase.odc.AuditEventType.UNKNOWN_TASK_TYPE=未知任务类型 +com.oceanbase.odc.AuditEventType.DATABASE_MANAGEMENT=数据库管理 +com.oceanbase.odc.AuditEventType.DATASOURCE_MANAGEMENT=数据源管理 +com.oceanbase.odc.AuditEventType.PROJECT_MANAGEMENT=项目管理 +com.oceanbase.odc.AuditEventType.SQL_SECURITY_RULE_MANAGEMENT=SQL 安全规则管理 +com.oceanbase.odc.AuditEventType.DATABASE_PERMISSION_MANAGEMENT=库权限管理 +com.oceanbase.odc.AuditEventType.TABLE_PERMISSION_MANAGEMENT=表权限管理 +com.oceanbase.odc.AuditEventType.ENVIRONMENT_MANAGEMENT=环境管理 +com.oceanbase.odc.AuditEventType.AUTOMATION_RULE_MANAGEMENT=自动授权规则管理 +com.oceanbase.odc.AuditEventType.NOTIFICATION_MANAGEMENT=消息推送管理 +com.oceanbase.odc.AuditEventType.SENSITIVE_COLUMN_MANAGEMENT=敏感列管理 +com.oceanbase.odc.AuditEventType.DATABASE_CHANGE_CHANGING_ORDER_TEMPLATE_MANAGEMENT=数据库变更顺序模板管理 # # AuditEventResult result type # -com.oceanbase.odc.AuditEventResult.SUCCESS=\u6210\u529F -com.oceanbase.odc.AuditEventResult.FAILED=\u5931\u8D25 -com.oceanbase.odc.AuditEventResult.UNFINISHED=\u672A\u5B8C\u6210 +com.oceanbase.odc.AuditEventResult.SUCCESS=成功 +com.oceanbase.odc.AuditEventResult.FAILED=失败 +com.oceanbase.odc.AuditEventResult.UNFINISHED=未完成 # # DBObjectType # -com.oceanbase.odc.DBObjectType.SCHEMA=\u6A21\u5F0F -com.oceanbase.odc.DBObjectType.TABLE=\u8868 -com.oceanbase.odc.DBObjectType.COLUMN=\u5217 -com.oceanbase.odc.DBObjectType.INDEX=\u7D22\u5F15 -com.oceanbase.odc.DBObjectType.CONSTRAINT=\u7EA6\u675F -com.oceanbase.odc.DBObjectType.PARTITION=\u5206\u533A -com.oceanbase.odc.DBObjectType.SUBPARTITION=\u4E8C\u7EA7\u5206\u533A -com.oceanbase.odc.DBObjectType.VIEW=\u89C6\u56FE -com.oceanbase.odc.DBObjectType.TRIGGER=\u89E6\u53D1\u5668 -com.oceanbase.odc.DBObjectType.SEQUENCE=\u5E8F\u5217 -com.oceanbase.odc.DBObjectType.PROCEDURE=\u5B58\u50A8\u8FC7\u7A0B -com.oceanbase.odc.DBObjectType.FUNCTION=\u51FD\u6570 -com.oceanbase.odc.DBObjectType.PACKAGE=\u7A0B\u5E8F\u5305 -com.oceanbase.odc.DBObjectType.PACKAGE_BODY=\u7A0B\u5E8F\u5305\u5305\u4F53 -com.oceanbase.odc.DBObjectType.SYNONYM=\u540C\u4E49\u8BCD -com.oceanbase.odc.DBObjectType.TYPE=\u7C7B\u578B +com.oceanbase.odc.DBObjectType.SCHEMA=模式 +com.oceanbase.odc.DBObjectType.TABLE=表 +com.oceanbase.odc.DBObjectType.COLUMN=列 +com.oceanbase.odc.DBObjectType.INDEX=索引 +com.oceanbase.odc.DBObjectType.CONSTRAINT=约束 +com.oceanbase.odc.DBObjectType.PARTITION=分区 +com.oceanbase.odc.DBObjectType.SUBPARTITION=二级分区 +com.oceanbase.odc.DBObjectType.VIEW=视图 +com.oceanbase.odc.DBObjectType.TRIGGER=触发器 +com.oceanbase.odc.DBObjectType.SEQUENCE=序列 +com.oceanbase.odc.DBObjectType.PROCEDURE=存储过程 +com.oceanbase.odc.DBObjectType.FUNCTION=函数 +com.oceanbase.odc.DBObjectType.PACKAGE=程序包 +com.oceanbase.odc.DBObjectType.PACKAGE_BODY=程序包包体 +com.oceanbase.odc.DBObjectType.SYNONYM=同义词 +com.oceanbase.odc.DBObjectType.TYPE=类型 # # DBTableType # -com.oceanbase.odc.DBTableType.LOCAL_TEMPORARY=\u4E34\u65F6\u8868 -com.oceanbase.odc.DBTableType.SYSTEM_TABLE=\u7CFB\u7EDF\u8868 -com.oceanbase.odc.DBTableType.SYSTEM_VIEW=\u7CFB\u7EDF\u89C6\u56FE -com.oceanbase.odc.DBTableType.TABLE=\u8868 -com.oceanbase.odc.DBTableType.VIEW=\u89C6\u56FE +com.oceanbase.odc.DBTableType.LOCAL_TEMPORARY=临时表 +com.oceanbase.odc.DBTableType.SYSTEM_TABLE=系统表 +com.oceanbase.odc.DBTableType.SYSTEM_VIEW=系统视图 +com.oceanbase.odc.DBTableType.TABLE=表 +com.oceanbase.odc.DBTableType.VIEW=视图 com.oceanbase.odc.DBTableType.UNKNOWN=UNKNOWN # # Builtin Resource # -com.oceanbase.odc.builtin-resource.MaskRule.NAME.Name=\u59D3\u540D -com.oceanbase.odc.builtin-resource.MaskRule.ID.Name=\u8EAB\u4EFD\u8BC1\u4EF6\u53F7\u7801 -com.oceanbase.odc.builtin-resource.MaskRule.MOBILE_PHONE.Name=\u624B\u673A\u53F7 -com.oceanbase.odc.builtin-resource.MaskRule.FIXED_PHONE.Name=\u56FA\u5B9A\u7535\u8BDD -com.oceanbase.odc.builtin-resource.MaskRule.BANKCARD.Name=\u94F6\u884C\u5361\u53F7 -com.oceanbase.odc.builtin-resource.MaskRule.MAIL.Name=\u90AE\u7BB1 -com.oceanbase.odc.builtin-resource.MaskRule.DEFAULT.Name=\u7F3A\u7701\u89C4\u5219 -com.oceanbase.odc.builtin-resource.FlowConfig.DATABASE_CHANGE_ALL.Name=\u6570\u636E\u5E93\u53D8\u66F4/\u5168\u90E8\u53D8\u66F4 -com.oceanbase.odc.builtin-resource.FlowConfig.MOCK_DATA.Name=\u6A21\u62DF\u6570\u636E -com.oceanbase.odc.builtin-resource.FlowConfig.EXPORT.Name=\u5BFC\u51FA -com.oceanbase.odc.builtin-resource.FlowConfig.IMPORT.Name=\u5BFC\u5165 -com.oceanbase.odc.builtin-resource.FlowConfig.PERMISSION_APPLY.Name=\u6743\u9650\u7533\u8BF7 -com.oceanbase.odc.builtin-resource.FlowConfig.SHADOWTABLE_SYNC.Name=\u5F71\u5B50\u8868\u540C\u6B65 -com.oceanbase.odc.builtin-resource.FlowConfig.PARTITION_PLAN.Name=\u5206\u533A\u7BA1\u7406 -com.oceanbase.odc.builtin-resource.TriggerEvent.NEW_USER_FROM_OAUTH2.description=\u5F53\u65B0\u7528\u6237\u9996\u6B21\u4ECE OAuth2 \u767B\u5F55\u65F6\u89E6\u53D1 -com.oceanbase.odc.builtin-resource.TriggerEvent.USER_CREATED.description=\u5F53\u65B0\u7528\u6237\u9996\u6B21\u521B\u5EFA\u65F6\u89E6\u53D1 -com.oceanbase.odc.builtin-resource.TriggerEvent.USER_UPDATED.description=\u5F53\u7528\u6237\u66F4\u65B0\u65F6\u89E6\u53D1 -com.oceanbase.odc.builtin-resource.TriggerEvent.LOGIN_SUCCESS.description=\u7528\u6237\u767B\u5F55\u6210\u529F\u65F6\u89E6\u53D1 -com.oceanbase.odc.builtin-resource.iam.organization.team.display-name=\u56E2\u961F\u7A7A\u95F4 -com.oceanbase.odc.builtin-resource.iam.organization.team.description=\u652F\u6301\u591A\u9879\u76EE\u548C\u591A\u6210\u5458\uFF0C\u63D0\u4F9B\u534F\u540C\u7BA1\u63A7\u80FD\u529B\uFF0C\u9002\u5408\u56E2\u961F\u8FDB\u884C\u6570\u636E\u5E93\u5B89\u5168\u53D8\u66F4 -com.oceanbase.odc.builtin-resource.iam.organization.individual.display-name=\u4E2A\u4EBA\u7A7A\u95F4 -com.oceanbase.odc.builtin-resource.iam.organization.individual.description=\u9002\u5408\u4E2A\u4EBA\u5F00\u53D1\u8005\u8FDB\u884C\u6570\u636E\u5E93\u7075\u6D3B\u53D8\u66F4 -com.oceanbase.odc.builtin-resource.FlowConfig.ALTER_SCHEDULE.Name=\u8BA1\u5212\u53D8\u66F4 +com.oceanbase.odc.builtin-resource.MaskRule.NAME.Name=姓名 +com.oceanbase.odc.builtin-resource.MaskRule.ID.Name=身份证件号码 +com.oceanbase.odc.builtin-resource.MaskRule.MOBILE_PHONE.Name=手机号 +com.oceanbase.odc.builtin-resource.MaskRule.FIXED_PHONE.Name=固定电话 +com.oceanbase.odc.builtin-resource.MaskRule.BANKCARD.Name=银行卡号 +com.oceanbase.odc.builtin-resource.MaskRule.MAIL.Name=邮箱 +com.oceanbase.odc.builtin-resource.MaskRule.DEFAULT.Name=缺省规则 +com.oceanbase.odc.builtin-resource.FlowConfig.DATABASE_CHANGE_ALL.Name=数据库变更/全部变更 +com.oceanbase.odc.builtin-resource.FlowConfig.MOCK_DATA.Name=模拟数据 +com.oceanbase.odc.builtin-resource.FlowConfig.EXPORT.Name=导出 +com.oceanbase.odc.builtin-resource.FlowConfig.IMPORT.Name=导入 +com.oceanbase.odc.builtin-resource.FlowConfig.PERMISSION_APPLY.Name=权限申请 +com.oceanbase.odc.builtin-resource.FlowConfig.SHADOWTABLE_SYNC.Name=影子表同步 +com.oceanbase.odc.builtin-resource.FlowConfig.PARTITION_PLAN.Name=分区管理 +com.oceanbase.odc.builtin-resource.TriggerEvent.NEW_USER_FROM_OAUTH2.description=当新用户首次从 OAuth2 登录时触发 +com.oceanbase.odc.builtin-resource.TriggerEvent.USER_CREATED.description=当新用户首次创建时触发 +com.oceanbase.odc.builtin-resource.TriggerEvent.USER_UPDATED.description=当用户更新时触发 +com.oceanbase.odc.builtin-resource.TriggerEvent.LOGIN_SUCCESS.description=用户登录成功时触发 +com.oceanbase.odc.builtin-resource.iam.organization.team.display-name=团队空间 +com.oceanbase.odc.builtin-resource.iam.organization.team.description=支持多项目和多成员，提供协同管控能力，适合团队进行数据库安全变更 +com.oceanbase.odc.builtin-resource.iam.organization.individual.display-name=个人空间 +com.oceanbase.odc.builtin-resource.iam.organization.individual.description=适合个人开发者进行数据库灵活变更 +com.oceanbase.odc.builtin-resource.FlowConfig.ALTER_SCHEDULE.Name=计划变更 # below regulation module built-in -com.oceanbase.odc.builtin-resource.regulation.ruleset.default-default-ruleset.name=\u9ED8\u8BA4\u73AF\u5883 SQL \u5F00\u53D1\u89C4\u8303 -com.oceanbase.odc.builtin-resource.regulation.ruleset.default-dev-ruleset.name=\u9ED8\u8BA4\u5F00\u53D1\u73AF\u5883 SQL \u5F00\u53D1\u89C4\u8303 -com.oceanbase.odc.builtin-resource.regulation.ruleset.default-sit-ruleset.name=\u9ED8\u8BA4\u6D4B\u8BD5\u73AF\u5883 SQL \u5F00\u53D1\u89C4\u8303 -com.oceanbase.odc.builtin-resource.regulation.ruleset.default-prod-ruleset.name=\u9ED8\u8BA4\u751F\u4EA7\u73AF\u5883 SQL \u5F00\u53D1\u89C4\u8303 -com.oceanbase.odc.builtin-resource.regulation.ruleset.default-default-ruleset.description=\u9ED8\u8BA4\u73AF\u5883 SQL \u5F00\u53D1\u89C4\u8303 -com.oceanbase.odc.builtin-resource.regulation.ruleset.default-dev-ruleset.description=\u9ED8\u8BA4\u5F00\u53D1\u73AF\u5883 SQL \u5F00\u53D1\u89C4\u8303 -com.oceanbase.odc.builtin-resource.regulation.ruleset.default-sit-ruleset.description=\u9ED8\u8BA4\u6D4B\u8BD5\u73AF\u5883 SQL \u5F00\u53D1\u89C4\u8303 -com.oceanbase.odc.builtin-resource.regulation.ruleset.default-prod-ruleset.description=\u9ED8\u8BA4\u751F\u4EA7\u73AF\u5883 SQL \u5F00\u53D1\u89C4\u8303 -com.oceanbase.odc.builtin-resource.collaboration.environment.default.name=\u9ED8\u8BA4 -com.oceanbase.odc.builtin-resource.collaboration.environment.dev.name=\u5F00\u53D1 -com.oceanbase.odc.builtin-resource.collaboration.environment.sit.name=\u6D4B\u8BD5 -com.oceanbase.odc.builtin-resource.collaboration.environment.prod.name=\u751F\u4EA7 -com.oceanbase.odc.builtin-resource.collaboration.environment.default.description=\u9ED8\u8BA4\u73AF\u5883 -com.oceanbase.odc.builtin-resource.collaboration.environment.dev.description=\u5F00\u53D1\u73AF\u5883 -com.oceanbase.odc.builtin-resource.collaboration.environment.sit.description=\u6D4B\u8BD5\u73AF\u5883 -com.oceanbase.odc.builtin-resource.collaboration.environment.prod.description=\u751F\u4EA7\u73AF\u5883 +com.oceanbase.odc.builtin-resource.regulation.ruleset.default-default-ruleset.name=默认环境 SQL 开发规范 +com.oceanbase.odc.builtin-resource.regulation.ruleset.default-dev-ruleset.name=默认开发环境 SQL 开发规范 +com.oceanbase.odc.builtin-resource.regulation.ruleset.default-sit-ruleset.name=默认测试环境 SQL 开发规范 +com.oceanbase.odc.builtin-resource.regulation.ruleset.default-prod-ruleset.name=默认生产环境 SQL 开发规范 +com.oceanbase.odc.builtin-resource.regulation.ruleset.default-default-ruleset.description=默认环境 SQL 开发规范 +com.oceanbase.odc.builtin-resource.regulation.ruleset.default-dev-ruleset.description=默认开发环境 SQL 开发规范 +com.oceanbase.odc.builtin-resource.regulation.ruleset.default-sit-ruleset.description=默认测试环境 SQL 开发规范 +com.oceanbase.odc.builtin-resource.regulation.ruleset.default-prod-ruleset.description=默认生产环境 SQL 开发规范 +com.oceanbase.odc.builtin-resource.collaboration.environment.default.name=默认 +com.oceanbase.odc.builtin-resource.collaboration.environment.dev.name=开发 +com.oceanbase.odc.builtin-resource.collaboration.environment.sit.name=测试 +com.oceanbase.odc.builtin-resource.collaboration.environment.prod.name=生产 +com.oceanbase.odc.builtin-resource.collaboration.environment.default.description=默认环境 +com.oceanbase.odc.builtin-resource.collaboration.environment.dev.description=开发环境 +com.oceanbase.odc.builtin-resource.collaboration.environment.sit.description=测试环境 +com.oceanbase.odc.builtin-resource.collaboration.environment.prod.description=生产环境 # below built-in regulation sql development rules -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-edit-resultset.name=\u7981\u6B62\u7F16\u8F91\u7ED3\u679C\u96C6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-edit-resultset.message=\u7981\u6B62\u7F16\u8F91\u7ED3\u679C\u96C6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-edit-resultset.description=\u7981\u6B62\u7F16\u8F91\u7ED3\u679C\u96C6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-edit-resultset.metadata.name=\u7981\u6B62\u7F16\u8F91\u7ED3\u679C\u96C6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-edit-resultset.metadata.description=\u7981\u6B62\u7F16\u8F91\u7ED3\u679C\u96C6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-export-resultset.name=\u7981\u6B62\u5BFC\u51FA\u7ED3\u679C\u96C6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-export-resultset.message=\u7981\u6B62\u5BFC\u51FA\u7ED3\u679C\u96C6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-export-resultset.description=\u7981\u6B62\u5BFC\u51FA\u7ED3\u679C\u96C6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-export-resultset.metadata.name=\u7981\u6B62\u5BFC\u51FA\u7ED3\u679C\u96C6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-export-resultset.metadata.description=\u7981\u6B62\u5BFC\u51FA\u7ED3\u679C\u96C6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.name=SQL \u7A97\u53E3\u8FD4\u56DE\u7ED3\u679C\u96C6\u6700\u5927\u884C\u6570 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.message=SQL \u7A97\u53E3\u8FD4\u56DE\u7ED3\u679C\u96C6\u6700\u5927\u884C\u6570\uFF1A{0} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.description=SQL \u7A97\u53E3\u8FD4\u56DE\u7ED3\u679C\u96C6\u6700\u5927\u884C\u6570 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.metadata.name=SQL \u7A97\u53E3\u8FD4\u56DE\u7ED3\u679C\u96C6\u6700\u5927\u884C\u6570 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.metadata.description=SQL \u7A97\u53E3\u8FD4\u56DE\u7ED3\u679C\u96C6\u6700\u5927\u884C\u6570 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.name=SQL \u7A97\u53E3\u5141\u8BB8\u6267\u884C\u7684 SQL \u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.message=SQL \u7A97\u53E3\u5141\u8BB8\u6267\u884C\u7684 SQL \u7C7B\u578B\uFF1A{0} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.description=SQL \u7A97\u53E3\u5141\u8BB8\u6267\u884C\u7684 SQL \u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.metadata.name=SQL \u7A97\u53E3\u5141\u8BB8\u6267\u884C\u7684 SQL \u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.metadata.description=SQL \u7A97\u53E3\u5141\u8BB8\u6267\u884C\u7684 SQL \u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.name=SQL \u7A97\u53E3\u5141\u8BB8\u6267\u884C\u7684\u6700\u5927 SQL \u4E2A\u6570 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.message=SQL \u7A97\u53E3\u5141\u8BB8\u6267\u884C\u7684\u6700\u5927 SQL \u4E2A\u6570\uFF1A{0} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.description=SQL \u7A97\u53E3\u5141\u8BB8\u6267\u884C\u7684\u6700\u5927 SQL \u4E2A\u6570 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.metadata.name=SQL \u7A97\u53E3\u5141\u8BB8\u6267\u884C\u7684\u6700\u5927 SQL \u4E2A\u6570 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.metadata.description=SQL \u7A97\u53E3\u5141\u8BB8\u6267\u884C\u7684\u6700\u5927 SQL \u4E2A\u6570 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-debug-pl.name=SQL \u7A97\u53E3\u7981\u6B62 PL \u8C03\u8BD5 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-debug-pl.message=SQL \u7A97\u53E3\u7981\u6B62 PL \u8C03\u8BD5 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-debug-pl.description=PL \u8C03\u8BD5\u4F1A\u6709\u5DE5\u4F5C\u7EBF\u7A0B\u5361\u4F4F\u7684\u98CE\u9669\uFF0C\u6050\u5F71\u54CD\u751F\u4EA7\u7CFB\u7EDF\u7684\u7A33\u5B9A\uFF0C\u4E0D\u5EFA\u8BAE\u542F\u7528 PL \u8C03\u8BD5 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-debug-pl.metadata.name=SQL \u7A97\u53E3\u7981\u6B62 PL \u8C03\u8BD5 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-debug-pl.metadata.description=SQL \u7A97\u53E3\u7981\u6B62 PL \u8C03\u8BD5 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-create-pl.name=SQL \u7A97\u53E3\u7981\u6B62\u521B\u5EFA PL -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-create-pl.message=SQL \u7A97\u53E3\u7981\u6B62\u521B\u5EFA PL -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-create-pl.description=SQL \u7A97\u53E3\u7981\u6B62\u521B\u5EFA PL -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-create-pl.metadata.name=SQL \u7A97\u53E3\u7981\u6B62\u521B\u5EFA PL -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-create-pl.metadata.description=SQL \u7A97\u53E3\u7981\u6B62\u521B\u5EFA PL -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.external-sql-interceptor.name=\u5916\u90E8 SQL \u5BA1\u6838\u96C6\u6210 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.external-sql-interceptor.message=\u5916\u90E8 SQL \u5BA1\u6838\u96C6\u6210\u7CFB\u7EDF\u62E6\u622A\u4E86\u8BE5 SQL -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.external-sql-interceptor.description=\u5916\u90E8 SQL \u5BA1\u6838\u96C6\u6210 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.external-sql-interceptor.metadata.name=\u5916\u90E8 SQL \u5BA1\u6838\u96C6\u6210 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.external-sql-interceptor.metadata.description=\u5916\u90E8 SQL \u5BA1\u6838\u96C6\u6210 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-edit-resultset.name=禁止编辑结果集 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-edit-resultset.message=禁止编辑结果集 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-edit-resultset.description=禁止编辑结果集 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-edit-resultset.metadata.name=禁止编辑结果集 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-edit-resultset.metadata.description=禁止编辑结果集 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-export-resultset.name=禁止导出结果集 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-export-resultset.message=禁止导出结果集 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-export-resultset.description=禁止导出结果集 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-export-resultset.metadata.name=禁止导出结果集 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-export-resultset.metadata.description=禁止导出结果集 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.name=SQL 窗口返回结果集最大行数 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.message=SQL 窗口返回结果集最大行数：{0} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.description=SQL 窗口返回结果集最大行数 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.metadata.name=SQL 窗口返回结果集最大行数 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.metadata.description=SQL 窗口返回结果集最大行数 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.name=SQL 窗口允许执行的 SQL 类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.message=SQL 窗口允许执行的 SQL 类型：{0} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.description=SQL 窗口允许执行的 SQL 类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.metadata.name=SQL 窗口允许执行的 SQL 类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.metadata.description=SQL 窗口允许执行的 SQL 类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.name=SQL 窗口允许执行的最大 SQL 个数 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.message=SQL 窗口允许执行的最大 SQL 个数：{0} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.description=SQL 窗口允许执行的最大 SQL 个数 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.metadata.name=SQL 窗口允许执行的最大 SQL 个数 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.metadata.description=SQL 窗口允许执行的最大 SQL 个数 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-debug-pl.name=SQL 窗口禁止 PL 调试 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-debug-pl.message=SQL 窗口禁止 PL 调试 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-debug-pl.description=PL 调试会有工作线程卡住的风险，恐影响生产系统的稳定，不建议启用 PL 调试 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-debug-pl.metadata.name=SQL 窗口禁止 PL 调试 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-debug-pl.metadata.description=SQL 窗口禁止 PL 调试 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-create-pl.name=SQL 窗口禁止创建 PL +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-create-pl.message=SQL 窗口禁止创建 PL +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-create-pl.description=SQL 窗口禁止创建 PL +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-create-pl.metadata.name=SQL 窗口禁止创建 PL +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-create-pl.metadata.description=SQL 窗口禁止创建 PL +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.external-sql-interceptor.name=外部 SQL 审核集成 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.external-sql-interceptor.message=外部 SQL 审核集成系统拦截了该 SQL +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.external-sql-interceptor.description=外部 SQL 审核集成 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.external-sql-interceptor.metadata.name=外部 SQL 审核集成 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.external-sql-interceptor.metadata.description=外部 SQL 审核集成 # # sql-check # -com.oceanbase.odc.CheckViolation.LocalizedMessage=\u4F4D\u4E8E\u7B2C {0} \u884C\uFF0C\u7B2C {1} \u5217\u7684\u8BED\u53E5\u53EF\u80FD\u5B58\u5728\u95EE\u9898\uFF0C\u8BE6\u60C5\uFF1A{2} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.syntax-error.message=\u8BED\u53E5\u5B58\u5728\u8BED\u6CD5\u9519\u8BEF\uFF0C\u8BE6\u60C5\uFF1A{0} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.syntax-error.name=\u8BED\u6CD5\u9519\u8BEF -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.syntax-error.description=\u5F85\u68C0\u6D4B\u7684\u8BED\u53E5\u5B58\u5728\u8BED\u6CD5\u9519\u8BEF -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-calculation.message=\u5217\u6761\u4EF6\u4E0A\u5B58\u5728\u8BA1\u7B97\uFF0C\u53EF\u80FD\u5BFC\u81F4\u7D22\u5F15\u5931\u6548 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-calculation.name=\u5217\u6761\u4EF6\u4E0A\u5B58\u5728\u8BA1\u7B97 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-calculation.description=\u5217\u6761\u4EF6\u4E0A\u5B58\u5728\u8BA1\u7B97\uFF0C\u53EF\u80FD\u5BFC\u81F4\u7D22\u5F15\u5931\u6548 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.message=LIKE \u8FD0\u7B97\u5B58\u5728\u5DE6\u6A21\u7CCA\u5339\u914D\uFF0C\u53EF\u80FD\u5BFC\u81F4\u7D22\u5F15\u5931\u6548 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.name=LIKE \u8FD0\u7B97\u5B58\u5728\u5DE6\u6A21\u7CCA\u5339\u914D -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.description=LIKE \u8FD0\u7B97\u5B58\u5728\u5DE6\u6A21\u7CCA\u5339\u914D\uFF0C\u53EF\u80FD\u5BFC\u81F4\u7D22\u5F15\u5931\u6548 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-implicit-conversion.message=\u5217\u6761\u4EF6\u4E0A\u5B58\u5728\u9690\u5F0F\u7C7B\u578B\u8F6C\u6362\uFF0C\u53EF\u80FD\u5BFC\u81F4\u7D22\u5F15\u5931\u6548 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-implicit-conversion.name=\u9690\u5F0F\u7C7B\u578B\u8F6C\u6362 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-implicit-conversion.description=\u5217\u6761\u4EF6\u4E0A\u5B58\u5728\u9690\u5F0F\u7C7B\u578B\u8F6C\u6362\uFF0C\u53EF\u80FD\u5BFC\u81F4\u7D22\u5F15\u5931\u6548 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.message=\u8FC7\u591A\u7684 IN \u503C\u5339\u914D\u53EF\u80FD\u5BFC\u81F4\u67E5\u8BE2\u6548\u7387\u964D\u4F4E\uFF0C\u5EFA\u8BAE\u4E0D\u8D85\u8FC7 {0} \u4E2A\uFF0C\u5B9E\u9645\u5B58\u5728 {1} \u4E2A -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.name=\u8FC7\u591A\u7684 IN \u503C\u5339\u914D -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.description=\u8FC7\u591A\u7684 IN \u503C\u5339\u914D\u53EF\u80FD\u5BFC\u81F4\u67E5\u8BE2\u6548\u7387\u964D\u4F4E -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count=IN \u6761\u4EF6\u7684\u6570\u91CF\u6700\u5927\u503C -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count.description=\u6761\u4EF6\u8C13\u8BCD\u4E2D IN \u8FD0\u7B97\u4E2D\u8868\u8FBE\u5F0F\u6570\u91CF\u7684\u6700\u5927\u503C -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.message=\u8FC7\u591A\u7684\u8868\u5BF9\u8C61 JOIN\uFF0C\u53EF\u80FD\u5BFC\u81F4\u6267\u884C\u8BA1\u5212\u65E0\u6CD5\u6700\u4F18\uFF0C\u63A8\u8350\u4E0D\u8D85\u8FC7 {0} \u4E2A\uFF0C\u5B9E\u9645 {1} \u4E2A\u8868\u8054\u7ED3 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.name=\u8FC7\u591A\u7684\u8868\u5BF9\u8C61 JOIN -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.description=\u8FC7\u591A\u7684\u8868\u5BF9\u8C61 JOIN\uFF0C\u53EF\u80FD\u5BFC\u81F4\u6267\u884C\u8BA1\u5212\u65E0\u6CD5\u6700\u4F18 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count=\u6700\u5927\u8868\u8054\u7ED3\u6570\u91CF -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count.description=\u6700\u5927\u8868\u8054\u7ED3\u6570\u91CF -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.message=NOT IN \u6761\u4EF6\u4E2D\u9700\u8981 NOT NULL \u6807\u8BC6\uFF0C\u907F\u514D\u9677\u5165\u5D4C\u5957\u5FAA\u73AF -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.name=NOT IN \u6761\u4EF6\u4E2D\u9700\u8981 NOT NULL \u6807\u8BC6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.description=NOT IN \u6761\u4EF6\u4E2D\u9700\u8981 NOT NULL \u6807\u8BC6\uFF0C\u907F\u514D\u9677\u5165\u5D4C\u5957\u5FAA\u73AF -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-valid-where-clause.message=UPDATE/DELETE \u8BED\u53E5\u4E2D\u7684 WHERE \u6761\u4EF6\u6052\u4E3A\u771F/\u5047\u9020\u6210\u6F5C\u5728\u98CE\u9669 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-valid-where-clause.name=UPDATE/DELETE \u8BED\u53E5\u4E2D\u6CA1\u6709\u6709\u6548\u7684 WHERE \u6761\u4EF6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-valid-where-clause.description=UPDATE/DELETE \u8BED\u53E5\u4E2D\u7684 WHERE \u6761\u4EF6\u6052\u4E3A\u771F/\u5047\u9020\u6210\u6F5C\u5728\u98CE\u9669 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-where-clause-exists.message=UPDATE/DELETE \u8BED\u53E5\u4E2D\u6CA1\u6709 WHERE \u6761\u4EF6\u9020\u6210\u6F5C\u5728\u98CE\u9669 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-where-clause-exists.name=UPDATE/DELETE \u8BED\u53E5\u4E2D\u6CA1\u6709 WHERE \u6761\u4EF6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-where-clause-exists.description=UPDATE/DELETE \u8BED\u53E5\u4E2D\u6CA1\u6709 WHERE \u6761\u4EF6\u9020\u6210\u6F5C\u5728\u98CE\u9669 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.message=INSERT/REPLACE \u8BED\u53E5\u4E2D\u6CA1\u6709\u5177\u4F53\u7684\u5217\uFF0C\u53EF\u80FD\u5BFC\u81F4\u9690\u5F0F\u8F6C\u6362\u9519\u8BEF -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.name=INSERT/REPLACE \u8BED\u53E5\u672A\u6307\u5B9A\u5217\u540D -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.description=INSERT/REPLACE \u8BED\u53E5\u4E2D\u6CA1\u6709\u5177\u4F53\u7684\u5217\uFF0C\u53EF\u80FD\u5BFC\u81F4\u9690\u5F0F\u8F6C\u6362\u9519\u8BEF -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-key-exists.message=\u5BF9\u5E26\u7D22\u5F15\u7684\u8868\u5BF9\u8C61 {0} \u64CD\u4F5C\u6CA1\u6709\u4F7F\u7528\u7D22\u5F15\uFF0C\u53EF\u80FD\u5BFC\u81F4\u5168\u8868\u626B\u63CF\uFF0C\u9020\u6210\u6027\u80FD\u4E0B\u964D -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-key-exists.name=\u6CA1\u6709\u4F7F\u7528\u7D22\u5F15 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-key-exists.description=\u5BF9\u5E26\u7D22\u5F15\u7684\u8868\u5BF9\u8C61\u64CD\u4F5C\u6CA1\u6709\u4F7F\u7528\u7D22\u5F15\uFF0C\u53EF\u80FD\u5BFC\u81F4\u5168\u8868\u626B\u63CF\uFF0C\u9020\u6210\u6027\u80FD\u4E0B\u964D -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-partition-key-exists.message=\u5BF9\u5206\u533A\u8868 {0} \u64CD\u4F5C\u6CA1\u6709\u4F7F\u7528\u5206\u533A\u952E\uFF0C\u5BFC\u81F4\u65E0\u6CD5\u8FDB\u884C\u5206\u533A\u88C1\u526A -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-partition-key-exists.name=\u6CA1\u6709\u4F7F\u7528\u5206\u533A\u952E -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-partition-key-exists.description=\u5BF9\u5206\u533A\u8868\u64CD\u4F5C\u6CA1\u6709\u4F7F\u7528\u5206\u533A\u952E\uFF0C\u5BFC\u81F4\u65E0\u6CD5\u8FDB\u884C\u5206\u533A\u88C1\u526A -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.message=\u8868\u5BF9\u8C61 {0} \u4E0A\u5B58\u5728\u8FC7\u591A\u7D22\u5F15\uFF0C\u53EF\u80FD\u5BFC\u81F4\u6027\u80FD\u4E0B\u964D\uFF0C\u63A8\u8350\u4E0D\u8981\u8D85\u8FC7 {1} \u4E2A -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.name=\u5355\u8868\u5305\u542B\u4E86\u8FC7\u591A\u7684\u7D22\u5F15 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.description=\u5355\u8868\u5305\u542B\u4E86\u8FC7\u591A\u7684\u7D22\u5F15\uFF0C\u53EF\u80FD\u5BFC\u81F4\u6027\u80FD\u4E0B\u964D -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count=\u6700\u5927\u7D22\u5F15\u6570\u91CF -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count.description=\u8868\u5B9A\u4E49\u4E2D\u80FD\u591F\u51FA\u73B0\u7684\u7D22\u5F15\u6570\u91CF\u6700\u5927\u503C -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prefer-local-index.message=\u63A8\u8350\u4F7F\u7528\u5C40\u90E8\u7D22\u5F15 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prefer-local-index.name=\u63A8\u8350\u4F7F\u7528\u5C40\u90E8\u7D22\u5F15 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prefer-local-index.description=\u63A8\u8350\u4F7F\u7528\u5C40\u90E8\u7D22\u5F15 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.message=\u8868\u5BF9\u8C61 {0} \u5B58\u5728\u8FC7\u591A\u7684\u5217\uFF0C\u63A8\u8350\u4E0D\u8D85\u8FC7 {1} \u4E2A\uFF0C\u5B9E\u9645 {2} \u4E2A -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.name=\u5355\u8868\u5305\u542B\u4E86\u8FC7\u591A\u7684\u5217 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.description=\u5355\u8868\u5305\u542B\u4E86\u8FC7\u591A\u7684\u5217 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count=\u6700\u5927\u5217\u5B9A\u4E49\u6570\u91CF -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count.description=\u8868\u5B9A\u4E49\u4E2D\u80FD\u591F\u51FA\u73B0\u7684\u5217\u6570\u91CF\u7684\u6700\u5927\u503C -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.message=\u8868\u5B9A\u4E49\u4E2D CHAR \u7C7B\u578B\u7684\u957F\u5EA6\u8FC7\u957F\uFF0C\u5EFA\u8BAE\u4E0D\u8981\u8D85\u8FC7 {0} \uFF0C\u5B9E\u9645\u503C\u4E3A {1} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.name=\u9650\u5236 CHAR \u7C7B\u578B\u5B57\u6BB5\u7684\u957F\u5EA6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.description=CHAR \u7C7B\u578B\u5B57\u6BB5\u7684\u957F\u5EA6\u4E0D\u8981\u592A\u957F -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length=CHAR \u6700\u5927\u5141\u8BB8\u957F\u5EA6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length.description=CHAR \u6700\u5927\u5141\u8BB8\u957F\u5EA6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.message=\u552F\u4E00\u7D22\u5F15\u7684\u547D\u540D {0} \u683C\u5F0F\u4E0D\u7B26\u5408\u89C4\u8303\uFF0C\u547D\u540D\u6A21\u5F0F\uFF1A{1} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name=\u9650\u5236\u552F\u4E00\u7D22\u5F15\u540D\u683C\u5F0F -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.description=\u552F\u4E00\u7D22\u5F15\u7684\u89C4\u8303\u547D\u540D\u6709\u52A9\u4E8E\u63D0\u9AD8\u6570\u636E\u5E93\u5F00\u53D1\u6548\u7387\uFF0C\u9ED8\u8BA4\uFF1Auk_${table-name}_${column-name-1}_${column-name-2}_... -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern=\u552F\u4E00\u7D22\u5F15\u540D\u79F0\u6B63\u5219\u8868\u8FBE\u5F0F -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern.description=\u552F\u4E00\u7D22\u5F15\u540D\u79F0\u6B63\u5219\u8868\u8FBE\u5F0F -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.foreign-constraint-exists.message=\u8868\u4E0D\u80FD\u4F7F\u7528\u5916\u952E -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.foreign-constraint-exists.name=\u8868\u4E0D\u80FD\u4F7F\u7528\u5916\u952E -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.foreign-constraint-exists.description=\u8868\u4E0D\u80FD\u4F7F\u7528\u5916\u952E -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.message=\u8868\u8981\u6709\u4E3B\u952E -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.name=\u8868\u8981\u6709\u4E3B\u952E -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.description=\u8868\u8981\u6709\u4E3B\u952E -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.message=\u8868\u5BF9\u8C61 {0} \u5FC5\u987B\u542B\u6709\u6CE8\u91CA -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.name=\u8868\u5FC5\u987B\u6CE8\u91CA -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.description=\u8868\u5BF9\u8C61\u5B58\u5728\u6CE8\u91CA\u53EF\u6709\u52A9\u4E8E\u5FEB\u901F\u4E86\u89E3\u8868\u7684\u4E1A\u52A1\u80CC\u666F -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.message=\u8868\u5BF9\u8C61\u4E0D\u80FD\u4EE5 {0} \u547D\u540D\uFF0C\u8BE5\u540D\u5B57\u5904\u4E8E\u9ED1\u540D\u5355\u4E2D\uFF0C\u9ED1\u540D\u5355\uFF1A{1} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.name=\u8868\u540D\u4E0D\u80FD\u5904\u4E8E\u9ED1\u540D\u5355 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.description=\u5EFA\u8868\u65F6\u7684\u8868\u540D\uFF0C\u4FEE\u6539\u8868\u540D\u65F6\u5747\u4E0D\u80FD\u4F7F\u7528\u5904\u4E8E\u9ED1\u540D\u5355\u4E2D\u7684\u8868\u540D -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list=\u9ED1\u540D\u5355 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list.description=\u8868\u540D\u9ED1\u540D\u5355 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.message=\u8868\u5BF9\u8C61\u4F7F\u7528\u7684\u5B57\u7B26\u96C6 {0} \u4E0D\u5728\u5141\u8BB8\u8303\u56F4\u5185\uFF0C\u5141\u8BB8\u4F7F\u7528\u7684\u5B57\u7B26\u96C6\u5305\u62EC\uFF1A{1} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.name=\u9650\u5236\u8868\u5B57\u7B26\u96C6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.description=\u8868\u5BF9\u8C61\u4F7F\u7528\u7684\u5B57\u7B26\u96C6\u53EA\u80FD\u4ECE\u767D\u540D\u5355\u4E2D\u7ED9\u51FA\u7684\u9009\u9879\u4E2D\u9009\u62E9 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.allowed-charsets=\u5141\u8BB8\u7684\u5B57\u7B26\u96C6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.allowed-charsets.description=\u5141\u8BB8\u7684\u5B57\u7B26\u96C6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.message=\u8868\u5BF9\u8C61\u4F7F\u7528\u7684\u6392\u5E8F\u89C4\u5219 {0} \u4E0D\u5728\u5141\u8BB8\u7684\u8303\u56F4\u5185\u3002\u5141\u8BB8\u7684\u5F52\u7C7B\u5305\u62EC\uFF1A{1} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.name=\u9650\u5236\u8868\u5BF9\u8C61\u7684\u6392\u5E8F\u89C4\u5219 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.description=\u8868\u5BF9\u8C61\u4F7F\u7528\u7684\u6392\u5E8F\u89C4\u5219\u53EA\u80FD\u4ECE\u767D\u540D\u5355\u4E2D\u7ED9\u51FA\u7684\u9009\u9879\u4E2D\u9009\u62E9 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.allowed-collations=\u5141\u8BB8\u7684\u6392\u5E8F\u89C4\u5219 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.allowed-collations.description=\u5141\u8BB8\u7684\u6392\u5E8F\u89C4\u5219 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.message=\u7D22\u5F15\u4E2D\u5B58\u5728 {0} \u6B21\u5217\u5F15\u7528\uFF0C\u6700\u591A\u4E0D\u5E94\u8D85\u8FC7 {1} \u6B21\u5217\u5F15\u7528 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.name=\u9650\u5236\u5355\u4E2A\u7D22\u5F15\u5305\u542B\u5217\u7684\u4E2A\u6570 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.description=\u5355\u4E2A\u7D22\u5F15\u5B9A\u4E49\u4E2D\u7684\u5217\u5F15\u7528\u6570\u76EE\u4E0D\u5E94\u8FC7\u591A -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count=\u6700\u5927\u5217\u5F15\u7528\u6570\u76EE -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count.description=\u6700\u5927\u5217\u5F15\u7528\u6570\u76EE -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.message=\u4E3B\u952E\u7EA6\u675F\u6D89\u53CA\u5217\u7684\u7C7B\u578B {0} \u4E0D\u5728\u5141\u8BB8\u8303\u56F4\u5185\uFF0C\u5141\u8BB8\u7684\u7C7B\u578B\u5305\u62EC\uFF1A{1} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.name=\u9650\u5236\u4E3B\u952E\u6570\u636E\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.description=\u8868\u7684\u4E3B\u952E\u6D89\u53CA\u7684\u5217\u5FC5\u987B\u662F\u89C4\u5219\u4E2D\u5B9A\u4E49\u7684\u6570\u636E\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes=\u5141\u8BB8\u4F5C\u4E3A\u4E3B\u952E\u7684\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes.description=\u5141\u8BB8\u4F5C\u4E3A\u4E3B\u952E\u7684\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.message=\u4E3B\u952E\u4E2D\u5B58\u5728 {0} \u6B21\u5217\u5F15\u7528\uFF0C\u6700\u591A\u4E0D\u5E94\u8D85\u8FC7 {1} \u6B21\u5217\u5F15\u7528 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.name=\u9650\u5236\u5355\u4E2A\u4E3B\u952E\u5305\u542B\u5217\u7684\u4E2A\u6570 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.description=\u5355\u4E2A\u4E3B\u952E\u5B9A\u4E49\u4E2D\u7684\u5217\u5F15\u7528\u6570\u76EE\u4E0D\u5E94\u8FC7\u591A -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count=\u6700\u5927\u5217\u5F15\u7528\u6570\u76EE -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count.description=\u6700\u5927\u5217\u5F15\u7528\u6570\u76EE -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-auto-increment.message=\u9650\u5236\u4E3B\u952E\u5217\u5FC5\u987B\u81EA\u589E -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-auto-increment.name=\u9650\u5236\u4E3B\u952E\u5217\u5FC5\u987B\u81EA\u589E -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-auto-increment.description=\u9650\u5236\u4E3B\u952E\u5217\u5FC5\u987B\u81EA\u589E -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.message=\u7D22\u5F15\u7684\u547D\u540D {0} \u683C\u5F0F\u4E0D\u7B26\u5408\u89C4\u8303\uFF0C\u547D\u540D\u6A21\u5F0F\uFF1A{1} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name=\u9650\u5236\u7D22\u5F15\u540D\u683C\u5F0F -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.description=\u7D22\u5F15\u7684\u89C4\u8303\u547D\u540D\u6709\u52A9\u4E8E\u63D0\u9AD8\u6570\u636E\u5E93\u5F00\u53D1\u6548\u7387\uFF0C\u9ED8\u8BA4\uFF1Aidx_${table-name}_${column-name-1}_${column-name-2}_... -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern=\u7D22\u5F15\u540D\u79F0\u6B63\u5219\u8868\u8FBE\u5F0F -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern.description=\u7D22\u5F15\u540D\u79F0\u6B63\u5219\u8868\u8FBE\u5F0F -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-name-exists.message=\u7D22\u5F15\u6216\u7EA6\u675F\u7684\u5B9A\u4E49\u6CA1\u6709\u547D\u540D -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-name-exists.name=\u7D22\u5F15\u6216\u7EA6\u675F\u9700\u8981\u8BBE\u7F6E\u540D\u79F0 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-name-exists.description=\u7D22\u5F15\u6216\u7EA6\u675F\u9700\u8981\u663E\u5F0F\u5B9A\u4E49\u540D\u79F0\uFF0C\u5426\u5219\u6570\u636E\u5E93\u4F1A\u81EA\u52A8\u547D\u540D -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.message=\u6570\u503C\u7C7B\u578B\u4F7F\u7528 ZEROFILL \u5C5E\u6027 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.name=\u6570\u503C\u7C7B\u578B\u4E0D\u80FD\u4F7F\u7528 ZEROFILL \u5C5E\u6027 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.description=\u6570\u503C\u7C7B\u578B\u4E0D\u80FD\u4F7F\u7528 ZEROFILL \u5C5E\u6027 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-charset-exists.message=\u4E0D\u80FD\u5BF9\u5217\u8BBE\u7F6E\u5B57\u7B26\u96C6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-charset-exists.name=\u4E0D\u80FD\u5BF9\u5217\u8BBE\u7F6E\u5B57\u7B26\u96C6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-charset-exists.description=\u4E0D\u80FD\u5BF9\u5217\u8BBE\u7F6E\u5B57\u7B26\u96C6 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.message=\u4E0D\u80FD\u5BF9\u5217\u8BBE\u7F6E\u6392\u5E8F\u89C4\u5219 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.name=\u4E0D\u80FD\u5BF9\u5217\u8BBE\u7F6E\u6392\u5E8F\u89C4\u5219 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.description=\u4E0D\u80FD\u5BF9\u5217\u8BBE\u7F6E\u6392\u5E8F\u89C4\u5219 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.message=\u7C7B\u578B\u4E3A {0} \u7684\u5217\u4E0D\u5141\u8BB8\u4E3A\u7A7A\uFF0C\u5141\u8BB8\u4E3A\u7A7A\u7684\u5217\u7684\u6570\u636E\u7C7B\u578B\uFF1A{1} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.name=\u9650\u5236\u5217\u4E0D\u53EF\u7A7A\uFF08NOT NULL) -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.description=\u9650\u5236\u5217\u4E0D\u53EF\u7A7A -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list=\u5141\u8BB8\u4E3A\u7A7A\u7684\u5217\u6570\u636E\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list.description=\u5141\u8BB8\u4E3A\u7A7A\u7684\u5217\u6570\u636E\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.message=\u6570\u636E\u7C7B\u578B\u4E3A {0} \u7684\u5217\u4E0D\u80FD\u6CA1\u6709\u9ED8\u8BA4\u503C\uFF0C\u5141\u8BB8\u6CA1\u6709\u9ED8\u8BA4\u503C\u7684\u5217\u7C7B\u578B\uFF1A{1} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.name=\u5217\u8981\u6709\u9ED8\u8BA4\u503C -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.description=\u5217\u8981\u6709\u9ED8\u8BA4\u503C -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list=\u5141\u8BB8\u6CA1\u6709\u9ED8\u8BA4\u503C\u7684\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list.description=\u5141\u8BB8\u6CA1\u6709\u9ED8\u8BA4\u503C\u7684\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-column-comment-exists.message=\u5217 {0} \u6CA1\u6709\u6CE8\u91CA -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-column-comment-exists.name=\u5217\u8981\u6709\u6CE8\u91CA -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-column-comment-exists.description=\u8868\u5B9A\u4E49\u4E2D\u5404\u4E2A\u5217\u9700\u8981\u6709\u6CE8\u91CA -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.message=\u5217\u4E0D\u80FD\u4EE5 {0} \u547D\u540D\uFF0C\u8BE5\u540D\u5B57\u5904\u4E8E\u9ED1\u540D\u5355\u4E2D\uFF0C\u9ED1\u540D\u5355\uFF1A{1} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.name=\u5217\u540D\u4E0D\u80FD\u5904\u4E8E\u9ED1\u540D\u5355 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.description=\u5EFA\u8868\u65F6\u7684\u5217\u540D\uFF0C\u4FEE\u6539\u5217\u540D\u65F6\u5747\u4E0D\u80FD\u4F7F\u7528\u5904\u4E8E\u9ED1\u540D\u5355\u4E2D\u7684\u5217\u540D -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list=\u9ED1\u540D\u5355 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list.description=\u5217\u540D\u9ED1\u540D\u5355 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.message=\u5217\u540D {0} \u7684\u5927\u5C0F\u5199\u8BBE\u5B9A\u4E0D\u7B26\u5408\u89C4\u8303 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.name=\u9650\u5236\u5217\u540D\u5927\u5C0F\u5199 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.description=\u9650\u5236\u5217\u540D\u5927\u5C0F\u5199 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-uppercase=\u662F\u5426\u9650\u5236\u5927\u5199 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-uppercase.description=\u662F\u5426\u9650\u5236\u5927\u5199 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-lowercase=\u662F\u5426\u9650\u5236\u5C0F\u5199 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-lowercase.description=\u662F\u5426\u9650\u5236\u5C0F\u5199 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.message=\u8868\u540D {0} \u7684\u5927\u5C0F\u5199\u8BBE\u5B9A\u4E0D\u7B26\u5408\u89C4\u8303 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.name=\u9650\u5236\u8868\u540D\u5927\u5C0F\u5199 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.description=\u9650\u5236\u8868\u540D\u5927\u5C0F\u5199 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-uppercase=\u662F\u5426\u9650\u5236\u5927\u5199 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-uppercase.description=\u662F\u5426\u9650\u5236\u5927\u5199 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-lowercase=\u662F\u5426\u9650\u5236\u5C0F\u5199 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-lowercase.description=\u662F\u5426\u9650\u5236\u5C0F\u5199 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.message=\u9650\u5236\u8868\u7684\u5217\u81EA\u589E\u521D\u59CB\u503C\u4E3A {0}\uFF0C\u5B9E\u9645\u4E3A {1} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.name=\u9650\u5236\u5EFA\u8868\u81EA\u589E\u521D\u59CB\u503C -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.description=\u9650\u5236\u5EFA\u8868\u81EA\u589E\u521D\u59CB\u503C -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.init-value=\u8868\u81EA\u589E\u521D\u59CB\u503C -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.init-value.description=\u8868\u81EA\u589E\u521D\u59CB\u503C -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.select-star-exists.message=SELECT \u8BED\u53E5\u4E2D\u4E0D\u5EFA\u8BAE\u4F7F\u7528 * \u6295\u5F71\u5168\u90E8\u5217 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.select-star-exists.name=SELECT \u8BED\u53E5\u4E0D\u5EFA\u8BAE\u4F7F\u7528 * -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.select-star-exists.description=SELECT \u8BED\u53E5\u4E0D\u5EFA\u8BAE\u4F7F\u7528 * -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.message=\u8868\u7F3A\u5C11\u5FC5\u8981\u7684\u5217\uFF1A{0} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.name=\u8868\u7F3A\u5C11\u5FC5\u8981\u7684\u5217 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.description=\u8868\u7F3A\u5C11\u5FC5\u8981\u7684\u5217 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names=\u5217\u540D\u96C6\u5408 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names.description=\u5217\u540D\u96C6\u5408 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-unsigned.message=\u81EA\u589E\u5217 {0} \u6700\u597D\u4F7F\u7528\u65E0\u7B26\u53F7\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-unsigned.name=\u9650\u5236\u81EA\u589E\u5217\u4F7F\u7528 UNSIGNED \u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-unsigned.description=\u65E0\u7B26\u53F7\u7C7B\u578B\u4E0D\u5B58\u50A8\u8D1F\u6570\uFF0C\u540C\u6837\u7684\u7C7B\u578B\u5B58\u653E\u7684\u6570\u503C\u8303\u56F4\u589E\u52A0\u4E00\u500D\uFF0C\u53EF\u4EE5\u907F\u514D\u81EA\u589E\u5217\u8D85\u51FA\u4E0A\u9650 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.message=\u5B58\u5728 {0} \u6761\u5BF9\u540C\u4E00\u4E2A\u8868 {1} \u7684\u4FEE\u6539\u8BED\u53E5\uFF0C\u8D85\u8FC7\u4E86\u6700\u5927\u9650\u5236 {2}\uFF0C\u5EFA\u8BAE\u5408\u5E76\u6210\u4E00\u4E2AALTER\u8BED\u53E5 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.name=\u9650\u5236\u540C\u8868\u4E00\u6B21\u6267\u884C\u7684 ALTER \u6570\u91CF -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.description=\u907F\u514D\u591A\u6B21 table rebuild \u5E26\u6765\u7684\u6D88\u8017\u3001\u4EE5\u53CA\u5BF9\u7EBF\u4E0A\u4E1A\u52A1\u7684\u5F71\u54CD -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count=\u540C\u8868\u5141\u8BB8 ALTER \u7684\u6570\u91CF -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count.description=\u540C\u4E00\u4E2A\u8868\u6700\u591A\u5141\u8BB8 ALTER \u7684\u6570\u91CF -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.not-null-column-without-default-value.message=\u5217\u88AB\u6807\u8BB0\u4E3A NOT NULL \u4F46\u6CA1\u6709\u8BBE\u7F6E\u9ED8\u8BA4\u503C\uFF0C\u53EF\u80FD\u4F1A\u5BFC\u81F4\u63D2\u5165\u62A5\u9519 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.not-null-column-without-default-value.name=\u5B57\u6BB5\u7EA6\u675F\u4E3A NOT NULL \u65F6\u5FC5\u987B\u5E26\u9ED8\u8BA4\u503C -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.not-null-column-without-default-value.description=\u5B57\u6BB5\u7EA6\u675F\u4E3A NOT NULL \u65F6\u5FC5\u987B\u5E26\u9ED8\u8BA4\u503C -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.message=\u4E3B\u952E\u7D22\u5F15\u7684\u547D\u540D {0} \u683C\u5F0F\u4E0D\u7B26\u5408\u89C4\u8303\uFF0C\u547D\u540D\u6A21\u5F0F\uFF1A{1} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name=\u9650\u5236\u4E3B\u952E\u7D22\u5F15\u540D\u683C\u5F0F -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.description=\u4E3B\u952E\u7D22\u5F15\u7684\u89C4\u8303\u547D\u540D\u6709\u52A9\u4E8E\u63D0\u9AD8\u6570\u636E\u5E93\u5F00\u53D1\u6548\u7387\uFF0C\u9ED8\u8BA4\uFF1Apk_${table-name}_${column-name-1}_${column-name-2}_... -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern=\u4E3B\u952E\u7D22\u5F15\u540D\u79F0\u6B63\u5219\u8868\u8FBE\u5F0F -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern.description=\u4E3B\u952E\u7D22\u5F15\u540D\u79F0\u6B63\u5219\u8868\u8FBE\u5F0F -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.message=\u6570\u636E\u7C7B\u578B {0} \u7981\u6B62\u4F7F\u7528\uFF0C\u7981\u7528\u7684\u6570\u636E\u7C7B\u578B\uFF1A{1} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.name=\u7981\u7528\u67D0\u4E9B\u6570\u636E\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.description=\u7981\u7528\u67D0\u4E9B\u6570\u636E\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names=\u7981\u7528\u7684\u6570\u636E\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names.description=\u7981\u7528\u7684\u6570\u636E\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes=\u5141\u8BB8\u7684\u6570\u636E\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes.description=\u7D22\u5F15\u4E2D\u5141\u8BB8\u7684\u6570\u636E\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.description=\u53EA\u6709\u7279\u5B9A\u6570\u636E\u7C7B\u578B\u7684\u5217\u53EF\u4EE5\u88AB\u7D22\u5F15\u5F15\u7528\uFF0C\u907F\u514D\u9519\u8BEF\u5EFA\u7ACB\u7D22\u5F15\u5BFC\u81F4\u7684\u5927\u91CF\u8D44\u6E90\u5E76\u4EA7\u751F\u4E25\u91CD\u7684\u6027\u80FD\u95EE\u9898 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.message=\u7D22\u5F15\u5F15\u7528\u5217 {0} \u7684\u6570\u636E\u7C7B\u578B\u662F {1}\uFF0C\u8BE5\u7C7B\u578B\u4E0D\u88AB\u5141\u8BB8\u5EFA\u7ACB\u7D22\u5F15\uFF0C\u5141\u8BB8\u7684\u7C7B\u578B\uFF1A{2} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.name=\u9650\u5236\u7D22\u5F15\u6570\u636E\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.message=\u5BF9\u8C61\u5220\u9664\u8BED\u53E5\u6D89\u53CA\u7684\u6570\u636E\u5E93\u5BF9\u8C61\u7C7B\u578B {0} \u4E0D\u5728\u5141\u8BB8\u8303\u56F4\u5185\uFF0C\u5141\u8BB8\u5220\u9664\u7684\u5BF9\u8C61\u7C7B\u578B\u5305\u62EC\uFF1A{1} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.name=\u9650\u5236\u53EF\u5220\u9664\u7684\u5BF9\u8C61\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.description=\u4E0D\u5141\u8BB8\u5220\u9664\u5141\u8BB8\u8303\u56F4\u4EE5\u5916\u7684\u6570\u636E\u5E93\u5BF9\u8C61 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types=\u5141\u8BB8\u5220\u9664\u7684\u6570\u636E\u5E93\u5BF9\u8C61\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types.description=\u5141\u8BB8\u5220\u9664\u7684\u6570\u636E\u5E93\u5BF9\u8C61\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-name-exists.message=\u4E3B\u952E\u7EA6\u675F/\u7D22\u5F15\u6CA1\u6709\u547D\u540D -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-name-exists.name=\u4E3B\u952E\u7EA6\u675F/\u7D22\u5F15\u9700\u8981\u8BBE\u7F6E\u540D\u79F0 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-name-exists.description=\u4E3B\u952E\u7EA6\u675F/\u7D22\u5F15\u9700\u8981\u663E\u5F0F\u5B9A\u4E49\u540D\u79F0\uFF0C\u5426\u5219\u6570\u636E\u5E93\u4F1A\u81EA\u52A8\u547D\u540D +com.oceanbase.odc.CheckViolation.LocalizedMessage=位于第 {0} 行，第 {1} 列的语句可能存在问题，详情：{2} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.syntax-error.message=语句存在语法错误，详情：{0} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.syntax-error.name=语法错误 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.syntax-error.description=待检测的语句存在语法错误 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-calculation.message=列条件上存在计算，可能导致索引失效 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-calculation.name=列条件上存在计算 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-calculation.description=列条件上存在计算，可能导致索引失效 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.message=LIKE 运算存在左模糊匹配，可能导致索引失效 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.name=LIKE 运算存在左模糊匹配 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.description=LIKE 运算存在左模糊匹配，可能导致索引失效 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-implicit-conversion.message=列条件上存在隐式类型转换，可能导致索引失效 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-implicit-conversion.name=隐式类型转换 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-implicit-conversion.description=列条件上存在隐式类型转换，可能导致索引失效 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.message=过多的 IN 值匹配可能导致查询效率降低，建议不超过 {0} 个，实际存在 {1} 个 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.name=过多的 IN 值匹配 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.description=过多的 IN 值匹配可能导致查询效率降低 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count=IN 条件的数量最大值 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count.description=条件谓词中 IN 运算中表达式数量的最大值 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.message=过多的表对象 JOIN，可能导致执行计划无法最优，推荐不超过 {0} 个，实际 {1} 个表联结 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.name=过多的表对象 JOIN +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.description=过多的表对象 JOIN，可能导致执行计划无法最优 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count=最大表联结数量 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count.description=最大表联结数量 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.message=NOT IN 条件中需要 NOT NULL 标识，避免陷入嵌套循环 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.name=NOT IN 条件中需要 NOT NULL 标识 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.description=NOT IN 条件中需要 NOT NULL 标识，避免陷入嵌套循环 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-valid-where-clause.message=UPDATE/DELETE 语句中的 WHERE 条件恒为真/假造成潜在风险 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-valid-where-clause.name=UPDATE/DELETE 语句中没有有效的 WHERE 条件 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-valid-where-clause.description=UPDATE/DELETE 语句中的 WHERE 条件恒为真/假造成潜在风险 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-where-clause-exists.message=UPDATE/DELETE 语句中没有 WHERE 条件造成潜在风险 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-where-clause-exists.name=UPDATE/DELETE 语句中没有 WHERE 条件 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-where-clause-exists.description=UPDATE/DELETE 语句中没有 WHERE 条件造成潜在风险 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.message=INSERT/REPLACE 语句中没有具体的列，可能导致隐式转换错误 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.name=INSERT/REPLACE 语句未指定列名 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.description=INSERT/REPLACE 语句中没有具体的列，可能导致隐式转换错误 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-key-exists.message=对带索引的表对象 {0} 操作没有使用索引，可能导致全表扫描，造成性能下降 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-key-exists.name=没有使用索引 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-key-exists.description=对带索引的表对象操作没有使用索引，可能导致全表扫描，造成性能下降 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-partition-key-exists.message=对分区表 {0} 操作没有使用分区键，导致无法进行分区裁剪 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-partition-key-exists.name=没有使用分区键 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-partition-key-exists.description=对分区表操作没有使用分区键，导致无法进行分区裁剪 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.message=表对象 {0} 上存在过多索引，可能导致性能下降，推荐不要超过 {1} 个 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.name=单表包含了过多的索引 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.description=单表包含了过多的索引，可能导致性能下降 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count=最大索引数量 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count.description=表定义中能够出现的索引数量最大值 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prefer-local-index.message=推荐使用局部索引 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prefer-local-index.name=推荐使用局部索引 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prefer-local-index.description=推荐使用局部索引 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.message=表对象 {0} 存在过多的列，推荐不超过 {1} 个，实际 {2} 个 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.name=单表包含了过多的列 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.description=单表包含了过多的列 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count=最大列定义数量 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count.description=表定义中能够出现的列数量的最大值 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.message=表定义中 CHAR 类型的长度过长，建议不要超过 {0} ，实际值为 {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.name=限制 CHAR 类型字段的长度 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.description=CHAR 类型字段的长度不要太长 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length=CHAR 最大允许长度 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length.description=CHAR 最大允许长度 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.message=唯一索引的命名 {0} 格式不符合规范，命名模式：{1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name=限制唯一索引名格式 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.description=唯一索引的规范命名有助于提高数据库开发效率，默认：uk_${table-name}_${column-name-1}_${column-name-2}_... +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern=唯一索引名称正则表达式 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern.description=唯一索引名称正则表达式 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.foreign-constraint-exists.message=表不能使用外键 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.foreign-constraint-exists.name=表不能使用外键 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.foreign-constraint-exists.description=表不能使用外键 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.message=表要有主键 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.name=表要有主键 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.description=表要有主键 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.message=表对象 {0} 必须含有注释 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.name=表必须注释 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.description=表对象存在注释可有助于快速了解表的业务背景 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.message=表对象不能以 {0} 命名，该名字处于黑名单中，黑名单：{1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.name=表名不能处于黑名单 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.description=建表时的表名，修改表名时均不能使用处于黑名单中的表名 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list=黑名单 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list.description=表名黑名单 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.message=表对象使用的字符集 {0} 不在允许范围内，允许使用的字符集包括：{1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.name=限制表字符集 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.description=表对象使用的字符集只能从白名单中给出的选项中选择 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.allowed-charsets=允许的字符集 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.allowed-charsets.description=允许的字符集 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.message=表对象使用的排序规则 {0} 不在允许的范围内。允许的归类包括：{1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.name=限制表对象的排序规则 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.description=表对象使用的排序规则只能从白名单中给出的选项中选择 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.allowed-collations=允许的排序规则 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.allowed-collations.description=允许的排序规则 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.message=索引中存在 {0} 次列引用，最多不应超过 {1} 次列引用 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.name=限制单个索引包含列的个数 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.description=单个索引定义中的列引用数目不应过多 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count=最大列引用数目 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count.description=最大列引用数目 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.message=主键约束涉及列的类型 {0} 不在允许范围内，允许的类型包括：{1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.name=限制主键数据类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.description=表的主键涉及的列必须是规则中定义的数据类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes=允许作为主键的类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes.description=允许作为主键的类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.message=主键中存在 {0} 次列引用，最多不应超过 {1} 次列引用 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.name=限制单个主键包含列的个数 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.description=单个主键定义中的列引用数目不应过多 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count=最大列引用数目 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count.description=最大列引用数目 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-auto-increment.message=限制主键列必须自增 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-auto-increment.name=限制主键列必须自增 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-auto-increment.description=限制主键列必须自增 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.message=索引的命名 {0} 格式不符合规范，命名模式：{1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name=限制索引名格式 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.description=索引的规范命名有助于提高数据库开发效率，默认：idx_${table-name}_${column-name-1}_${column-name-2}_... +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern=索引名称正则表达式 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern.description=索引名称正则表达式 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-name-exists.message=索引或约束的定义没有命名 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-name-exists.name=索引或约束需要设置名称 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-name-exists.description=索引或约束需要显式定义名称，否则数据库会自动命名 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.message=数值类型使用 ZEROFILL 属性 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.name=数值类型不能使用 ZEROFILL 属性 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.description=数值类型不能使用 ZEROFILL 属性 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-charset-exists.message=不能对列设置字符集 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-charset-exists.name=不能对列设置字符集 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-charset-exists.description=不能对列设置字符集 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.message=不能对列设置排序规则 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.name=不能对列设置排序规则 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.description=不能对列设置排序规则 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.message=类型为 {0} 的列不允许为空，允许为空的列的数据类型：{1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.name=限制列不可空（NOT NULL) +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.description=限制列不可空 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list=允许为空的列数据类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list.description=允许为空的列数据类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.message=数据类型为 {0} 的列不能没有默认值，允许没有默认值的列类型：{1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.name=列要有默认值 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.description=列要有默认值 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list=允许没有默认值的类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list.description=允许没有默认值的类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-column-comment-exists.message=列 {0} 没有注释 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-column-comment-exists.name=列要有注释 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-column-comment-exists.description=表定义中各个列需要有注释 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.message=列不能以 {0} 命名，该名字处于黑名单中，黑名单：{1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.name=列名不能处于黑名单 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.description=建表时的列名，修改列名时均不能使用处于黑名单中的列名 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list=黑名单 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list.description=列名黑名单 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.message=列名 {0} 的大小写设定不符合规范 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.name=限制列名大小写 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.description=限制列名大小写 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-uppercase=是否限制大写 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-uppercase.description=是否限制大写 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-lowercase=是否限制小写 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-lowercase.description=是否限制小写 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.message=表名 {0} 的大小写设定不符合规范 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.name=限制表名大小写 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.description=限制表名大小写 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-uppercase=是否限制大写 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-uppercase.description=是否限制大写 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-lowercase=是否限制小写 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-lowercase.description=是否限制小写 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.message=限制表的列自增初始值为 {0}，实际为 {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.name=限制建表自增初始值 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.description=限制建表自增初始值 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.init-value=表自增初始值 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.init-value.description=表自增初始值 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.select-star-exists.message=SELECT 语句中不建议使用 * 投影全部列 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.select-star-exists.name=SELECT 语句不建议使用 * +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.select-star-exists.description=SELECT 语句不建议使用 * +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.message=表缺少必要的列：{0} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.name=表缺少必要的列 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.description=表缺少必要的列 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names=列名集合 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names.description=列名集合 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-unsigned.message=自增列 {0} 最好使用无符号类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-unsigned.name=限制自增列使用 UNSIGNED 类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-unsigned.description=无符号类型不存储负数，同样的类型存放的数值范围增加一倍，可以避免自增列超出上限 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.message=存在 {0} 条对同一个表 {1} 的修改语句，超过了最大限制 {2}，建议合并成一个ALTER语句 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.name=限制同表一次执行的 ALTER 数量 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.description=避免多次 table rebuild 带来的消耗、以及对线上业务的影响 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count=同表允许 ALTER 的数量 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count.description=同一个表最多允许 ALTER 的数量 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.not-null-column-without-default-value.message=列被标记为 NOT NULL 但没有设置默认值，可能会导致插入报错 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.not-null-column-without-default-value.name=字段约束为 NOT NULL 时必须带默认值 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.not-null-column-without-default-value.description=字段约束为 NOT NULL 时必须带默认值 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.message=主键索引的命名 {0} 格式不符合规范，命名模式：{1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name=限制主键索引名格式 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.description=主键索引的规范命名有助于提高数据库开发效率，默认：pk_${table-name}_${column-name-1}_${column-name-2}_... +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern=主键索引名称正则表达式 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern.description=主键索引名称正则表达式 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.message=数据类型 {0} 禁止使用，禁用的数据类型：{1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.name=禁用某些数据类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.description=禁用某些数据类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names=禁用的数据类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names.description=禁用的数据类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes=允许的数据类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes.description=索引中允许的数据类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.description=只有特定数据类型的列可以被索引引用，避免错误建立索引导致的大量资源并产生严重的性能问题 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.message=索引引用列 {0} 的数据类型是 {1}，该类型不被允许建立索引，允许的类型：{2} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.name=限制索引数据类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.message=对象删除语句涉及的数据库对象类型 {0} 不在允许范围内，允许删除的对象类型包括：{1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.name=限制可删除的对象类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.description=不允许删除允许范围以外的数据库对象 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types=允许删除的数据库对象类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types.description=允许删除的数据库对象类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-name-exists.message=主键约束/索引没有命名 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-name-exists.name=主键约束/索引需要设置名称 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-name-exists.description=主键约束/索引需要显式定义名称，否则数据库会自动命名 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.message=\u5217 {0} \u88AB\u6807\u8BB0\u4E3A\u81EA\u589E\u5217\uFF0C\u8BE5\u5217\u7684\u7C7B\u578B\u4E3A {1}\uFF0C\u4E0D\u5728\u5141\u8BB8\u7684\u7C7B\u578B\u8303\u56F4\u5185\uFF1A{2}\uFF0C\u8FD9\u53EF\u80FD\u4F1A\u5BFC\u81F4\u6570\u503C\u6EA2\u51FA\u7B49\u975E\u9884\u671F\u60C5\u51B5\u7684\u53D1\u751F -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.name=\u9650\u5236\u81EA\u589E\u5217\u7684\u53EF\u9009\u6570\u636E\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.description=\u88AB\u6807\u8BB0\u4E3A\u81EA\u589E\u7684\u5217\u5E94\u8BE5\u8C28\u614E\u9009\u62E9\u6570\u636E\u7C7B\u578B\uFF0C\u907F\u514D\u6570\u503C\u6EA2\u51FA\u7B49\u9884\u671F\u5916\u60C5\u51B5\u7684\u53D1\u751F -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.allowed-datatypes=\u5141\u8BB8\u7684\u6570\u636E\u7C7B\u578B -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.allowed-datatypes.description=\u5141\u8BB8\u88AB\u6807\u8BB0\u4E3A\u81EA\u589E\u7684\u5217\u7684\u6570\u636E\u7C7B\u578B\u96C6\u5408 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.message=列 {0} 被标记为自增列，该列的类型为 {1}，不在允许的类型范围内：{2}，这可能会导致数值溢出等非预期情况的发生 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.name=限制自增列的可选数据类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.description=被标记为自增的列应该谨慎选择数据类型，避免数值溢出等预期外情况的发生 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.allowed-datatypes=允许的数据类型 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.allowed-datatypes.description=允许被标记为自增的列的数据类型集合 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.object-name-using-reserved-words.message={0} \u662F\u4FDD\u7559\u5B57\uFF0C\u4E0D\u5E94\u8BE5\u4EE5\u6B64\u4F5C\u4E3A\u5BF9\u8C61\u540D -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.object-name-using-reserved-words.name=\u4FDD\u7559\u5B57\u4E0D\u5E94\u8BE5\u4F5C\u4E3A\u5BF9\u8C61\u540D -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.object-name-using-reserved-words.description=\u4FDD\u7559\u5B57\u4E0D\u5E94\u8BE5\u4F5C\u4E3A\u5BF9\u8C61\u540D +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.object-name-using-reserved-words.message={0} 是保留字，不应该以此作为对象名 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.object-name-using-reserved-words.name=保留字不应该作为对象名 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.object-name-using-reserved-words.description=保留字不应该作为对象名 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.offline-schema-change-exists.message={0} \u662F offline \u7ED3\u6784\u53D8\u66F4\u8BED\u53E5\uFF0C\u53EF\u80FD\u8017\u65F6\u8F83\u4E45\uFF0C\u5EFA\u8BAE\u4F7F\u7528\u65E0\u9501\u7ED3\u6784\u53D8\u66F4\u6267\u884C\u8BE5\u8BED\u53E5 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.offline-schema-change-exists.name=offline \u7ED3\u6784\u53D8\u66F4\u8BED\u53E5\u5B58\u5728 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.offline-schema-change-exists.description=offline \u7ED3\u6784\u53D8\u66F4\u8BED\u53E5\u53EF\u80FD\u6D89\u53CA\u5168\u91CF\u6570\u636E\u4FEE\u6539\uFF0C\u4F1A\u9020\u6210\u6BD4\u8F83\u4E45\u8017\u65F6\uFF0C\u53EF\u80FD\u4F1A\u5F71\u54CD\u5230\u7EBF\u4E0A\u4E1A\u52A1\uFF0C\u63A8\u8350\u4F7F\u7528\u65E0\u9501\u7ED3\u6784\u53D8\u66F4\u6267\u884C\u8BE5\u64CD\u4F5C +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.offline-schema-change-exists.message={0} 是 offline 结构变更语句，可能耗时较久，建议使用无锁结构变更执行该语句 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.offline-schema-change-exists.name=offline 结构变更语句存在 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.offline-schema-change-exists.description=offline 结构变更语句可能涉及全量数据修改，会造成比较久耗时，可能会影响到线上业务，推荐使用无锁结构变更执行该操作 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.message=truncate \u8BED\u53E5\u5C06\u4F1A\u6E05\u7A7A\u8868\u7684\u6570\u636E\uFF0C\u8BF7\u8C28\u614E\u4F7F\u7528 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.name=\u5B58\u5728 truncate \u8BED\u53E5 -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.description=truncate \u8BED\u53E5\u5C06\u4F1A\u6E05\u7A7A\u8868\u6570\u636E\uFF0C\u5728\u751F\u4EA7\u73AF\u5883\u4E2D\u5341\u5206\u5371\u9669\uFF0C\u8BF7\u8C28\u614E\u4F7F\u7528 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.message=truncate 语句将会清空表的数据，请谨慎使用 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.name=存在 truncate 语句 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.description=truncate 语句将会清空表数据，在生产环境中十分危险，请谨慎使用 # below masking algorithm built-in -com.oceanbase.odc.builtin-resource.masking-algorithm.mask-all.name=\u5168\u90E8\u906E\u63A9\uFF08\u7CFB\u7EDF\u9ED8\u8BA4\uFF09 -com.oceanbase.odc.builtin-resource.masking-algorithm.personal-name-chinese.name=\u4E2A\u4EBA\u59D3\u540D\uFF08\u6C49\u5B57\u7C7B\u578B\uFF09 -com.oceanbase.odc.builtin-resource.masking-algorithm.personal-name-alphabet.name=\u4E2A\u4EBA\u59D3\u540D\uFF08\u5B57\u6BCD\u7C7B\u578B\uFF09 -com.oceanbase.odc.builtin-resource.masking-algorithm.nickname.name=\u6635\u79F0 -com.oceanbase.odc.builtin-resource.masking-algorithm.email.name=\u90AE\u7BB1 -com.oceanbase.odc.builtin-resource.masking-algorithm.address.name=\u5730\u5740 -com.oceanbase.odc.builtin-resource.masking-algorithm.phone-number.name=\u624B\u673A\u53F7\u7801 -com.oceanbase.odc.builtin-resource.masking-algorithm.fixed-line-phone-number.name=\u56FA\u5B9A\u7535\u8BDD -com.oceanbase.odc.builtin-resource.masking-algorithm.certificate-number.name=\u8BC1\u4EF6\u53F7\u7801 -com.oceanbase.odc.builtin-resource.masking-algorithm.bank-card-number.name=\u94F6\u884C\u5361\u53F7 -com.oceanbase.odc.builtin-resource.masking-algorithm.license-plate-number.name=\u8F66\u724C\u53F7 -com.oceanbase.odc.builtin-resource.masking-algorithm.device-id.name=\u8BBE\u5907\u552F\u4E00\u8BC6\u522B\u53F7 -com.oceanbase.odc.builtin-resource.masking-algorithm.ip.name=IP \u5730\u5740 -com.oceanbase.odc.builtin-resource.masking-algorithm.mac.name=MAC \u5730\u5740 +com.oceanbase.odc.builtin-resource.masking-algorithm.mask-all.name=全部遮掩（系统默认） +com.oceanbase.odc.builtin-resource.masking-algorithm.personal-name-chinese.name=个人姓名（汉字类型） +com.oceanbase.odc.builtin-resource.masking-algorithm.personal-name-alphabet.name=个人姓名（字母类型） +com.oceanbase.odc.builtin-resource.masking-algorithm.nickname.name=昵称 +com.oceanbase.odc.builtin-resource.masking-algorithm.email.name=邮箱 +com.oceanbase.odc.builtin-resource.masking-algorithm.address.name=地址 +com.oceanbase.odc.builtin-resource.masking-algorithm.phone-number.name=手机号码 +com.oceanbase.odc.builtin-resource.masking-algorithm.fixed-line-phone-number.name=固定电话 +com.oceanbase.odc.builtin-resource.masking-algorithm.certificate-number.name=证件号码 +com.oceanbase.odc.builtin-resource.masking-algorithm.bank-card-number.name=银行卡号 +com.oceanbase.odc.builtin-resource.masking-algorithm.license-plate-number.name=车牌号 +com.oceanbase.odc.builtin-resource.masking-algorithm.device-id.name=设备唯一识别号 +com.oceanbase.odc.builtin-resource.masking-algorithm.ip.name=IP 地址 +com.oceanbase.odc.builtin-resource.masking-algorithm.mac.name=MAC 地址 com.oceanbase.odc.builtin-resource.masking-algorithm.md5.name=MD5 com.oceanbase.odc.builtin-resource.masking-algorithm.sha256.name=SHA256 com.oceanbase.odc.builtin-resource.masking-algorithm.sha512.name=SHA512 com.oceanbase.odc.builtin-resource.masking-algorithm.sm3.name=SM3 -com.oceanbase.odc.builtin-resource.masking-algorithm.round.name=\u6570\u503C\u53D6\u6574 -com.oceanbase.odc.builtin-resource.masking-algorithm.null.name=\u7F6E\u7A7A -com.oceanbase.odc.builtin-resource.masking-algorithm.default.name=\u7F3A\u7701\u89C4\u5219 +com.oceanbase.odc.builtin-resource.masking-algorithm.round.name=数值取整 +com.oceanbase.odc.builtin-resource.masking-algorithm.null.name=置空 +com.oceanbase.odc.builtin-resource.masking-algorithm.default.name=缺省规则 # # builtin approval flow config # -com.oceanbase.odc.builtin-resource.regulation.approval.flow.config.auto-approval.name=\u81EA\u52A8\u5BA1\u6279 -com.oceanbase.odc.builtin-resource.regulation.approval.flow.config.auto-approval.description=\u81EA\u52A8\u5BA1\u6279 -com.oceanbase.odc.builtin-resource.regulation.approval.flow.config.project-owner.name=\u9879\u76EE\u7BA1\u7406\u5458 -com.oceanbase.odc.builtin-resource.regulation.approval.flow.config.project-owner.description=\u9879\u76EE\u7BA1\u7406\u5458 -com.oceanbase.odc.builtin-resource.regulation.approval.flow.config.project-dba.name=\u9879\u76EE DBA -com.oceanbase.odc.builtin-resource.regulation.approval.flow.config.project-dba.description=\u9879\u76EE DBA -com.oceanbase.odc.builtin-resource.regulation.approval.flow.config.project-owner-dba.name=\u9879\u76EE\u7BA1\u7406\u5458 --> \u9879\u76EE DBA -com.oceanbase.odc.builtin-resource.regulation.approval.flow.config.project-owner-dba.description=\u9879\u76EE\u7BA1\u7406\u5458 --> \u9879\u76EE DBA -com.oceanbase.odc.builtin-resource.regulation.risklevel.default-risk.name=\u9ED8\u8BA4\u98CE\u9669 -com.oceanbase.odc.builtin-resource.regulation.risklevel.default-risk.description=\u9ED8\u8BA4\u98CE\u9669 -com.oceanbase.odc.builtin-resource.regulation.risklevel.low-risk.name=\u4F4E\u98CE\u9669 -com.oceanbase.odc.builtin-resource.regulation.risklevel.low-risk.description=\u4F4E\u98CE\u9669 -com.oceanbase.odc.builtin-resource.regulation.risklevel.moderate-risk.name=\u4E2D\u98CE\u9669 -com.oceanbase.odc.builtin-resource.regulation.risklevel.moderate-risk.description=\u4E2D\u98CE\u9669 -com.oceanbase.odc.builtin-resource.regulation.risklevel.high-risk.name=\u9AD8\u98CE\u9669 -com.oceanbase.odc.builtin-resource.regulation.risklevel.high-risk.description=\u9AD8\u98CE\u9669 +com.oceanbase.odc.builtin-resource.regulation.approval.flow.config.auto-approval.name=自动审批 +com.oceanbase.odc.builtin-resource.regulation.approval.flow.config.auto-approval.description=自动审批 +com.oceanbase.odc.builtin-resource.regulation.approval.flow.config.project-owner.name=项目管理员 +com.oceanbase.odc.builtin-resource.regulation.approval.flow.config.project-owner.description=项目管理员 +com.oceanbase.odc.builtin-resource.regulation.approval.flow.config.project-dba.name=项目 DBA +com.oceanbase.odc.builtin-resource.regulation.approval.flow.config.project-dba.description=项目 DBA +com.oceanbase.odc.builtin-resource.regulation.approval.flow.config.project-owner-dba.name=项目管理员 --> 项目 DBA +com.oceanbase.odc.builtin-resource.regulation.approval.flow.config.project-owner-dba.description=项目管理员 --> 项目 DBA +com.oceanbase.odc.builtin-resource.regulation.risklevel.default-risk.name=默认风险 +com.oceanbase.odc.builtin-resource.regulation.risklevel.default-risk.description=默认风险 +com.oceanbase.odc.builtin-resource.regulation.risklevel.low-risk.name=低风险 +com.oceanbase.odc.builtin-resource.regulation.risklevel.low-risk.description=低风险 +com.oceanbase.odc.builtin-resource.regulation.risklevel.moderate-risk.name=中风险 +com.oceanbase.odc.builtin-resource.regulation.risklevel.moderate-risk.description=中风险 +com.oceanbase.odc.builtin-resource.regulation.risklevel.high-risk.name=高风险 +com.oceanbase.odc.builtin-resource.regulation.risklevel.high-risk.description=高风险 # # OBInstanceType # -com.oceanbase.odc.OBInstanceType.CLUSTER=\u96C6\u7FA4\u5B9E\u4F8B -com.oceanbase.odc.OBInstanceType.MYSQL_TENANT=MySQL \u79DF\u6237\u5B9E\u4F8B -com.oceanbase.odc.OBInstanceType.ORACLE_TENANT=Oracle \u79DF\u6237\u5B9E\u4F8B +com.oceanbase.odc.OBInstanceType.CLUSTER=集群实例 +com.oceanbase.odc.OBInstanceType.MYSQL_TENANT=MySQL 租户实例 +com.oceanbase.odc.OBInstanceType.ORACLE_TENANT=Oracle 租户实例 # # TaskType # -com.oceanbase.odc.TaskType.MULTIPLE_ASYNC=\u591A\u5E93\u53D8\u66F4 -com.oceanbase.odc.TaskType.ASYNC=\u6570\u636E\u5E93\u53D8\u66F4 -com.oceanbase.odc.TaskType.IMPORT=\u5BFC\u5165 -com.oceanbase.odc.TaskType.EXPORT=\u5BFC\u51FA -com.oceanbase.odc.TaskType.MOCKDATA=\u6A21\u62DF\u6570\u636E -com.oceanbase.odc.TaskType.ROLLBACK=\u56DE\u6EDA\u4EFB\u52A1 -com.oceanbase.odc.TaskType.PERMISSION_APPLY=\u6743\u9650\u7533\u8BF7 -com.oceanbase.odc.TaskType.SHADOWTABLE_SYNC=\u5F71\u5B50\u8868\u540C\u6B65 -com.oceanbase.odc.TaskType.PARTITION_PLAN=\u5206\u533A\u8BA1\u5212 -com.oceanbase.odc.TaskType.SQL_CHECK=SQL \u68C0\u67E5 -com.oceanbase.odc.TaskType.ALTER_SCHEDULE=\u8BA1\u5212\u53D8\u66F4 -com.oceanbase.odc.TaskType.ONLINE_SCHEMA_CHANGE=\u65E0\u9501\u7ED3\u6784\u53D8\u66F4 -com.oceanbase.odc.TaskType.GENERATE_ROLLBACK=\u5907\u4EFD\u56DE\u6EDA\u65B9\u6848\u751F\u6210 -com.oceanbase.odc.TaskType.PRE_CHECK=\u9884\u68C0\u67E5 -com.oceanbase.odc.TaskType.EXPORT_RESULT_SET=\u5BFC\u51FA\u7ED3\u679C\u96C6 -com.oceanbase.odc.TaskType.APPLY_PROJECT_PERMISSION=\u7533\u8BF7\u9879\u76EE\u6743\u9650 -com.oceanbase.odc.TaskType.APPLY_DATABASE_PERMISSION=\u7533\u8BF7\u6570\u636E\u5E93\u6743\u9650 -com.oceanbase.odc.TaskType.APPLY_TABLE_PERMISSION=\u7533\u8BF7\u6570\u636E\u8868\u6743\u9650 -com.oceanbase.odc.TaskType.SQL_PLAN=SQL \u8BA1\u5212 -com.oceanbase.odc.TaskType.DATA_ARCHIVE=\u6570\u636E\u5F52\u6863 -com.oceanbase.odc.TaskType.DATA_DELETE=\u6570\u636E\u6E05\u7406 -com.oceanbase.odc.TaskType.STRUCTURE_COMPARISON=\u7ED3\u6784\u5BF9\u6BD4 +com.oceanbase.odc.TaskType.MULTIPLE_ASYNC=多库变更 +com.oceanbase.odc.TaskType.ASYNC=数据库变更 +com.oceanbase.odc.TaskType.IMPORT=导入 +com.oceanbase.odc.TaskType.EXPORT=导出 +com.oceanbase.odc.TaskType.MOCKDATA=模拟数据 +com.oceanbase.odc.TaskType.ROLLBACK=回滚任务 +com.oceanbase.odc.TaskType.PERMISSION_APPLY=权限申请 +com.oceanbase.odc.TaskType.SHADOWTABLE_SYNC=影子表同步 +com.oceanbase.odc.TaskType.PARTITION_PLAN=分区计划 +com.oceanbase.odc.TaskType.SQL_CHECK=SQL 检查 +com.oceanbase.odc.TaskType.ALTER_SCHEDULE=计划变更 +com.oceanbase.odc.TaskType.ONLINE_SCHEMA_CHANGE=无锁结构变更 +com.oceanbase.odc.TaskType.GENERATE_ROLLBACK=备份回滚方案生成 +com.oceanbase.odc.TaskType.PRE_CHECK=预检查 +com.oceanbase.odc.TaskType.EXPORT_RESULT_SET=导出结果集 +com.oceanbase.odc.TaskType.APPLY_PROJECT_PERMISSION=申请项目权限 +com.oceanbase.odc.TaskType.APPLY_DATABASE_PERMISSION=申请数据库权限 +com.oceanbase.odc.TaskType.APPLY_TABLE_PERMISSION=申请数据表权限 +com.oceanbase.odc.TaskType.SQL_PLAN=SQL 计划 +com.oceanbase.odc.TaskType.DATA_ARCHIVE=数据归档 +com.oceanbase.odc.TaskType.DATA_DELETE=数据清理 +com.oceanbase.odc.TaskType.STRUCTURE_COMPARISON=结构对比 # # Notification event name # -com.oceanbase.odc.event.ALL_EVENTS.name=\u6240\u6709\u4E8B\u4EF6 -com.oceanbase.odc.event.TASK.EXECUTION_SUCCEEDED.name=\u6267\u884C\u6210\u529F -com.oceanbase.odc.event.TASK.EXECUTION_FAILED.name=\u6267\u884C\u5931\u8D25 -com.oceanbase.odc.event.TASK.EXECUTION_TIMEOUT.name=\u6267\u884C\u8D85\u65F6 -com.oceanbase.odc.event.TASK.PENDING_APPROVAL.name=\u5F85\u5BA1\u6279 -com.oceanbase.odc.event.TASK.APPROVED.name=\u5BA1\u6279\u901A\u8FC7 -com.oceanbase.odc.event.TASK.APPROVAL_REJECTION.name=\u5BA1\u6279\u62D2\u7EDD -com.oceanbase.odc.event.TASK.SCHEDULING_FAILED.name=\u8C03\u5EA6\u5931\u8D25 -com.oceanbase.odc.event.TASK.SCHEDULING_TIMEOUT.name=\u8C03\u5EA6\u8D85\u65F6 +com.oceanbase.odc.event.ALL_EVENTS.name=所有事件 +com.oceanbase.odc.event.TASK.EXECUTION_SUCCEEDED.name=执行成功 +com.oceanbase.odc.event.TASK.EXECUTION_FAILED.name=执行失败 +com.oceanbase.odc.event.TASK.EXECUTION_TIMEOUT.name=执行超时 +com.oceanbase.odc.event.TASK.PENDING_APPROVAL.name=待审批 +com.oceanbase.odc.event.TASK.APPROVED.name=审批通过 +com.oceanbase.odc.event.TASK.APPROVAL_REJECTION.name=审批拒绝 +com.oceanbase.odc.event.TASK.SCHEDULING_FAILED.name=调度失败 +com.oceanbase.odc.event.TASK.SCHEDULING_TIMEOUT.name=调度超时 # # Notification channel test message # -com.oceanbase.odc.notification.channel-test-message=\u3010ODC\u3011\u6D88\u606F\u901A\u9053\u9A8C\u8BC1\u6210\u529F\uFF0C\u8BF7\u5FFD\u7565\u672C\u6761\u6D4B\u8BD5\u901A\u77E5 +com.oceanbase.odc.notification.channel-test-message=【ODC】消息通道验证成功，请忽略本条测试通知 # # Permission Apply # -com.oceanbase.odc.builtin-resource.permission-apply.project.description=\u7533\u8BF7\u9879\u76EE\u3010{0}\u3011\u7684\u3010{1}\u3011\u6743\u9650 -com.oceanbase.odc.builtin-resource.permission-apply.database.description=\u7533\u8BF7\u6570\u636E\u5E93\u7684\u3010{0}\u3011\u6743\u9650 -com.oceanbase.odc.builtin-resource.permission-apply.table.description=\u7533\u8BF7\u6570\u636E\u8868\u7684\u3010{0}\u3011\u6743\u9650 +com.oceanbase.odc.builtin-resource.permission-apply.project.description=申请项目【{0}】的【{1}】权限 +com.oceanbase.odc.builtin-resource.permission-apply.database.description=申请数据库的【{0}】权限 +com.oceanbase.odc.builtin-resource.permission-apply.table.description=申请数据表的【{0}】权限 # # Multiple Async # -com.oceanbase.odc.builtin-resource.multiple-async.sub-ticket.description=\u3010{0}\u3011\u591A\u5E93\u53D8\u66F4 {1} \u7B2C{2}\u6279 {3}.{4} +com.oceanbase.odc.builtin-resource.multiple-async.sub-ticket.description=【{0}】多库变更 {1} 第{2}批 {3}.{4} # # ResourceRoleName # -com.oceanbase.odc.ResourceRoleName.OWNER=\u7BA1\u7406\u5458 +com.oceanbase.odc.ResourceRoleName.OWNER=管理员 com.oceanbase.odc.ResourceRoleName.DBA=DBA -com.oceanbase.odc.ResourceRoleName.DEVELOPER=\u666E\u901A\u6210\u5458 -com.oceanbase.odc.ResourceRoleName.SECURITY_ADMINISTRATOR=\u5B89\u5168\u7BA1\u7406\u5458 -com.oceanbase.odc.ResourceRoleName.PARTICIPANT=\u53C2\u4E0E\u8005 +com.oceanbase.odc.ResourceRoleName.DEVELOPER=普通成员 +com.oceanbase.odc.ResourceRoleName.SECURITY_ADMINISTRATOR=安全管理员 +com.oceanbase.odc.ResourceRoleName.PARTICIPANT=参与者 # # check generated update table ddl # -com.oceanbase.odc.generate-update-table-ddl-check.drop-index.message=\u8BE5\u64CD\u4F5C\u6D89\u53CA\u5220\u9664\u7D22\u5F15\uFF0C\u9AD8\u98CE\u9669\u53D8\u66F4\u8BF7\u8C28\u614E\u64CD\u4F5C! -com.oceanbase.odc.generate-update-table-ddl-check.create-index.message=\u8BE5\u64CD\u4F5C\u6D89\u53CA\u521B\u5EFA\u7D22\u5F15\uFF0C\u5982\u679C\u8868\u6570\u636E\u89C4\u6A21\u8F83\u5927\uFF0C\u6B64\u8FC7\u7A0B\u53EF\u80FD\u4F1A\u6BD4\u8F83\u8017\u65F6\uFF0C\u5EFA\u8BAE\u901A\u8FC7\u6570\u636E\u5E93\u53D8\u66F4\u4EFB\u52A1\u6267\u884C -com.oceanbase.odc.generate-update-table-ddl-check.drop-and-create-index.message=\u8BE5\u64CD\u4F5C\u6D89\u53CA\u5220\u9664\u548C\u521B\u5EFA\u7D22\u5F15\uFF0C\u9AD8\u98CE\u9669\u53D8\u66F4\u8BF7\u8C28\u614E\u64CD\u4F5C\uFF01\u6574\u4E2A\u8FC7\u7A0B\u4E0D\u662F\u539F\u5B50\u6027\u64CD\u4F5C\uFF0C\u5EFA\u8BAE\u5206\u6B65\u6267\u884C\uFF0C\u5148\u521B\u5EFA\u65B0\u7684\u7D22\u5F15\uFF0C\u5F85\u65B0\u7D22\u5F15\u521B\u5EFA\u5B8C\u6BD5\u5E76\u751F\u6548\u540E\u518D\u5220\u9664\u65E7\u7684\u7D22\u5F15\u3002\u5982\u679C\u8868\u6570\u636E\u89C4\u6A21\u8F83\u5927\uFF0C\u6B64\u8FC7\u7A0B\u53EF\u80FD\u4F1A\u6BD4\u8F83\u8017\u65F6\uFF0C\u5EFA\u8BAE\u901A\u8FC7\u6570\u636E\u5E93\u53D8\u66F4\u4EFB\u52A1\u6267\u884C +com.oceanbase.odc.generate-update-table-ddl-check.drop-index.message=该操作涉及删除索引，高风险变更请谨慎操作! +com.oceanbase.odc.generate-update-table-ddl-check.create-index.message=该操作涉及创建索引，如果表数据规模较大，此过程可能会比较耗时，建议通过数据库变更任务执行 +com.oceanbase.odc.generate-update-table-ddl-check.drop-and-create-index.message=该操作涉及删除和创建索引，高风险变更请谨慎操作！整个过程不是原子性操作，建议分步执行，先创建新的索引，待新索引创建完毕并生效后再删除旧的索引。如果表数据规模较大，此过程可能会比较耗时，建议通过数据库变更任务执行 # DatabasePermissionType -com.oceanbase.odc.DatabasePermissionType.QUERY=\u67E5\u8BE2 -com.oceanbase.odc.DatabasePermissionType.CHANGE=\u53D8\u66F4 -com.oceanbase.odc.DatabasePermissionType.EXPORT=\u5BFC\u51FA -com.oceanbase.odc.DatabasePermissionType.ACCESS=\u8BBF\u95EE +com.oceanbase.odc.DatabasePermissionType.QUERY=查询 +com.oceanbase.odc.DatabasePermissionType.CHANGE=变更 +com.oceanbase.odc.DatabasePermissionType.EXPORT=导出 +com.oceanbase.odc.DatabasePermissionType.ACCESS=访问 -com.oceanbase.odc.PartitionPlanVariableKey.INTERVAL=\u95F4\u9694 -com.oceanbase.odc.PartitionPlanVariableKey.LAST_PARTITION_VALUE=\u6700\u540E\u4E00\u4E2A\u5206\u533A\u89C4\u5219\u5BF9\u5E94\u4F4D\u7F6E\u8868\u8FBE\u5F0F\u7684\u503C -com.oceanbase.odc.partitionplan.TimeDataType=\u65F6\u95F4\u7C7B\u578B -com.oceanbase.odc.partitionplan.NumberDataType=\u6570\u5B57\u7C7B\u578B +com.oceanbase.odc.PartitionPlanVariableKey.INTERVAL=间隔 +com.oceanbase.odc.PartitionPlanVariableKey.LAST_PARTITION_VALUE=最后一个分区规则对应位置表达式的值 +com.oceanbase.odc.partitionplan.TimeDataType=时间类型 +com.oceanbase.odc.partitionplan.NumberDataType=数字类型 From 78c6d637bf2ca3926afb4ab7a83196a7c48cc5ee Mon Sep 17 00:00:00 2001 From: IL MARE Date: Wed, 3 Jul 2024 11:34:47 +0800 Subject: [PATCH 22/64] fix(flow): failed to startup a ticket (#2798) * fix timeout setting is not useful * optimize log printing * optimize exception throwing * optimize the logic * optimize the logic * add long * format the code --- .../odc/service/flow/FlowInstanceService.java | 6 ++---- .../flow/task/BaseODCFlowTaskDelegate.java | 17 +++++++++++++++-- .../task/DatabaseChangeRuntimeFlowableTask.java | 3 --- .../flow/task/MockDataRuntimeFlowableTask.java | 3 +-- .../permission/DBResourcePermissionHelper.java | 15 ++++++++++++++- .../service/session/ConnectConsoleService.java | 3 ++- 6 files changed, 34 insertions(+), 13 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java index f0acc1ee47..e1e2c3f03a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java @@ -263,8 +263,6 @@ public class FlowInstanceService { @Autowired private EnvironmentRepository environmentRepository; @Autowired - private DBResourcePermissionHelper dbResourcePermissionHelper; - @Autowired private EnvironmentService environmentService; private final List> dataTransferTaskInitHooks = new ArrayList<>(); @@ -790,8 +788,8 @@ private void checkCreateFlowInstancePermission(CreateFlowInstanceReq req) { } }); } - List unauthorizedDBResources = - dbResourcePermissionHelper.filterUnauthorizedDBResources(resource2Types, false); + List unauthorizedDBResources = this.permissionHelper + .filterUnauthorizedDBResources(resource2Types, false); if (CollectionUtils.isNotEmpty(unauthorizedDBResources)) { throw new BadRequestException(ErrorCodes.DatabaseAccessDenied, new Object[] {unauthorizedDBResources.stream() diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/BaseODCFlowTaskDelegate.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/BaseODCFlowTaskDelegate.java index d10318e9ae..b8779086a5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/BaseODCFlowTaskDelegate.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/BaseODCFlowTaskDelegate.java @@ -26,6 +26,7 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.collections4.CollectionUtils; import org.flowable.engine.delegate.DelegateExecution; @@ -102,7 +103,7 @@ public abstract class BaseODCFlowTaskDelegate extends BaseRuntimeFlowableDele private void init(DelegateExecution execution) { this.taskId = FlowTaskUtil.getTaskId(execution); - this.timeoutMillis = FlowTaskUtil.getExecutionExpirationIntervalMillis(execution); + this.timeoutMillis = getTimeoutMillis(execution); this.taskService.updateExecutorInfo(taskId, new ExecutorInfo(hostProperties)); SecurityContextUtils.setCurrentUser(FlowTaskUtil.getTaskCreator(execution)); } @@ -113,6 +114,7 @@ private void initMonitorExecutor() { .build(); scheduleExecutor = new ScheduledThreadPoolExecutor(1, threadFactory); int interval = RuntimeTaskConstants.DEFAULT_TASK_CHECK_INTERVAL_SECONDS; + AtomicBoolean isCancelled = new AtomicBoolean(false); scheduleExecutor.scheduleAtFixedRate(() -> { try { updateHeartbeatTime(); @@ -124,6 +126,13 @@ private void initMonitorExecutor() { } try { if (isCompleted() || isTimeout()) { + if (isTimeout() && !isCancelled.getAndSet(true)) { + try { + cancel(true); + } catch (Exception e) { + log.warn("Task is timeout, failed to cancel it, errorMessage={}", e.getMessage()); + } + } taskLatch.countDown(); } } catch (Exception e) { @@ -188,7 +197,7 @@ protected void run(DelegateExecution execution) throws Exception { } catch (Exception e) { log.warn("Task timeout callback method execution failed, taskId={}", taskId, e); } - throw new InterruptedException(); + throw new ServiceTaskCancelledException(); } if (!isSuccessful()) { // 监控线程出错导致闭锁失效，此种情况任务必须终止 @@ -312,6 +321,10 @@ protected void onTimeout(Long taskId, TaskService taskService) { } } + protected long getTimeoutMillis(DelegateExecution execution) { + return FlowTaskUtil.getExecutionExpirationIntervalMillis(execution); + } + /** * This method is scheduled periodically to update the progress of the task */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/DatabaseChangeRuntimeFlowableTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/DatabaseChangeRuntimeFlowableTask.java index df804f412d..652a4ad26b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/DatabaseChangeRuntimeFlowableTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/DatabaseChangeRuntimeFlowableTask.java @@ -183,9 +183,6 @@ protected void onProgressUpdate(Long taskId, TaskService taskService) { if (Objects.nonNull(asyncTaskThread)) { double progress = asyncTaskThread.getProgressPercentage(); taskService.updateProgress(taskId, progress); - if (System.currentTimeMillis() - asyncTaskThread.getStartTimestamp() > getTimeoutMillis()) { - asyncTaskThread.stopTaskAndKillQuery(sessionManageFacade); - } } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/MockDataRuntimeFlowableTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/MockDataRuntimeFlowableTask.java index 8e096dcbaf..e98a73e3da 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/MockDataRuntimeFlowableTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/MockDataRuntimeFlowableTask.java @@ -87,8 +87,7 @@ public boolean isCancelled() { private MockTaskConfig getMockTaskConfig(Long taskId, DelegateExecution execution) { OdcMockTaskConfig config = FlowTaskUtil.getMockParameter(execution); this.connectionConfig = FlowTaskUtil.getConnectionConfig(execution); - return FlowTaskUtil.generateMockConfig(taskId, execution, getTimeoutMillis(), - config, mockProperties); + return FlowTaskUtil.generateMockConfig(taskId, execution, getTimeoutMillis(), config, mockProperties); } @Override diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java index 21aab9aec4..674cbce4f4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java @@ -22,6 +22,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; @@ -32,12 +33,15 @@ import org.springframework.stereotype.Component; import com.oceanbase.odc.core.authority.util.SkipAuthorize; +import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.constant.OrganizationType; import com.oceanbase.odc.core.shared.constant.ResourceRoleName; import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.core.shared.exception.AccessDeniedException; import com.oceanbase.odc.core.shared.exception.BadRequestException; +import com.oceanbase.odc.metadb.connection.ConnectionConfigRepository; +import com.oceanbase.odc.metadb.connection.ConnectionEntity; import com.oceanbase.odc.metadb.connection.DatabaseEntity; import com.oceanbase.odc.metadb.connection.DatabaseRepository; import com.oceanbase.odc.metadb.dbobject.DBObjectEntity; @@ -83,6 +87,9 @@ public class DBResourcePermissionHelper { @Autowired private PermissionCheckWhitelist permissionCheckWhitelist; + @Autowired + private ConnectionConfigRepository connectionConfigRepository; + private static final Set ORACLE_DATA_DICTIONARY = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); private static final Set MYSQL_DATA_DICTIONARY = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); @@ -106,13 +113,19 @@ public void checkDBPermissions(Collection databaseIds, Collection entities = databaseRepository.findByIdIn(databaseIds); + Set connectionIds = entities.stream() + .map(DatabaseEntity::getConnectionId) + .filter(Objects::nonNull).collect(Collectors.toSet()); + Map id2Entity = this.connectionConfigRepository.findByIdIn(connectionIds) + .stream().collect(Collectors.toMap(ConnectionEntity::getId, e -> e)); List toCheckDatabaseIds = new ArrayList<>(); Set projectIds = getPermittedProjectIds(); for (DatabaseEntity e : entities) { if (e.getProjectId() == null) { throw new AccessDeniedException("Database is not belong to any project"); } - if (permissionCheckWhitelist.containsDatabase(e.getName(), e.getDialectType()) + DialectType dialectType = id2Entity.get(e.getConnectionId()).getDialectType(); + if (permissionCheckWhitelist.containsDatabase(e.getName(), dialectType) || projectIds.contains(e.getProjectId())) { continue; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java index c6e3ff673e..6ed7f89265 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java @@ -340,7 +340,8 @@ public AsyncExecuteResultResp getMoreResults(@NotNull String sessionId, String r Objects.isNull(timeoutSeconds) ? DEFAULT_GET_RESULT_TIMEOUT_SECONDS : timeoutSeconds; boolean shouldRemoveContext = context.isFinished(); try { - List resultList = context.getMoreSqlExecutionResults(gettingResultTimeoutSeconds * 1000); + List resultList = + context.getMoreSqlExecutionResults(gettingResultTimeoutSeconds * 1000L); List results = resultList.stream().map(jdbcGeneralResult -> { SqlExecuteResult result = generateResult(connectionSession, jdbcGeneralResult, context.getContextMap()); try (TraceStage stage = result.getSqlTuple().getSqlWatch().start(SqlExecuteStages.SQL_AFTER_CHECK)) { From 059f0bb98c436fcf3d35c8419a956ac0247f272c Mon Sep 17 00:00:00 2001 From: yiminpeng <101048604+yiminpeng@users.noreply.github.com> Date: Wed, 3 Jul 2024 17:18:51 +0800 Subject: [PATCH 23/64] fix(migrate): fix login process resource load faild (#2883) * delete useless resource add * delete useless resource add --- .../service/collaboration/IndividualOrganizationMigrator.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/IndividualOrganizationMigrator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/IndividualOrganizationMigrator.java index 286c2298ed..ff3b2146c1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/IndividualOrganizationMigrator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/IndividualOrganizationMigrator.java @@ -99,9 +99,6 @@ private ResourceConfig getResourceConfig(DataSource dataSource, Long userId, Lon List resourceLocations = new LinkedList<>(); resourceLocations.add("migrate/common/V_3_2_0_6__iam_permission.yaml"); resourceLocations.add("migrate/common/V_3_3_0_4__iam_permission.yaml"); - resourceLocations.add("migrate/common/V_3_4_0_2__iam_permission.yaml"); - resourceLocations.add("migrate/common/V_3_4_0_13__data_masking_rule.yaml"); - resourceLocations.add("migrate/common/V_3_4_0_14__data_masking_rule_segment.yaml"); resourceLocations.add("migrate/common/V_4_1_0_7__automation_event_metadata.yaml"); resourceLocations.add("migrate/common/V_4_1_0_14__iam_permission.yaml"); resourceLocations.add("migrate/common/V_4_2_0_24__resource_role.yaml"); From c0863786fb410d08022d6b76a27699cbd93004bd Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Wed, 3 Jul 2024 19:19:53 +0800 Subject: [PATCH 24/64] fix(sql): the sql of modifying session parameter in oracle is error (#2872) * fix oracle sql of modifying the session parameters * implement method getAlterSessionVariableStatement * modify method name * unsupport global paramters now * add UnsupportedOperationException when modifying the global or system variable of oracle --- .../oceanbase/odc/service/db/DBVariablesService.java | 12 ++++++++---- .../plugin/connect/api/SessionExtensionPoint.java | 2 ++ .../connect/obmysql/OBMySQLSessionExtension.java | 5 +++++ .../connect/oboracle/OBOracleSessionExtension.java | 5 +++++ .../connect/oracle/OracleSessionExtension.java | 8 ++++++++ 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBVariablesService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBVariablesService.java index f23c7588a8..38d223da76 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBVariablesService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBVariablesService.java @@ -27,10 +27,13 @@ import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.session.ConnectionSessionConstants; +import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.constant.OdcConstants; import com.oceanbase.odc.core.sql.execute.model.SqlTuple; +import com.oceanbase.odc.plugin.connect.api.SessionExtensionPoint; import com.oceanbase.odc.service.db.browser.DBSchemaAccessors; import com.oceanbase.odc.service.db.model.OdcDBVariable; +import com.oceanbase.odc.service.plugin.ConnectionPluginUtil; import com.oceanbase.odc.service.session.interceptor.NlsFormatInterceptor; import com.oceanbase.tools.dbbrowser.model.DBVariable; import com.oceanbase.tools.dbbrowser.model.datatype.DataTypeUtil; @@ -139,7 +142,7 @@ public List list(ConnectionSession connectionSession, String vari } public boolean update(@NonNull ConnectionSession session, @NonNull OdcDBVariable resource) { - String dml = getUpdateDml(resource.getVariableScope(), resource); + String dml = getUpdateDml(resource.getVariableScope(), resource, session.getDialectType()); if (StringUtils.isEmpty(dml)) { return false; } @@ -148,12 +151,13 @@ public boolean update(@NonNull ConnectionSession session, @NonNull OdcDBVariable return true; } - private String getUpdateDml(@NonNull String variableScope, @NonNull OdcDBVariable resource) { + private String getUpdateDml(@NonNull String variableScope, @NonNull OdcDBVariable resource, + @NonNull DialectType dialectType) { String value = resource.getValue(); if (!DataTypeUtil.isNumericValue(value)) { value = "'" + value + "'"; } - return String.format("set %s %s=%s", variableScope, resource.getKey(), value); + SessionExtensionPoint extensionPoint = ConnectionPluginUtil.getSessionExtension(dialectType); + return extensionPoint.getAlterVariableStatement(variableScope, resource.getKey(), value); } - } diff --git a/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/SessionExtensionPoint.java b/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/SessionExtensionPoint.java index 7432ac17fb..69f002776e 100644 --- a/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/SessionExtensionPoint.java +++ b/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/SessionExtensionPoint.java @@ -34,4 +34,6 @@ public interface SessionExtensionPoint extends ExtensionPoint, SessionOperations String getCurrentSchema(Connection connection); String getVariable(Connection connection, String variableName); + + String getAlterVariableStatement(String variableScope, String variableName, String variableValue); } diff --git a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLSessionExtension.java b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLSessionExtension.java index 1f2a4e47a8..72ea8b6106 100644 --- a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLSessionExtension.java +++ b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLSessionExtension.java @@ -92,4 +92,9 @@ public String getVariable(Connection connection, String variableName) { } return null; } + + @Override + public String getAlterVariableStatement(String variableScope, String variableName, String variableValue) { + return String.format("set %s %s=%s", variableScope, variableName, variableValue); + } } diff --git a/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleSessionExtension.java b/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleSessionExtension.java index 749c12004b..c629d7d891 100644 --- a/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleSessionExtension.java +++ b/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleSessionExtension.java @@ -102,4 +102,9 @@ public String getVariable(Connection connection, String variableName) { } return null; } + + @Override + public String getAlterVariableStatement(String variableScope, String variableName, String variableValue) { + return String.format("set %s %s=%s", variableScope, variableName, variableValue); + } } diff --git a/server/plugins/connect-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oracle/OracleSessionExtension.java b/server/plugins/connect-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oracle/OracleSessionExtension.java index ed1e0c5dd8..a4853812da 100644 --- a/server/plugins/connect-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oracle/OracleSessionExtension.java +++ b/server/plugins/connect-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oracle/OracleSessionExtension.java @@ -89,4 +89,12 @@ public String getVariable(@NonNull Connection connection, @NonNull String variab } return value; } + + @Override + public String getAlterVariableStatement(String variableScope, String variableName, String variableValue) { + if ("system".equals(variableScope) || "global".equals(variableScope)) { + throw new UnsupportedOperationException("modifying the global or system variable of oracle is unsupported"); + } + return String.format("alter %s set %s=%s", variableScope, variableName, variableValue); + } } From 1f623b7f5ed416c2c3a4056faa81d76e74a4801c Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Thu, 4 Jul 2024 15:27:42 +0800 Subject: [PATCH 25/64] fix(mock data): failed to cancel the mock data task (#2850) * Optimized error when cancelling mockdata ticket * Error message localization * modify the message * merge conflict * Modified according to pr requirements * modify code format * rename the enum * modify the code format * Modify according to pr requirements * use synchronized to ensure that context is present when canceling mock data ticket * Complete the cancellation logic when cancel method precedes the start method --- .../i18n/BusinessMessages_zh_TW.properties | 2 +- .../task/MockDataRuntimeFlowableTask.java | 28 +++++++++++++------ .../flow/task/model/MockDataTaskResult.java | 23 +++++++++------ 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties index b06c5b00ed..3a3c7e0c44 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties @@ -899,4 +899,4 @@ com.oceanbase.odc.DatabasePermissionType.ACCESS=訪問 com.oceanbase.odc.PartitionPlanVariableKey.INTERVAL=間隔 com.oceanbase.odc.PartitionPlanVariableKey.LAST_PARTITION_VALUE=最後一個分區規則對應位置表達式的值 com.oceanbase.odc.partitionplan.TimeDataType=時間類型 -com.oceanbase.odc.partitionplan.NumberDataType=數字類型 +com.oceanbase.odc.partitionplan.NumberDataType=數字類型 \ No newline at end of file diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/MockDataRuntimeFlowableTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/MockDataRuntimeFlowableTask.java index e98a73e3da..601c385855 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/MockDataRuntimeFlowableTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/MockDataRuntimeFlowableTask.java @@ -58,18 +58,25 @@ public class MockDataRuntimeFlowableTask extends BaseODCFlowTaskDelegate { private Exception thrown = null; private volatile MockContext context; private volatile ConnectionConfig connectionConfig; + private volatile boolean cancelled = false; @Override - public boolean cancel(boolean mayInterruptIfRunning, Long taskId, TaskService taskService) { - Verify.notNull(context, "MockContext is null"); + public synchronized boolean cancel(boolean mayInterruptIfRunning, Long taskId, TaskService taskService) { + MockTaskStatus status = MockTaskStatus.CANCELED; + boolean isInterrupted = true; + if (this.context == null) { + this.cancelled = true; + } else { + isInterrupted = context.shutdown(); + TableTaskContext tableContext = context.getTables().get(0); + status = tableContext == null ? null : tableContext.getStatus(); + } Map variables = new HashMap<>(); variables.putIfAbsent("mocktask.workspace", taskId + ""); TraceContextHolder.span(variables); - boolean isInterrupted = context.shutdown(); taskService.cancel(taskId, getResult(taskId)); - TableTaskContext tableContext = context.getTables().get(0); - log.info("The mock data task was cancelled, taskId={}, status={}, cancelResult={}", taskId, - tableContext == null ? null : tableContext.getStatus(), isInterrupted); + log.info("MockData task was cancelled, taskId={}, status={}, cancelResult={}", taskId, + status, isInterrupted); TraceContextHolder.clear(); return isInterrupted; } @@ -77,7 +84,7 @@ public boolean cancel(boolean mayInterruptIfRunning, Long taskId, TaskService ta @Override public boolean isCancelled() { if (context == null) { - return false; + return this.cancelled; } TableTaskContext tableTaskContext = context.getTables().get(0); Verify.notNull(tableTaskContext, "TableTaskContext"); @@ -91,11 +98,16 @@ private MockTaskConfig getMockTaskConfig(Long taskId, DelegateExecution executio } @Override - protected Void start(Long taskId, TaskService taskService, DelegateExecution execution) { + protected synchronized Void start(Long taskId, TaskService taskService, DelegateExecution execution) { try { Map variables = new HashMap<>(); variables.putIfAbsent("mocktask.workspace", taskId + ""); TraceContextHolder.span(variables); + if (this.cancelled) { + log.warn("MockData task has been cancelled, taskId={}, activityId={}", taskId, + execution.getCurrentActivityId()); + return null; + } log.info("MockData task starts, taskId={}, activityId={}", taskId, execution.getCurrentActivityId()); MockTaskConfig mockTaskConfig = getMockTaskConfig(taskId, execution); ObMockerFactory factory = new ObMockerFactory(mockTaskConfig); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/model/MockDataTaskResult.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/model/MockDataTaskResult.java index 4d0c142aae..37adac1dd5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/model/MockDataTaskResult.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/model/MockDataTaskResult.java @@ -62,16 +62,21 @@ public class MockDataTaskResult extends AbstractFlowTaskResult { @Deprecated private List tableTaskIds; - public MockDataTaskResult(@NonNull ConnectionConfig connectionConfig, @NonNull MockContext context) { - List tableTaskContexts = context.getTables(); - if (CollectionUtils.isNotEmpty(tableTaskContexts)) { - TableTaskContext tableTaskContext = tableTaskContexts.get(0); - this.taskStatus = tableTaskContext.getStatus(); - this.writeCount = tableTaskContext.getTotalWriteCount(); - this.totalGen = tableTaskContext.getTotalGenerateCount(); + public MockDataTaskResult(ConnectionConfig connectionConfig, MockContext context) { + if (context != null) { + List tableTaskContexts = context.getTables(); + if (CollectionUtils.isNotEmpty(tableTaskContexts)) { + TableTaskContext tableTaskContext = tableTaskContexts.get(0); + this.taskStatus = tableTaskContext.getStatus(); + this.writeCount = tableTaskContext.getTotalWriteCount(); + this.totalGen = tableTaskContext.getTotalGenerateCount(); + } } - this.sessionName = connectionConfig.getName(); - this.dbMode = connectionConfig.getDialectType(); + if (context != null) { + this.sessionName = connectionConfig.getName(); + this.dbMode = connectionConfig.getDialectType(); + } + } public MockDataTaskResult(@NonNull ConnectionConfig connectionConfig) { From eae924313861d34ca8de2284e9c2298599917644 Mon Sep 17 00:00:00 2001 From: IL MARE Date: Tue, 9 Jul 2024 10:31:10 +0800 Subject: [PATCH 26/64] refactor: change the CODEOWNERS (#2931) --- .github/CODEOWNERS | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5ae66039e9..b205a78b07 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -27,7 +27,7 @@ /server/odc-core/**/session/ @yhilmare /server/odc-core/**/shared/ @yhilmare @yizhouxw /server/odc-core/**/sql/ @yhilmare @LuckyPickleZZ -/server/odc-core/**/task/ @yhilmare @krihy +/server/odc-core/**/task/ @yhilmare @yizhouxw # service common /server/odc-service/ @yhilmare @yizhouxw @@ -50,7 +50,7 @@ /server/odc-service/**/service/dml/ @LuckyPickleZZ @PeachThinking /server/odc-service/**/service/encryption/ @PeachThinking @yizhouxw /server/odc-service/**/service/feature/ @MarkPotato777 @yizhouxw -/server/odc-service/**/service/flow/ @yhilmare @krihy +/server/odc-service/**/service/flow/ @yhilmare @yizhouxw /server/odc-service/**/service/i18n/ @LuckyPickleZZ /server/odc-service/**/service/iam/ @MarkPotato777 @PeachThinking /server/odc-service/**/service/info/ @yhilmare @yizhouxw @@ -59,11 +59,11 @@ /server/odc-service/**/service/monitor/ @ungreat @yizhouxw /server/odc-service/**/service/notification/ @LuckyPickleZZ @MarkPotato777 /server/odc-service/**/service/objectstorage/ @MarkPotato777 @yizhouxw -/server/odc-service/**/service/onlineschemachange/ @krihy @LuckyPickleZZ +/server/odc-service/**/service/onlineschemachange/ @LioRoger @LuckyPickleZZ /server/odc-service/**/service/partitionplan/ @guowl3 @yhilmare /server/odc-service/**/service/permission/ @MarkPotato777 @yhilmare -/server/odc-service/**/service/pldebug/ @yhilmare @krihy -/server/odc-service/**/service/plugin/ @yhilmare @krihy +/server/odc-service/**/service/pldebug/ @yhilmare @yizhouxw +/server/odc-service/**/service/plugin/ @yhilmare @LuckyPickleZZ /server/odc-service/**/service/quartz/ @guowl3 @yhilmare /server/odc-service/**/service/requlation/ @MarkPotato777 @yhilmare /server/odc-service/**/service/resourcegroup/ @MarkPotato777 @yhilmare @@ -77,16 +77,16 @@ /server/odc-service/**/service/sqlcheck/ @yhilmare @PeachThinking /server/odc-service/**/service/structurecompare/ @PeachThinking @yhilmare /server/odc-service/**/service/systemconfig/ @MarkPotato777 @yizhouxw -/server/odc-service/**/service/task/ @yhilmare @krihy @yizhouxw +/server/odc-service/**/service/task/ @yhilmare @yizhouxw /server/odc-service/**/service/websocket/ @LuckyPickleZZ @yizhouxw # plugins /server/plugins/ @yhilmare @yizhouxw -/server/plugins/connect-plugin-doris/ @yhilmare @krihy -/server/plugins/connect-plugin-mysql/ @yhilmare @krihy -/server/plugins/connect-plugin-ob-mysql/ @yhilmare @krihy -/server/plugins/connect-plugin-ob-oracle/ @yhilmare @krihy -/server/plugins/connect-plugin-oracle/ @yhilmare @krihy +/server/plugins/connect-plugin-doris/ @yhilmare @yizhouxw +/server/plugins/connect-plugin-mysql/ @yhilmare @yizhouxw +/server/plugins/connect-plugin-ob-mysql/ @yhilmare @yizhouxw +/server/plugins/connect-plugin-ob-oracle/ @yhilmare @yizhouxw +/server/plugins/connect-plugin-oracle/ @yhilmare @yizhouxw /server/plugins/schema-plugin-api/ @PeachThinking @MarkPotato777 /server/plugins/schema-plugin-doris/ @PeachThinking @MarkPotato777 From f56a42a866847aed5c04c8b702ab9971d3fe6a11 Mon Sep 17 00:00:00 2001 From: IL MARE Date: Tue, 9 Jul 2024 15:12:03 +0800 Subject: [PATCH 27/64] fix(pl-debug): avoid npe during the pl debugging (#2930) --- .../oceanbase/odc/service/pldebug/session/PLDebugSession.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/PLDebugSession.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/PLDebugSession.java index 22c86061c8..6f2eb7a732 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/PLDebugSession.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/PLDebugSession.java @@ -125,6 +125,10 @@ public synchronized void end() throws Exception { public synchronized PLDebugContextResp getContext() { PLDebugContextResp plDebugContextResp = new PLDebugContextResp(); + if (debuggerSession == null || debuggeeSession == null) { + throw OBException.executeFailed(ErrorCodes.DebugTimeout, + String.format("Debug timeout for %d ms", timeoutMilliSeconds)); + } // terminated boolean terminated = !debuggerSession.detectSessionAlive() || !debuggeeSession.detectSessionAlive(); plDebugContextResp.setTerminated(terminated); From ac46fd84340ac2468b91bdbe9b4cc4330b6898a8 Mon Sep 17 00:00:00 2001 From: IL MARE Date: Wed, 10 Jul 2024 14:14:18 +0800 Subject: [PATCH 28/64] fix(import): add template api and supports mysql, oracle and doris datasource importing (#2936) * add template api and supports mysql,oracle and doris importing * response to cr comment --- .../odc/core/shared/constant/FieldName.java | 3 +++ .../i18n/BusinessMessages.properties | 3 +++ .../i18n/BusinessMessages_zh_CN.properties | 3 +++ .../i18n/BusinessMessages_zh_TW.properties | 3 +++ .../controller/v2/DataSourceController.java | 12 ++++++++++ .../web/controller/v2/IamController.java | 10 ++++++++ .../ConnectionBatchImportPreviewer.java | 22 ++++++++++++++---- .../service/connection/ConnectionService.java | 22 ++++++++++++++++++ .../odc/service/iam/UserService.java | 22 ++++++++++++++++++ ...OrganizationAuthenticationInterceptor.java | 4 +++- .../template/en-us/datasource_template.xlsx | Bin 0 -> 9542 bytes .../template/en-us/user_template.xlsx | Bin 0 -> 9300 bytes .../template/zh-cn/datasource_template.xlsx | Bin 0 -> 9619 bytes .../template/zh-cn/user_template.xlsx | Bin 0 -> 9342 bytes .../template/zh-tw/datasource_template.xlsx | Bin 0 -> 9647 bytes .../template/zh-tw/user_template.xlsx | Bin 0 -> 9329 bytes 16 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 server/odc-service/src/main/resources/template/en-us/datasource_template.xlsx create mode 100644 server/odc-service/src/main/resources/template/en-us/user_template.xlsx create mode 100644 server/odc-service/src/main/resources/template/zh-cn/datasource_template.xlsx create mode 100644 server/odc-service/src/main/resources/template/zh-cn/user_template.xlsx create mode 100644 server/odc-service/src/main/resources/template/zh-tw/datasource_template.xlsx create mode 100644 server/odc-service/src/main/resources/template/zh-tw/user_template.xlsx diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/FieldName.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/FieldName.java index 5c6774f1d3..841e0fba2b 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/FieldName.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/FieldName.java @@ -41,6 +41,9 @@ public enum FieldName implements Translatable { DATASOURCE_ENVIRONMENT_DEV, DATASOURCE_ENVIRONMENT_PROD, DATASOURCE_ENVIRONMENT_SIT, + DATASOURCE_SERVICE_NAME, + DATASOURCE_SID, + DATASOURCE_USER_ROLE, /** * Batch Import User diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages.properties index e175d91034..157a0353f1 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages.properties @@ -102,6 +102,9 @@ com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT_DEFAULT=default com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT_DEV=dev com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT_PROD=prod com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT_SIT=sit +com.oceanbase.odc.FieldName.DATASOURCE_SERVICE_NAME=Service Name +com.oceanbase.odc.FieldName.DATASOURCE_SID=SID +com.oceanbase.odc.FieldName.DATASOURCE_USER_ROLE=User Role com.oceanbase.odc.FieldName.USER_ACCOUNTNAME=Account com.oceanbase.odc.FieldName.USER_NAME=Username com.oceanbase.odc.FieldName.USER_PASSWORD=Password diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties index a4078edcda..3ba6f8223f 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties @@ -102,6 +102,9 @@ com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT_DEFAULT=默认 com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT_DEV=开发 com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT_PROD=生产 com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT_SIT=测试 +com.oceanbase.odc.FieldName.DATASOURCE_SERVICE_NAME=服务名 +com.oceanbase.odc.FieldName.DATASOURCE_SID=SID +com.oceanbase.odc.FieldName.DATASOURCE_USER_ROLE=角色 com.oceanbase.odc.FieldName.USER_ACCOUNTNAME=账号 com.oceanbase.odc.FieldName.USER_NAME=姓名 com.oceanbase.odc.FieldName.USER_PASSWORD=密码 diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties index 3a3c7e0c44..c3aeef9b74 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties @@ -102,6 +102,9 @@ com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT_DEFAULT=默認 com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT_DEV=開發 com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT_PROD=生產 com.oceanbase.odc.FieldName.DATASOURCE_ENVIRONMENT_SIT=測試 +com.oceanbase.odc.FieldName.DATASOURCE_SERVICE_NAME=服務名 +com.oceanbase.odc.FieldName.DATASOURCE_SID=SID +com.oceanbase.odc.FieldName.DATASOURCE_USER_ROLE=角色 com.oceanbase.odc.FieldName.USER_ACCOUNTNAME=帳號 com.oceanbase.odc.FieldName.USER_NAME=姓名 com.oceanbase.odc.FieldName.USER_PASSWORD=密碼 diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DataSourceController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DataSourceController.java index 6ca37bea77..026918a166 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DataSourceController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DataSourceController.java @@ -24,9 +24,11 @@ import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.InputStreamResource; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort.Direction; import org.springframework.data.web.PageableDefault; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; @@ -43,6 +45,7 @@ import com.oceanbase.odc.service.common.response.PaginatedResponse; import com.oceanbase.odc.service.common.response.Responses; import com.oceanbase.odc.service.common.response.SuccessResponse; +import com.oceanbase.odc.service.common.util.WebResponseUtils; import com.oceanbase.odc.service.connection.ConnectionBatchImportPreviewer; import com.oceanbase.odc.service.connection.ConnectionHelper; import com.oceanbase.odc.service.connection.ConnectionService; @@ -54,6 +57,7 @@ import com.oceanbase.odc.service.connection.model.ConnectionPreviewBatchImportResp; import com.oceanbase.odc.service.connection.model.GenerateConnectionStringReq; import com.oceanbase.odc.service.connection.model.QueryConnectionParams; +import com.oceanbase.odc.service.flow.model.BinaryDataResult; import io.swagger.annotations.ApiOperation; @@ -178,6 +182,14 @@ public SuccessResponse previewBatchImportDataS return Responses.success(connectionBatchImportPreviewer.preview(file)); } + @ApiOperation(value = "downloadDatasourceTemplate", notes = "Download Datasource Template File") + @RequestMapping(value = "/datasources/template", method = RequestMethod.GET) + public ResponseEntity downloadTemplate() throws IOException { + BinaryDataResult result = this.connectionService.getBatchImportTemplateFile(); + return WebResponseUtils.getFileAttachmentResponseEntity( + new InputStreamResource(result.getInputStream()), (result.getName())); + } + @ApiOperation(value = "batchCreateDataSources", notes = "Batch create datasources") @RequestMapping(value = "/datasources/batchCreate", method = RequestMethod.POST) public SuccessResponse> batchCreateDataSources( diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/IamController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/IamController.java index fae6f3bfea..9a39dbbf87 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/IamController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/IamController.java @@ -49,6 +49,7 @@ import com.oceanbase.odc.service.common.response.Responses; import com.oceanbase.odc.service.common.response.SuccessResponse; import com.oceanbase.odc.service.common.util.WebResponseUtils; +import com.oceanbase.odc.service.flow.model.BinaryDataResult; import com.oceanbase.odc.service.iam.OrganizationService; import com.oceanbase.odc.service.iam.ResourceRoleService; import com.oceanbase.odc.service.iam.RoleService; @@ -407,6 +408,14 @@ public SuccessResponse previewBatchImportUser(@Reque return Responses.single(userBatchImportPreviewer.preview(file)); } + @ApiOperation(value = "downloadUserTemplate", notes = "Download User Template File") + @RequestMapping(value = "/users/template", method = RequestMethod.GET) + public ResponseEntity downloadTemplate() throws IOException { + BinaryDataResult result = this.userService.getBatchImportTemplateFile(); + return WebResponseUtils.getFileAttachmentResponseEntity( + new InputStreamResource(result.getInputStream()), (result.getName())); + } + /** * 批量导入用户 * @@ -418,4 +427,5 @@ public SuccessResponse previewBatchImportUser(@Reque public ListResponse batchCreateUsers(@RequestBody List createUserReqs) { return Responses.list(userService.batchImport(createUserReqs)); } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionBatchImportPreviewer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionBatchImportPreviewer.java index 263c639650..ead3bfbfda 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionBatchImportPreviewer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionBatchImportPreviewer.java @@ -25,11 +25,14 @@ import static com.oceanbase.odc.core.shared.constant.FieldName.DATASOURCE_NAME; import static com.oceanbase.odc.core.shared.constant.FieldName.DATASOURCE_PASSWORD; import static com.oceanbase.odc.core.shared.constant.FieldName.DATASOURCE_PORT; +import static com.oceanbase.odc.core.shared.constant.FieldName.DATASOURCE_SERVICE_NAME; +import static com.oceanbase.odc.core.shared.constant.FieldName.DATASOURCE_SID; import static com.oceanbase.odc.core.shared.constant.FieldName.DATASOURCE_SYSTENANTPASSWORD; import static com.oceanbase.odc.core.shared.constant.FieldName.DATASOURCE_SYSTENANTUSERNAME; import static com.oceanbase.odc.core.shared.constant.FieldName.DATASOURCE_TENANTNAME; import static com.oceanbase.odc.core.shared.constant.FieldName.DATASOURCE_TYPE; import static com.oceanbase.odc.core.shared.constant.FieldName.DATASOURCE_USERNAME; +import static com.oceanbase.odc.core.shared.constant.FieldName.DATASOURCE_USER_ROLE; import java.io.IOException; import java.io.InputStream; @@ -48,6 +51,7 @@ import com.oceanbase.odc.core.authority.util.PreAuthenticate; import com.oceanbase.odc.core.shared.constant.ConnectType; import com.oceanbase.odc.core.shared.constant.ErrorCodes; +import com.oceanbase.odc.plugin.connect.model.oracle.UserRole; import com.oceanbase.odc.service.collaboration.environment.EnvironmentService; import com.oceanbase.odc.service.collaboration.environment.model.Environment; import com.oceanbase.odc.service.collaboration.environment.model.QueryEnvironmentParam; @@ -131,11 +135,7 @@ private BatchImportConnection createBatchImportConnection(Map ma String clusterName = map.get(DATASOURCE_CLUSTERNAME.getLocalizedMessage()); batchImportConnection.setClusterName(clusterName); String tenantName = map.get(DATASOURCE_TENANTNAME.getLocalizedMessage()); - if (StringUtils.isEmpty(tenantName) || !SPACE_PATTERN.matcher(tenantName).matches()) { - batchImportConnection.setErrorMessage("file content error:" + DATASOURCE_TENANTNAME.getLocalizedMessage()); - } else { - batchImportConnection.setTenantName(tenantName); - } + batchImportConnection.setTenantName(tenantName); String username = map.get(DATASOURCE_USERNAME.getLocalizedMessage()); if (StringUtils.isEmpty(username) || !SPACE_PATTERN.matcher(username).matches()) { batchImportConnection.setErrorMessage("file content error:" + DATASOURCE_USERNAME.getLocalizedMessage()); @@ -160,6 +160,18 @@ private BatchImportConnection createBatchImportConnection(Map ma batchImportConnection.setSysTenantUsername(sysTenantUsername); String sysTenantPassword = map.get(DATASOURCE_SYSTENANTPASSWORD.getLocalizedMessage()); batchImportConnection.setSysTenantPassword(sysTenantPassword); + String serviceName = map.get(DATASOURCE_SERVICE_NAME.getLocalizedMessage()); + if (StringUtils.isNotEmpty(serviceName) && SPACE_PATTERN.matcher(serviceName).matches()) { + batchImportConnection.setServiceName(serviceName); + } + String sid = map.get(DATASOURCE_SID.getLocalizedMessage()); + if (StringUtils.isNotEmpty(sid) && SPACE_PATTERN.matcher(sid).matches()) { + batchImportConnection.setSid(sid); + } + String oracleUserRole = map.get(DATASOURCE_USER_ROLE.getLocalizedMessage()); + if (StringUtils.isNotEmpty(oracleUserRole) && SPACE_PATTERN.matcher(oracleUserRole).matches()) { + batchImportConnection.setUserRole(UserRole.valueOf(oracleUserRole)); + } return batchImportConnection; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java index aa48eb9a7c..607e886a3e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java @@ -15,6 +15,8 @@ */ package com.oceanbase.odc.service.connection; +import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -37,9 +39,11 @@ import javax.validation.constraints.NotNull; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; +import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.data.domain.Example; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; @@ -108,6 +112,8 @@ import com.oceanbase.odc.service.connection.util.ConnectionIdList; import com.oceanbase.odc.service.connection.util.ConnectionMapper; import com.oceanbase.odc.service.db.schema.syncer.DBSchemaSyncProperties; +import com.oceanbase.odc.service.flow.model.BinaryDataResult; +import com.oceanbase.odc.service.flow.model.ByteArrayDataResult; import com.oceanbase.odc.service.iam.HorizontalDataPermissionValidator; import com.oceanbase.odc.service.iam.PermissionService; import com.oceanbase.odc.service.iam.ProjectPermissionValidator; @@ -220,6 +226,8 @@ public class ConnectionService { private static final String UPDATE_DS_SCHEMA_LOCK_KEY_PREFIX = "update-ds-schema-lock-"; + private static final String DATASOURCE_TEMPLATE_FILE_NAME = "datasource_template.xlsx"; + @PreAuthenticate(actions = "create", resourceType = "ODC_CONNECTION", isForAll = true) public ConnectionConfig create(@NotNull @Valid ConnectionConfig connection) { return create(connection, currentUserId(), false); @@ -740,6 +748,20 @@ public boolean checkPermission(@NotNull Collection connectionIds, @NotNull return true; } + @SkipAuthorize + public BinaryDataResult getBatchImportTemplateFile() throws IOException { + String locale = LocaleContextHolder.getLocale().toLanguageTag().toLowerCase(); + try (InputStream input = this.getClass().getClassLoader().getResourceAsStream( + "template/" + locale + "/" + DATASOURCE_TEMPLATE_FILE_NAME)) { + if (input == null) { + throw new UnexpectedException(DATASOURCE_TEMPLATE_FILE_NAME + " is not found"); + } + byte[] buffer = new byte[input.available()]; + IOUtils.read(input, buffer); + return new ByteArrayDataResult(DATASOURCE_TEMPLATE_FILE_NAME, buffer); + } + } + private Page innerList(@NotNull QueryConnectionParams params, @NotNull Pageable pageable) { Specification spec = Specification .where(ConnectionSpecs.organizationIdEqual(authenticationFacade.currentOrganizationId())); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java index 676fdf24b8..d3a77c9a3a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java @@ -15,6 +15,8 @@ */ package com.oceanbase.odc.service.iam; +import java.io.IOException; +import java.io.InputStream; import java.sql.Timestamp; import java.time.Duration; import java.util.ArrayList; @@ -40,8 +42,10 @@ import javax.validation.constraints.NotNull; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang.Validate; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; @@ -77,6 +81,7 @@ import com.oceanbase.odc.core.shared.exception.BadArgumentException; import com.oceanbase.odc.core.shared.exception.BadRequestException; import com.oceanbase.odc.core.shared.exception.NotFoundException; +import com.oceanbase.odc.core.shared.exception.UnexpectedException; import com.oceanbase.odc.core.shared.exception.UnsupportedException; import com.oceanbase.odc.metadb.iam.LastSuccessLoginHistory; import com.oceanbase.odc.metadb.iam.LoginHistoryRepository; @@ -98,6 +103,8 @@ import com.oceanbase.odc.service.common.response.CustomPage; import com.oceanbase.odc.service.common.response.PaginatedData; import com.oceanbase.odc.service.common.util.SpringContextUtil; +import com.oceanbase.odc.service.flow.model.BinaryDataResult; +import com.oceanbase.odc.service.flow.model.ByteArrayDataResult; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.iam.auth.AuthorizationFacade; import com.oceanbase.odc.service.iam.model.ChangePasswordReq; @@ -184,6 +191,7 @@ public class UserService { private final List> preUserDeleteHooks = new ArrayList<>(); private static final int FAILED_LOGIN_ATTEMPT_TIMES = 5; private static final long WITHOUT_ROLE_ID = 0L; + private static final String USER_TEMPLATE_FILE_NAME = "user_template.xlsx"; /** * 10 minutes lock if failed login attempt FAILED_LOGIN_ATTEMPT_TIMES times
* 10 * 60 * 1000L @@ -444,6 +452,20 @@ public Set getCurrentUserRoleIds() { return roleEntities.stream().map(RoleEntity::getId).collect(Collectors.toSet()); } + @SkipAuthorize + public BinaryDataResult getBatchImportTemplateFile() throws IOException { + String locale = LocaleContextHolder.getLocale().toLanguageTag().toLowerCase(); + try (InputStream input = this.getClass().getClassLoader().getResourceAsStream( + "template/" + locale + "/" + USER_TEMPLATE_FILE_NAME)) { + if (input == null) { + throw new UnexpectedException(USER_TEMPLATE_FILE_NAME + " is not found"); + } + byte[] buffer = new byte[input.available()]; + IOUtils.read(input, buffer); + return new ByteArrayDataResult(USER_TEMPLATE_FILE_NAME, buffer); + } + } + public Set getCurrentUserResourceRoleIdentifiers() { long currentUserId = authenticationFacade.currentUserId(); long currentOrganizationId = authenticationFacade.currentOrganizationId(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/OrganizationAuthenticationInterceptor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/OrganizationAuthenticationInterceptor.java index 5199830a69..a78f844973 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/OrganizationAuthenticationInterceptor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/OrganizationAuthenticationInterceptor.java @@ -64,7 +64,9 @@ public class OrganizationAuthenticationInterceptor implements HandlerInterceptor "/api/v2/connect/sessions/*/sqls/*/download", "/api/v2/datasource/sessions/*/sqls/*/download", "/api/v2/config/**", - "/api/v2/snippet/builtinSnippets" + "/api/v2/snippet/builtinSnippets", + "/api/v2/datasource/datasources/template", + "/api/v2/iam/users/template" }; @Autowired diff --git a/server/odc-service/src/main/resources/template/en-us/datasource_template.xlsx b/server/odc-service/src/main/resources/template/en-us/datasource_template.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..823cbe91e04e8bb0dd80ae7e494446767873b8bd GIT binary patch literal 9542 zcmeHN1y@|j)@|I~Jg{ZsgTjY8l#!zXM?R?P5pgjM z4j)tX;1MJ`X=$G2V-pFp5a{d2m}LRd9b5X9*~Pa+WGe|RaBu|!c5;@7)i5@O+K|h& z%6LXrdN&o-)-kxp-p``%Ok>SiS<-pyUG6Z*Ku1w{uj0Dt6@$n%ROpP9+nnyxMtztFNtl^J*i#+p2sbx2@Z>N}a*I5RQ+cK;t8|BE^Jr&q5` zR8ZfU*>mwa^eIMlLIkX0;!s?Hj6kg+1Z> za31;NnGBo}XCw2aYM&7s^6z(qswx&dmbFH?PP`PJ`X)9XF2%EYu%3Jv6thQ^DcCV@ zS*N6isq(-3Yc;Z;kLP#}a3SmniXtmp2P@RyNfIcSoOTHwk#;ai2m$aA9yUzB z^TgfW$=b-?-ugH7`Zs4Fz!C<|^51<`Dl5nhFr#-N{Rm@r&vL`TSaM;aI95GEgCA|A zTcIXr^Sjz0dfBe4u^`6;WgqH!HaYBe!-2XDh4sCczAO?G+6&9(j1R_s@^TUi-sq%? zG)M{t3-kE!sNw_(HPacZO)zofRW3I7ms?sYg0Ocb6yjHHGokEOADLZQt(-rRjDayT z-nbiZAEI(x71Y?$@*m@FUXT@`jP-ah9AWfF#UCooc^{~;u`*<^U+Mf z^P_d|yL;q4;tDy^at;BXP?r~BBT{7kbcxP;g&7^HXIn0!4`XyaD;KZEug=VNM~CM( zz%%`K$TX02a|c5K0BPXe4;r`y7&3nbOQou=(he`WZ_mt+Fxs;r$4q++wIT)A1cSHv zNO2|p*|Dp{;Ekoy|E|P_wLALa&R{Bk_HLDo3Wgt>>E_~>g9s6XP@^% zpy{B59FEN(Fty8|F&W3W+_v_J8^X?VZAm*soH)2wy`tTKO%IvjXbxArHuPmaQ$>3k zP)nN_YEk)|{iG$Sb=PDnq9M=Us4cOR^DRK59y zx0{JIa|psa3RTFf5V~O}oe}A0re`TE6WF**urqgT-2mU}nNUNqxgQF%B7wvaH3Fpa)5vTFC4a zOerfE9wbw#u_AWEJi*H@{iry5RU7ytxZJWpY}MtlNHhm(M+>P0=LbC$nh~K zVM>F`-B_nTy*~-0MwM50VbcmvLUz5A+?ii`A{Th(jBd{3HSi^0pSNMyfG%!X_|knU z+s%p56@A1UaJw+%pr2MQdlIqZ_0Z6x`J@4%jd>qyN3vz5_Wnthg30Cbg{UU}J#DN2 znuLg;L<4u8S9}V*lfmarcbADKq{ayw*Ai8`@2c{fev1t{Hn8fm9^3je%qTMyz z4WNzMWIkMS)$=~i)6`VMXO$ADLN(@{S~MtKac|_DqN%S0>*fev={m8^1|H}B;2PEr z?>3a;ojQwk$@c5e(jhtU&R`-v-KRc##%JK1|MYGeoz&Xn8tI?-8qDYcD+K0i7Xko) z@;hIhT|8|}oqt=}JJm*GcR10z;L`nHc&J;~?S2j|lI&Q6giVKrXos3eNu|xgXM~jC z+C^|0H|?e@U4?7gwGJ9JFQOf5_G;EMYm+vK@cCY@hx7ib6N`9Fd5x8djW`zu|Fvd$ zWwcwv38N;9w9OmQj!}?k%WB{MEQ1sA|l^z zu%livz+Q_J;teasK> zV^X0@QY)gZ(U-=Dz$`E3UAnol0xeu-k^Wn=4dHF7#-wqzp3~fUB86+i;@u6}-9 zdR>tRQgO822DUxA(x$*hi{_=}me(l!i(~k-=loUpOSCers>ij)_`=*&dymV_x8wJy zZtB1`thdZl70CUDhE0OV40julq~_I}R06m-fK z2F1BYx8GwRpKcCchKakm1W3<*dz#{_D{22a#kZn?hWMCd2Q>J09F-bWnIDUjUrb?*{t)els45JCKTQYJ1^XGAUh)e9QgQsG<9owVERl+HecY2U|dPB1SYH= z3xX(9Zej*kmLg`2be{!BOq#k(LUb#QQfC5`5c1lZ-6#!GKlGkFX(KN!dvlO9^OHRK z4-tgttBsUru<-p|84GKl(e&T|fXNrX838|)(Z#~l)|Bbzo#i*a9cyYw5piMrFk?>feK8kr>z;(Vm#r z5EVC=8?+l z6OX{QG7!J(&VN|;aji>UZ=$t2kl^(^=<9!@s-UJVe+=M z^{vHqk?+2C?G>tVQ^=kqX~l;`i8>80U#8abTj|GYbVMke&Q8)r|--Pppz%e*Cq zX}qO7IJ}O6+Yd={y#w{b2(up^Lk?0Rf2E+E35ew38d!$3*xQU`?`S}P_ig5}+4RV7 zU_X)DtQ&I>*CCE&*>LqBfZmF+zrWv;)m7=^KJ_WAs8n@Z2W=hXYTi_UUWA+}S6^zz zBAqN{<~FpPlJ77oi(BQXujs3u(*z9jAcdkE!E@elZe3PNI5mnrQ9jDh)>k^PqGb#l z$7>h7(d}w${ZgEm%`kqRH8G$!=J|^4uqM}8m>8v$E zL!zxF)s7l97+3Tkm``iZrX_5EZIoe*IUV6FSWCuXYw&|?rIVBq^2Z(9=*rlhiy1Dx zH|bqsb8G_;R7Cy0Q3mv@Q`A9O!qp`*oa8AVk3QB>okGCxl>F#e zd8#j`2FY&l%<~%mczlJaok;|?rxqFg!TG@~yo^I6mst!NCN#=UWB}2hvGAiR1E>3# z!m&bD7z84)Vm(6UVq$y$%@WoRKGV340G{BW;i_9z*&K!Bdgt;FOCb+vsT>0AO4Y{P zG7CIMo#i|=R*%oVstuM*d5f(3O%t9B;H{AtLzBE-x3FoFyQj+(&HCqy40PF>y`XK~ zf(}8*b&DY={J9g_wIrP;fH{WG1fe{x-B?2EId{?-2EmDx1#DG8y`6!gM2XkSP6Dh? zEHnM>?;@2B7JN%RJBNoXOB;05bSLNDLXP@$Vpk6p$Wf`dROPy#sU5lsxs-QQ%0VR` zN8uaFn-RTs#ec1jDvF(J7NWP5p)u{EF%3FQE6|d3+(Trx#li8|yWW)O+6xm|76~;Y zPTn`hDQ(Q?jIKufm?7zv#P&V|0%XL*_9D(jWvSiK)5_JdTtKRb0T}Jl5W_sf9E7*^ zif=OYL35}*BW#XUsm-X3$cgPRW)Z_$f}*rFkeXWM(*U2{U6U1j8a2Q)IFsl|PVeiiU&|u7Bu`_apCpi|#8wGMJOgG8x@7HA2>EMWO7p3AtY_9%gvE7p*gHyFNV;d^w zyr(Sx&YW*7R4+3_-UO?HW~X9*WJAO2l}5H#@W@JYoVT{FlH3C4^)x~wIUluLg->t>dWDf0$mkz#nah-z-=7yOnoqHn>%`Q+S^Ou6S_tji|`D4;WQU~o`o`}kI#!t900~|?T|#J zSP=5}+A&X>UkG)lZG}KD)HNF^w}9f=G%mQGZw=k%%*=@ZcrOZ?U@WtbN?x0ILPahm zoaPS-HE~ArJhHEhxWQ`7!_g5?Ic7^mMx-K#yl)FDIMQ;^u)DB@J%o_H?7Rc*e zK<9_1$kMRlPQF+!HXtdz^-&-hI#MaGxr);U$a01?iI1iTTLXP|MKL?IK}**g)7m^--L)%2XLjseUVLkEeo{{d53&}Ipr6yHvpJ2rYVcmF`96=S#Nps)lM{I{i z`=LPNEc+LMe8MQQkfj|{pbi{q&3H^2pB}r$^}7)o9jtdDC%{hV9YtaTjVF1>wx_j< zdloB6_34(KsKENOb0+G~%t-VM{t;swUm>TIVK`RgZdA#H0zY%2nmeQIY8kN$LQy@o z^}89<_;|k)bHR+Gw!|!DCwRc}xK;>X8M#@obbak1(j{DL0!5`nvxFo53`LlDsPOm` z9$*AKKikZ#AoejM!)~h6c?M=`#udT1=AxZ}hCa}zG)*kFh#8t4=-%Fq4CaAa7hPZZ zUd3kaYjfbDzwx+}lAgsx;Vy4?nIFVO1Wk)dTOQ#SX-l==M>Jt`Z<{2M>m(~9yV(@e z-p1x&8#7)j<1?Zv%eQjNvg~bB4n{{cU^_E1>Kp#BlG@$PBoCIfBbZSyoZXrwc~H9j z*|ndyfg}a%9WWN3Bd3>^HDVuevI8hr45YX1Am8TV+A<86RPUwin#mpb*UZ==S{Ag_ zmu9#RSbQ-wo2g3)W?nlJJ9BCqboG&14e!O+I|oBAWhK zl?~ef?fmWhfT8EC5d!th+p|c3>{PF3p^87XpF1PwQzn@Wf%X~pL`vcz#jR9?83L^q zXm^V{Ty*nMHR^qXfFRWp|ELe4sJtaA5Y3#~Far*O+;PiuPAXbpWr*9%deaEpJ#$$}w$xkl;AC-#2|7jt`FOP8 z4>X2>#(9Q5GMZ7LN?nfD=bN#z3)QC+{=v(IgV$`>h})ggM$rBI6wH$NBA8nrVo}2` zgTy1k9_7kxn^xkRI(nAOle6VrQ_i|e?wR?zWTiT6h^kR@)>ie{-5k5fFdW{vTTFE) z>{zn{KH6JGcM4UgebXvS%7pT!naO_J6}j}foNo=$P>t#5bo^RVm``R6c?GsWW5CM_ zHn;`Y`*OB0bTTzjadEP=Gyi#{u1pq@gJMB{dlvLezH6rs7H?*7IOq& zFvn*gP?v@W(@<6M^1qhG9a_ zd#(pWk)zIo3+`rMx^W$}OL~g&mw_%m*e0X!!+DsIEuJ>j*t;a7B;)H4Z88L?U8-G@ z3S*_iIzf3!)H$*iXOmUD5?uBH!jH;7Y8F{1?G-0r2C9UhW{m{Ld%2PKh+Lm1jAnT2 z{@GcSh}4SL1dpyA*o?*io6#or#vmtq2WKW@dneO>l~MoaL4#rEllWSx{{{Bj4TVQE zlrxe~ht2WLa&yO{5?CD;K0t>K3~dX*+WlGa31^FiPl)sBFb|hWl&^}EzCjKvuat`@ z2wDsO{S~#;krTnx>s|iE#6?N5SY8K<*E;&9g{FNP9?c~%M>wst@r}lXlhX?isfNb` zxP+(Fy#>J<1S<|?v2MZIjDokY4x%I2T$^pXxjUrzEi@FoFN(RzSCq8T0CdcAp8~E? zXAuU4^w63e{ZWy}G8$fYTZ@&7tvWYjzw>Z9%i$$(?^rWQLuC>5i`-C)V-lO!IPg=4 z1SQDyhQOT0fDfG-JMw$)O3RfY3!9*_w%DY4CNueZ5Tk0aM>?v%HjlC^jPUpOGy%xk zqnQGtMR_^hcwW{VJc+f5ONb^?mar?z$dp8bBoDlwuLgbc+&V2#A%4)T518O4dCI6r znjLfU07)SR;~y@Tp4ClN*01zJD64Op&@fm9bZO_|+Myb?=2{iq$KzRJB)S`IG2C&43m zs<9-HpGvdidPogy@>*>ENch69xVxD;6#L~^b+3jJ`TWvDj!5ZlqXNr5;iPV3<6>Ll z?3*eSZk-Ew-9WjrT@?{s@~b25z!;f|SX>f9L>9SgdZur0O!CML(bU3G3t^OyKexn; zqIYnlzjahcp~QB!@a%Y6X*^2gp?l zgbT%RAJAM3XaAUefx`4f=GlWuY&K}4!695w>zQh;G0OoVByNSDvD_5}L+(dFe={14 z>tIxLHJ9eXW{Za4H9y`xeJhEQ?$ngoT~>CFtFp?)LD_eG@C5yCQ$av7fCcNHzgYSA zH2(Yf58tnV(oRBg(H? ze%*)sk>!Z!_c!=;SMqCyziRCt@c@7f2>|dn<^47MuS?^f!&(1) zGn1M33%;+q*R9p--g9o>sv01CYR zQ)efzr4!g#!~3PBn<1B%qXSJIB0NhD03LSzf7^fX2$UxdD|PeW$Xq?Yd%DG`uv{&P zB6tYt$753!@90VBD>c{8vbBB4inzs*&Ly-Hs=^yze#Lh>YE$jxSPu#BYf;093L5Cp zG$I$`?(K&%ej+A~ch%E9&LJR|ubU&)hiJhSnB+uq_v7GJoJ7FXLAyN+Rm2smC?6NPI<&Ax>w6Y9!7;Kv^wgnW@=5Y zH_4!OHhhGNv zAHRZ5Z0iRfohI!E@cjS+01poc0F}SNvQCqS{u~x-FJP!chr!a=)zZO@o9pNCzi|8y z#^7Hby*yq~xtj+g^!WK**ud4y%3B<1kf+r17FrGeK>20tx|nA5$V+6%()2As}(pM&h+jPLhUXaI2nXQQJ1X380w!O2@+3C{stS7W*A7w z04Q)?4&1-v#M9Z;-ptwA{wI6=ff+a$g~7c1XSZ@SMNkh9P8<4U})g?{OuFwSjS_HLn|P)4GlDDtXA6(O;49Df!Ppm>d9}UsPa!Ry#bSZ)3of@oidjw zXRUWuir!=Po%uHy+Z1IWs!~BQ-%v13*nk{$0BgM4c3x_W=J7gM`gVx*)6yBo@cFUT z)?ok48f>Qj88UTv0D%0G<(urO1 z92}Brh!N=&2(o#cwDEhY8c>?ODi)7L#|BCtmI-}lgixY+i5lR1K$YBrYrGFhD>zg! z%~zTF)79!#$vyFK>9kXd4Vn>hIF$6cx%*H}UKY~c48x>YfnfYBDW~@+Jb0MweB;aQ z@g=c=fw7B!1)SsxB1^LCk@<3gNSRGd|86;OI58?xK_`;bX$LY{K1iCN8KKzy(nThU ztOCck3LTiO6qVTf#F1TVm@wRg@tx$S!6@&ke9e8XmxP3KhX%^tg2>@zc^q4g$dR>d z7ueQ<#OI|UACr3~li=IA?!w`M7qStHD)LL|2wT_Hn~*2GWPI5D%}k`T3QU?ZgIC@< zaLg;_+BKRoE?ZZ-3h5jnknpyX#st=iR$RYrwaybEcVZHq&n9AgY%H&x|Lr?S@4dpi z=j^uh{kpWeL^*IA-yvcPIEPOhqVURlPEe$-P3U*=b1fYdmeiCa2gIHTHy=%a6PZ!W zH58ca*mQgBQAG{Z+6M&gSdtE%ZF2$93en z=ZpBg2=lO;frwBa8}Dx6NZ1!ThZFtsK5+fmreo;6i=;1r9j^*<^j4iFHAWnu<3e;_;vRKs+KVQZVfqG0&9V(1wmU}F z8gVsw0jysPnN=Bq2U}xd4!==Urq+q82z$CTEoR2hEa)1ox~H4ZK;apELa(s0jW*q^ zThmDjRB6_&_qbf0pRUj};R8Dw;qO@c|jY_aOUFsFm1bsa( z8Oc+9p}wU2rB$50g{1d#9o|rkzEfeahTrsX=YvG$Kqb=MkBWSq9gTsAoPEL8GNlE9 z0!$P;LYx!77mvWL6CVTfG6Pc__yo>*z>X^sQsxMFa^}q5G-X}c;@===FRujht6%SE z`Yl!X2{Ik5jP8Kzsw$@vR=+8mzq`7adf4#-P9{6Z?QZ$X_3v$;Zh6VA<)3by(d|10 z2SgKj&Zw9d(=N}Qd~yC)DC1?7T8P7-ynqS-VEzteH?X&ZrQ6R{zgBDX%@hw#JN+qE z((%Hb0ITCW3z>acY50)ZsGb?cjM*-UxXns693ieJsLV&~Tu(3|{U#X&jzmQB)R&(Z z)o;_CchX6P#GMf3b~WZ+3?kmjQORoTl$yA{`=V`~%HLV?=owK6yF7ktp`f@DwRpIF z&GtE1@x2s)_5kri%zLX!{at&u=Y9ojZ0pHtwo(`l7ef{7it?&UAV^ElZj7N+N)Kh% zi>*00+-IK_G?Xfz$zbU7jg8dzc0J#!DQ15!pzE5mcuhN~x*6*pNY>`|Sufxly-uZO zg@PMmVY2`iW3;~a34QnvgfV+|4RlaisL33IbKRdP3rkcEZzJX(b{7H#W zfy+HnCO*~~a9(lKz@;z_?j4PMd);$78_$i~-PpBHzIFBSD18l?buINw&A__H1*$gK zb$VCYNWmle`S!YC1FXv8E2y>Y!ZPp3qk_9o)B4F-9Ij2@?3<1vDhyi;LA#P9QU{e! z;!9d@y-*LQEz88NVyD;@>-}*sxf(0F1&r7XC2xgv~IHBXu} z`XnQYT{;I8 zVSU(I6EmYreGF+RGUb|8sA+gBmX29Q@WajH2Dx|mTp z3w_{sJRoQDnb1jIv@`xOFt&jh(9(E)x{uH*;D?&O9qQ6;p*>M=c_jp>vB6UOxIb|L$D8aYfZ@EA`X<>EOv)#aZts}vlnzXVk@v0Q>+C?ysu@+ zNAd2CEPmKD3lG9uZ;Uc43FpP}$Lydpqh*f}M>%{&T1~hL_PoaW@bb0+E)+sNW{2fm zcNBa$m`&w?O64GWb4Y-+`1EMX1So~-m&xL7Ib;}}-nxk+1j$tSd?rJaT zL*4F;|J9NH+2jb1lt{wk3J)~Q3K~3z2fUCL^)9hHW>5euk@_cR=6!s?HbMpfMrnRZ z1ix`6*v8V)lKZzK?@zcvb@ijjg$Vq)&m}Ov9bG@#9L94#6WGN#h zm0yiP4T=|kC!PB?sg*#%_(nwmol~F|`<{CxequOwFU|@YH|tuN9-p*gNECaQnV;|W zMm$rb`Fbof#YZj$y(>P+xLxH;EF*e8g<&cwz)hG%z*qTsVl=KLs?&FLjyuo#HU9&; z#dzE|Rmf)%^U|CW@viln9Fe#f^e#c8FX7h|Nn+zdB)csaEc3iWtp8BW^} zdvh)qEu>}VZ|^6$1AH$|8nDGfB?4ZF$Ypmuiy*KwdG@_M4Nf0I_E?;QJwzv*YkL(p zkCVEHyk?WOo!&2&j&?mN$@sH`zss!|uA`;2hSOWYE#M7v!Z2cA7@-(*HdQ$jCWNP? zPWwhMaLC=@)9Qx{dv-svKv{YnTf3-=8l!z6=`5g-nzH%Tg;AZuI_CY1VER(-P2CAu z;#ch1lVB<0R-1J}mfkS?>yJ~=bBBAf#0`7L`^#OAp=w-BN^nMG^MPgl1cG-;xZyg&6w4xU;Y;9r;Ip~F9N3hLRXTd` za(yUHz?OK@^kDd?LCbRRFvr4f=>FoOI3VCO=xP_+_{YVU8t~UCFVl@|-`mNA>;S(s zA03yY$o}krhpql&YKgn8SZ6BX+6EC(p7p1A=Ie&N{i~Qn-SCu`n+QM5@VZfOLGUs; zOS#S5^ypshK_%#Oo%QI>E+(YJC!g=?Ef4&AcH@Pu+VOgcUdHk+nlA4LvfFWWcXxa9 zxhs$DG93Y>Qv$Cf-8fu_9j!HB?`*iR53ja4voHSHA4PuWMfxj0sqHi%h;Ac zkzh!F#kHnFreb2XTWQBa$Srn~pqPMirMa;DtjIxYsmKSry9Ynb8e8t{IlkSx5pT{- zpIaZ!x)(<6l9mac-r#ZO)%mkj9EGcG$cABVi#YtMO&;;k2 zP+ZcfaKbSz>7kqrup+nG$wWz}$cL?!6n}#Eb9Zxor1Ji(U$J*qdK@E%kx<4tGfiaw51$`kO++;F@0f0uIf&rYK$dKkYyEOw2-PZ3D%iZ z`I?-oC+o6}#^Xpx=(TH0f5iuQ@ernoqgOtcl!}|7*+~uDuLh zabpk@llnx@W2f(Rb|}g0^aA$Z@%Ko&`@Mio^wavpdFqXRr1Du#*U6?yq{P)>&k{uq z5|oC3ym2Y75_>p}IRoBd3QSVuIkIa9;;iayrb)6pr@7#O9}jr(b<*30u8^o^DNzTl zt5?4CFY7g?DrZ=r=xU>miETp=-4((q(9z*$M}89#7!~aLb(A+K{Uc%A_IJ;(CPOxJ zt&yKa65r5^MyYt@rC4Nj2}V?oOe(WelPE8wb><#$JxM7=xO}I)Vog0W=8ai-BzXU$eJ3} zrCNRt6b$9*N7s(-vch7g)yxtDnPYnHk{kyLnb$Wv%dIn3$PGbHd|n0!7pv{5bY%+4 zshk%Iy=8UmAZntoA;T5-c2m;(v7$DQ)%@A0=e{Nyj zY%DFoZrs0hzj1O#!r+^wpZXBe4Ho3+O{);<1_l$kXhYtCZrr|3xJ6TZb?#nEvIkQt z96ZKnxYmi<>-&OdN>p$3;>2{A71bwfaMa*qcnTdJA>E()5n`~qWC<~97X?AGIwr&R z`>_*(tMs)IydWWcgteFLl>9r-Fw(>Ct1`R~Ade3_6lICG+wCBpQ8;KU znwf~F+*lT=Q-XFw5~Gc4IVHa)A;Czo%%1Mm{sV{>f;=70vcKr&I8v{F9a4!CQE5DZ z+jQcJ?WR(i!(Tpqe5jrC!MLOghR znySqknLyT5Ks+L8<*ljnskxMkI@azN1`3P>;TL(r2Sta|f)rMa9%JNwxL!<|r2G5` zYtB3>`UPQyBCbgia}J(d%sILH>x%r0ce@yZv!bgSVqj!8|x2#=U1v1S}G?xHNT= z1r&4_(=GTIIxel?wnQz>wKmu`o{{<3*XWh?BWh%x_~}zZUF1*?$Iho?d-jP)LMlPu z-`Wt893^|RKW{~?Sl$n}luN8IIgSiNu40uA?6&}KdYlOSxt4xYpc~dcZyDL z=m6rL0NeASXhvb_q8J+-x`i2j8&+y#o1SlhQb~w)#M%doQPo}tVjr^cxX1Fq`ZDfQ z8vC*Qvv66=oL%wwN@uzFiwaq{55iZ@>hG>PlD_Xb93fo}S)}Te zr@iz5&n>DXR8J6Bn93a}t>w1yER8_vvsx7^=`X;~S zPw3$0JDjymdDo>_fm9(5yi~nLT_KaVK2&s>dcv(w4Az2hF$F>*Q1ylLJ7zyrW`w6L zGTNGIj9$$uNSL8j??h`b%W<*V#B5tI5@Q5Mho>>US`r{nmuBD*8!zB==old|wubuG zc;dCG2jFlJ;E9&4ZdZxgK(Bf!QqNsV`b^F<%1B)L&udV0j-l2M=~dWt#|EK{(ac#y zBH#?XcrMlX3*)&PqgattF;)M3N*3N|RXFoD*>*<8id`K24T zxwEU~&wT@|mhoR*78WhO@v6$fJOr=T6z{MLj#bdcMu0kbR?1vyjWV11Q%3c&HuCzG zw^u~-IQ$n?V!QrgyG09T#@e0CBqaN7+AwxmKE7e?WUHW&L;h%^ zYG7p1nrlGr#n%kC$uQrMkXYC{I62l7YUc1A%g&1aSjC8=?tSVvxtdDcNQfKFZwm0(w*gzzuU9}v`+jFEjTH_8hi3$#!19wl;K z)4VxqdzWA3xKyHeOCm1O`JpUqx806Kvwv8Q``^z&;pL-wf_F*Hysk?1K!R z^h`R|n9Ww)ZGccWE>GqrnkGL@6O(5l;9i%2Sv=H?KDXRn;1SAaFwoMPl6}~;SE$^B z?o`}d18qCPr%`bnMbVtTY|j*@xe8ts^#MOeTaGIbnj!SqE)S__&VI4v z@x=6rFZT9la*^8uaVRea=aNtoQ{I2DC4DiT>?CxX6jbLkS3gEZ<6qcb&lF0)HdNWE zqf9%qaGNPryj82nyGu4|SX(>S6hEa=fhlZohGG~5D%nz(GNe5}&<~1{FMC5oNruJ? zx@6}*(Xhy-HO1Bn$I3%e#+Yx28N_LMq&kf)5peh8;&A-x6{9Y-{ZnDb%)ZNtENn~) z3ZY9NJu-TK2ssFAs@M)G&IpjD9E2Q->)E3_7cMZCafZpgBmdyV{d5YlR`W7kTFF3&r1_;l^Q-Z%BJiKa%^1Iv@c&7|zncCkYyD|TkMo=9 zuOip44t`C<|8!7I{C|J^A8GloUVcp+{`B&k{P!pLHI4Yy!{1r+Vc{f5A+> zwW`)#@9wkiy>)h-Q~Q*XEI0%P2owkm2nYxXh|yW5g&rsfNDKrB2s#K1n2xBuowKQ( zv%adwM^h(VMt55q;yeg2>KqU-;Qjw=|HWURJYh_sj~PY$Ug}wNk3nXwRtSdUG;j!w z_Ps!NfBaynu}+qy*a(+t(|RSVCZ0*GD1YaaJQNs9v9QV z&kvviOYho(MdcCl??nR0*yCYFHzZsy96I_lbjSdoaeDuFLy_w`Ui1RK$(aV=E`7g+7B$ zmQVmm;LH+;l66EqRt2lwluy4P=2y9~i(caOCyH%!{QgZmlh&;07!w5?Qd|11f}t!i zMIJ8NK;NR_k!J30#ZzN^Z@a>2=sgJWhc}En)E}+a-6AL>-6Z5bA*WlzKX|=PzU}LL zI=f8z;m7(41p@N+1`eY1H(EBRF_YZ@eN7I?I(Q&0^&L%ZoR}DY>;Ff`|6&gQ<<-mM z6j$pP`=%d>NR;=-^8Z-S}eT_{l%L>A!AsXlpXDX zbxLxGBInjmyPo}WEYqW(3%0Lp_+lh5f~a@{7@_`?Bmn{m$=ARy(h4LA00aiq-G=E8 zp19dNS{vEhTmKfl{>>RsU8_Zl%g#)sCu+yYFeCLe ztCS>czBikAw5__D3$je$_Q4+K6GN`|97sFh=(jy|#o=fWp6E8`ypZ-2*Aw6{MyC~o zfs&BuXeURK#;epa;?PH7=|)AEF1y^x+%{-9wM~9n4kxvzffd zq;=2NM}&R+GFie>4u0=oXX21yNg_X*IH&!*)Hb#AZD*0EQJSvREBdjUbF;mXq4`bV zO#i3Le2^EV`3w#Mk_^m#kb!pqUFNUOQm$sJxcUy~Mc~J42=(TtUH0=lIoo(EDQ1$x zEiy-fWHEAw%F}O-)2~lHC!CNAmPboUmtG2!9$DAiMcG{Pw)`w4bQ@8&-a9K;)-+6| zcZN28I^&s3jrpXu!d4r#H811&FF#MR6^pDH36nDfwL!j(*67mC7r;ns*9H%xZ`5G9 zs()Y?>rv$+UWS!`(_$@519(*8Qlo;d z4<-a(M7dmYL)w>zMj0!uoT~$Es0c1RKxV);(fV|Ph*bW?Umxae>GL4o280fDl@%>* z^B$cG*f$H=M*|J{ z{d|r6NlJjH&C^!crwZ9xT-WOKJGD!h-GTKNp4PWq+`CTeo=1`PiY4t%fQ(|rgzh|+ zw=D)g@-8)mk9D1c24*S#D%s-oy|upPtrD0HK4c{$!M3Hk@e|~b@7_L%R8(HKG}@-y z4s$k-R@~GxUs^vpJ|5>GB7xH>7{jbi1x+HJ52kVRri(q8jSlJjIm!VfCoDeU*Ze{y zKr0)xF?EJmwgH|CN#c*5#Kgjbws@GCt>&N9AD_7|GnR-vCda4R`9iPgv(wfNrz9S^ z7!_6Ue_Zg-(94p}iJ&~f=+{^`risW4;eR3CdP+KB4=p-_p?-AYbMwwwiFDMv4>jg;2|77eZMt3MdAY(gV zK|m1yV62m~hmEPzZ;N}o`bg9+CrSr&iXXANhIP%}QgFUR+d3Fj3Iu2?_;_Lxbvh0s zm^jxStmBwzCwb8tbjzM~;D~uX^(fFK^vqhMOv1cxOZ6~IZraiD*Ok{I)S_OSX!wiEtt=He;`1kDCPgmp3i*w z=}_S$)0b5F#M*PL+?!j`nnFH8rXilPFTN+H=+xR-mU|e_P*72JAW;HJHIV%C0%=%| z-9TH{M*Gtfk#$*^f%kd2-D=Vy)|{(U{1W3)17RZ^RBDsNX6?rh3Y7<&OH(VgKYN*< zVn!u{mnBz4nj^1`kI2$JnfGYsigUHGn1%Zu%r=E~DC*Ma_VSDM}<@-2?yP+#&@;4D*1H>;gg8{-IZQ|v#lG(L}IT|e}3+P$Oz7}xI z5K@YEi)?*FML63Ux(*R@b@rEXb%)~Sp|q}Qj^@b!Dk?7JZ}$-DN6i!S@k*ZTva zkLd`e;S0Qji1C#cwmO7vAZGKK0G8?l|C*)`n|YE<-=Z-qMxxTwcZU$N2L-3YPa*b9 zG;iVC)Cxm}FD##Xv#_`49&<81rhB!J?CWRg`?_=){4%O4!-vx$$5OEsEp{g2_eW)S z#`dk$#y?RvKl?Z@`u51Lz5#@GLoW07KN202ECoE@jwEeQ_fMZI%H;5W7l@OulIpeK_>!z49UIw9rPv+|E{L$cZZ|>&-v_ZTM_B)kn!Pbliupwj zp2Mq!;LHwH(@Ox}@wlK9dW%|8m`HgQUS~?MUTo}GgH-ZS) zXAhjaV`%o59IGZ=uv%0F(b_t9)MdE1(i>l31L6b{MRH@3+A(DGAC+X_8QANQUzpb8 zzKum6#+sePE;v@E$0e=nVnsh^e*NkcC6LL}`Y@T9;whN|-y4^t->G!PpAor~LOz@1 z=fq9T?yV@55Q%CEYxfhL{@JZ@)AyKkB@Q*J2Kp*&NrXW#&aqLAJ{%Q+%pqXnBiN=q zPIPRLqyR${&pW6N76x^h0&Nxv^2-j0ftBL zyBLl=s7I>!F^ItsOn$0t!g5N)K<45GjYkguQI`*08&$W~bUpxo2{KJftB8s^y&q7x z3m^qV1g&1TdJQ()h%fUT>8rVq4HtlfBjkmPPr}&k7TX-u10mKA)w3rzHZOpL9c$|c z&ysccqaZgnvZGmz)>khr?hRnqptouB0wT6khR^hO+dxak&rfmnI zL6n5T$))*PQnq0M!mMtn6wn#&Z6*NWoe8C02!v$X0^t(4w|V;_x3@)pewP9F z2gv%bw_A12N3-sRJK5e()A8AUK53qs4rk#**?w<(L+3<-&wJ7KgxvK_JnxvdMbV76 zbq9tv5V88e2<~>lUyacEU{Gbj#B)}2Tban<-CY8T;TL-v;q4s^aIr6Do*PY%eftmM zxXn7z2CzOxv#c1d{qU!=V(jbd^I&yRoIIdBgA$RfY-yve0b9$S^4ANKHRb9}N?oLp zAy3dV`H4l`}zZbZC7s{>6N)D|~x-Y^@9^6d-0ZOEpVe@4D z>Rn{3`g)%vCt4$99@1_|ind*Q(08-}#zavJmD%k!DG5fhdu(r$oktOo{Gt@rD#1aK zW|K+>%_`Izx^CvP>ho!F8?qMi5XQ{5P!{xMVlBGQ{Yg=gPKgHd_t{LY4>u_6LJyp{b~2g9!ClP5!ioL;ApX}p_<^Ay#_(p4&o%>91gj&5$7 z0N94bAUMw4DfN2%hXxRHRPS+s9G2Z^Y|7iagqf1}%%~KB((X%zXwM@ovYc9L$xaP;suvaywT)auIYcZ7Y`rPdJId zF_bgId+&ntULQ#WBg-sEZ#h+S+F5g2=_ol@Tf$)49j+F50})xpEc#j=!NGM|Ag(z)&n^9*w!_BK85 zMADPiU~6i~9IIlBQ45|U+tHUrRBLh4qUHcfN|kT@ymmhutYDJq$k?raq95Y)4Y@@{t=%Xk0w|G(yK9J_kX$_8H`#iwujH~Bt((Di%@dSOEJmn&ZQ4}cj z-b03Ocg`mYyoZ?~dz@8Xt6iZlysqy3YCRi041(eu=Yy?_1ULU>EtTMK=A^cZ5dE=C z=ELLnwf1>4=%xTZ&yRlGMaqXty_p=6N;d^uF*LT_SS@6A08{}FCk5@-b>$_b*2HQZ zYK~rCl47U=5i%PM_*3Tcq$k6cxntF{o*vS#&G>~)ci_#vIT0skdeK$un?qwdY4BuN z*r}b_OjdojFFq|htzAT==^mS8LhszH6NT$mPUheyth|(9JToP7mn&T0I<%WM@NQZS zzw$I6cu4~A>&w=27VI&s_U)Qb=43{nCF+!rX%k(Q?71F=pY+rRyGKfb*kzCp`f1|mRJ-U^ub;`|Ygf7_}$S(utSJ2Cw}{2r4t;9k)woFmT*_%u=MXXWAd=P;1$|TS>z7vm-H=iiHo{g1D=Y;`i#aI- zhgg7T70a(FeTsFtjU3&hlhV>@V(%3&(Gh>Z8au&@Y9ww|&v_T+IPmxTn;`8rHCUXc z7Cta=4~)Am27f5SLbWFO%?1%2ppD@zsaQCfI z1<@Ptuf59#!2;o!-e7(-h=E6yGU~e2d?O|>6Yk}_WnJT<# z=!Qts7vT{_LN42}!2uGQ_kMoPO&ajI* z7ac#A?O1YQWJwc?NDm%NA-|wyG_4;yb zxyt@-V9G67P~e5D5%x^}tPXWA4&*b&W02vJ3!)AeOmGaOdy1z5&SO~J80Scl@w-~{ zr@El&iP-ak*^rVmZ$sWt0GpF>Wl{m|mJI#)a~>6FwQ37}V@=XT!!hWwkQGC8Qz?o! zMXYv}=O2Yy%usFVf**B%Vu9YBf9$->al)I`PE z(bCTR_kB086eI)Af)RWlc#cxrgexva9?wWC;s$vO8k}0s_Dy(AKlU2gp@+OjL8mj42)zrSI3oMwr-kcI6Wc zqQv<_U{#~IE{2(KDO7cAnlP|(NwC<#%9M04u!8wDRE~VP;~5Da0IwPPSvWLlnGT_# zgwf`3EPQj=0lj|W3y&|-PUoZ$99XvV7Tflp3{X+&!+eiQ76>Hbw#eC$Sg&MhsGTSy z0;ENVkHb)n+q;+TNl;KDeP_^n7sW;W*2-LGq4L@^mS4NE9M@Euyf^{N9)i{a=6#JU z)9bIQN{^=bjL|3Wt=vne2v6JCOQyf6R^?W?+4hU+`j6}l+9Q{8q#%uL9AEn+xCG~D z)*g8ll_1&EXYYAB%nur_@#7M;zef9y6Nz;GqqMDme}$?E{KpO`ZKDFEZ4-NAB}e;@ zPE5x3j;8-A%KlH$270G=+IEWcYx6nuqO&K%>-oquWYg@>7zYEt57Ct~KEgW|HX z+|KmN{@nDE%X+!&5DtmpK-F=`L5)@UO*I{8@o<4bFWCJcmNiAWL*do5u^J8N^kBfm zWtj1j*+iy($bbgio~`~(n@JWW`PijBL-!ElEIjeom?(K&q)}NoU-3ycJTfw7dZ{fzsm)> zBLJAGp#0Sx4L^STpEm;C@Lw$>4j9mWtI2`iAqAg($*`ji7gA@*$Me*z0g?7PFnNbX zDI44m7|AkCCV;v@bX{33?^|57-)GHXZfw{t)heq^-RT;7*rbMZN4$}u*VE4SWP zCp3K`B;F~O0!%*;o)Erfh@A1}Si=faG$NM1Fp|NWR!n}5F<-855NamUbV>%(2)y5N zZB$8ZVat@+4bd~M$itV?LGn+BK$o=68MmIpVcuz%Gm*nPHC*X^8 zSrirl0FFiWj*jU<)g+t55LrDGDGyQ+VX5iM2ud60`#EGmzvtK6)2Vwe3N0dQQEuDJ z!MlnqWJG*?t~)3)X!xNZJXxgKA}h#PJ&-KL0O(*;w|=d~Q1;1;D@3O6(r@leqO*aU zbss|owBIP!>(f8N21T#(F_yX@qRPJd_cbCzx(q}_R&r@AY&B^L-0@*Q(lrw(>P}6W z{Y=m3a#2>f`cZtV51gPsv}I5*24GnG=i@K`p2mNV|8N*aN%rps{$8m6H}JPH8OW8t zl^TxPGZ`{R;k7T>lfCi||L}|39hySLm;5 z;h#`=l;5GhYKOlX_;r8tr-3x=|9kNN*zNpk<=1TGPb+Bnf4sr3xy!E>{vK-o!~
    aJd0`&8GtXWxD9Ii;ou2agLt1Rw(d04jj_X^yQ43;^&N9ss}rAj29y zb#eBva`rIQ@_laQZp`lE z^PPkaVl%6Yee6pfsIV~1wX=K7jlKg)7U0XCJt%eI*DalZo+Fz6+&t! z){Wx9Tq#u{0p`%4(xKrN!H=pZ7Nh~rC6h=y@X|MrxL2UVw)2mYz~PTnw1JT)8$*X8 z{>Pto4Z}{qrX2=zKOh1CkB@KwwZGA_QJa(I0vc;dP}ZSAX=&=1A{`34lI{p`P z@Gp;Em87iN%ZVOwCU+k>bUnNJ8Yrpg4VG)C(h7PZzk<~mS4c~;+QmQ$)FKXplMU_+ zx*u6y6^;8mM0K^rTNR6qD?-!cT^*5r>*|TbOz)N^<65=Z2lAS|oW07FRq$o>>P%p+ zY$+>H9A2l9nf@yE4hUh_B}Tz1CXK*9Z}aTkpDRuk`+9k zR&t0T9wnr7Je7$zkm;aqmJl*M4f`{6jAi6FR)pO3>K zo1avZhsq-0U>|=zsyso%eB+MODwZ_Fnu{yAcf&|W6j@k8Ep^d46~Svi!Rg6u@BWEw z1R67=4LdJx!!?hqLmQf#Dc>+z?>@FZUA{uh3-zU#JSRet7sc=eOdiZovD|b^Um<_% zxVKjJop9(byhYz7tyCnh;Cm7f;XxTWBuf#@l;pnqHnUy(Y|}&XZiK08`5Wu##hLZa z@Zjt^bf*6$WFkaD0^k6E3TXC&1-$|qGJg%0D(y3uEK%UYgTfsJaJ`(XD=9w-1@@qN zMnyZTk%9#&`xzc5Pl>$tN$@(64hiY0+6$O)NiQtmz2j7Rou5&hm1eOU3K0w*B1V@o$K=@KsKAs)K z=cz+oRcl#Pc=1Ah6DVcJ30z*KV3}DG$=qWfDDkDAi7=qknkde9PP-uuE-IfcQKE}Y zZ#v#q+Y8l5y3b;skf@gUh2FcQ;eB$g`HHcwg&CvQ+9+mcb(JUZ0Z(qsa1b3tQIuq5 zwOMgGb{%{XKiN_S;)?Tw2F^6qRWZN^-#{jsyg7(u2Cb#yJUJPt4&zM zXwSlIPeZbbA&~j1BJVH{zA0W)xngGR&j#`anLA2TqMsP*b_{QwpL!_Iyu+?q$ll0| z-$N2IMPn;&e zg4|g!vtA`po!8iQ7GdbQY~Io{yTYJ9A6`2FBoI1{6%pJH$2+M{ey8HLOd#x$<8;vE@!@7fofVCG zN8|M|$V5xPgDepro(~>gbUi6<#m;g`Vp87KUeL#ySbaw*f~2%br28wit3MZprS{Dc z)|!gujJR_2alX6u^p%5Y++omlT`M{Nm8ajl|ATTBbA9lwcYHH2P+rwTr_-Rhy2b!;#X=jV~yi881mePkZlCa?ov3;}3*>gSOiQ6?AKmm496oyk4 zc2SJb!Drb;PWbo>6jxJAns-?)ds##RV$Sfg`&#o#!|->qdAiH^ zrSQX93$A~mOEhlVn_M0i z3YBdQdCfv2ry}_;WdcR6Kk##KFux%#C~q6N7AL}_Qp$g5l%uxw-n!dQSnmpGY)V2N zXwGSDZ=!EQY!oR{x5a8SxYk7o8#63)(DyLHtSP-dwy|4W;Tq#Fx{okxnw$f&ZwKVv zek>tJw?pT%FH0kGRO=F3)_LuNax!C8`Q$obnnk%O2#CSnQr*jI!fY&YcPe$=?QWxB zp`r+9H|f~ev@WClrB!P{BD&PIdq_dPr`o)ofvci%Mm@vhg z%?shg6EaE-?SMOTnlvq&laBhhS$&GL(B={|_Bn<6=GRZMFe-5$+`UeLcR!MmDYbQ3 z9u7_5(n?4LpY{3V&x7>U^gzMl9L@C^jaP76! z{nTW2EdZ#u#Z>;hNMv@5w$OC8hIU~OR(!v?=lEz5igmQ#@Qef|`bf3Ya(|=&h$2>% zx2V6BP>r>eVZ%4(dm}?Kj(vY>`C!{7I1FpEHO`E1;j-SK_K!#;<&%!OZUFo;g zcPtYrSO+%mX-D3|Tpy&>HtydBU7s3$n;hc=3nhzGd!b@fQ(`+lVu!bD_B^>~Pz;8q zN&i&Dw{?$LCP)ClIOWd*z;BA^VQb}N#qryj>nGig^$lZ51aN~mF2vE#PjC3QN3mVL z@-15mpy)FcCm0xBF;)^0$*;wsgd~ZjN*26M>%dhoy;V~{W8-bW`oXc9^l3EVAkq3b zan7wKD=BT&7(d}Yr?AjHUMxqb?Pek;!%sE?ttTnX^n=>BC)uwSGH9pMg53ofc>`4C zQeJ_qP@KP`vEF+(tp^=ZFC~HE-ywaAUXWxHPjYM0W{m-%)3}C=KZjjcCQL{Smlb1+ z7ZOJ7Ogm|#OkZfL$R5Y&rGpX+% z-oO}!5{rnjvV`kJvG1;d-m+1YkkoHeeV_?Upr+c4O*5Sm4|2UT2RT_uYB|5=+d;f# zNFIeBh{SuskVjrcj}gjQ)~I{S_hQ7;sB5kE(t#zA_=OCOo}GPcb-l?UBGDY6n1Za$ z|I(z*11Rb`Ue2o2(4mWkv#}|%2h*P#49B%x|SJ6JF z1!2T6%45!dDz4KxVb^UBRU%wCDC8a@5CJmwfTVsJCpd(Cw@U^r+_|Ql8`z zx^gG8J#yP#VhnA|Ca9b~`nWxoCS!?Q3f{=qd2 z{9ahHt8KUkbL?JZkRq&f{&GPZ2MwB!XGj^^e0LL?i|aEYf-h6|O;$%iefvp*)*rC@ z@t-GfEt#zxzF@Iu@9pjN<@Qva*rz{5l$5P$ZD)K3yOIZaVG^xqCD4G*FVY>ogH$6RKQvE%sG7O3;{_7DC}{do18@T)eR z)m~YC>?VY_nA?#V2F@Mfd)WQ#sZVh=r#IW>WY}r02?8v)ZY3p)N;9}?#Rnu?ENfi# zYC#t)A30C!&L*WDXl>XxNsINB3F_7c}mA=qm_=^w}2YBzVn$L-PakNPiJ`g zVCYDC17n}DtU%~Pvm|Ot^3GT; z=|S69@4U~ABN52>>jQF00)=387Q1Wu+Fl(SX+u!(7p6VOg{4oi&5Do3a^GWI9kZ!` zAh2{?TUorOFUt(e(Mq$E6>29Z3s5=~@=?PBT!nYv0LWzAt_AEm$*GQxH6T4XwP?=$ zXTDd2$D_*}Z5&|4t~zw!o%@}2R2d&Qm-8t+LPV@HxDPd${q2M{8^8C6^09JOBn&D| zrSU!CRjsiQl%Eqi>ZlN^ zwZDH1)ULPV$eZWhZyfVw+xENjV{240`5~|d5_UMC1Gi?NK#@++^a2AJ*IiUjCpLh2oS_7ptRmxSclrZu znUOQxs;%a&ByK#P+Zk9$w@IF#1Mc61=WdxBcF~dKlkIj@VI2D6;lJu<;i|z~Hn%?#s2^N<( zu0RiwAs_B;8oP*9LiJn~ijYmsn&&~4{iftqw2P!YofL5iop8eY0>C0YJq{M6_~;k0 zVQ!ztxk9q)@Dg{wdw+g5VmsdvGbNM~Pa_x@K%rm4$*(b@5Gi!4bP? zMg`nes_Lo@#q5YXqTQZNNk8-F8IR<{Bat19%5P6+-ZK|F|4?Cbt8rj&#l)WVRS?~t z35ybjv&F$Vw z$E-C{bBL(l^I*YJjRUow96njKi(-M-Oimy1TWRWvL1Mn{Dh3a$8Vi_hsda{od_6%_ zWr)R+G>*DxC!AGjcV?|KM_Q-d-PDCGq$SN)a4kLgvB#$-3AL;1gQJFT&}i@pGC$;T z*!N!k2yESI>!PU0@?EEq5cYOR1=lYf&!CRm`>P@Q<;WB)R(qm$8Z@twT(p@L3bpL} z%Mz0|RIcXFx!~IGIyVDn6h`i4>eaBAQ@zw&1a851COQKjM>KDT0$@&Oy_}#mzkiYA z;zo6wMCdOhP)R2IU5Po_6c z;GhS+JBuy-!8b|rs}kVck4bln`>NB0{33>Mf>MohHzEK$tgwZKuayI$=4yTvkVE_D&QlF_xus!NGK6$-SxVvr5zx={Vv^1)?RmW z`v#3s3&>0>npkwXTi>^vCL^3^Z^d4sY_AP(;BAuSS>4{GCSkAGRv7l%WQPH0iX#!hEYIZ{}wvU0Zh(R(A}0ls3BKua3XtQl~JjeZzTOZwa~? z<;}b~XKgu8uM`{^I5Zbk-3qqkXG1uBzCXgFhPOjCHcwShZ?`Vq1y6dC^=^M*Peg75 zQ;mk!WImUq+IzK679ByZ;YWDK2=#d*~()Kgn_(Dp| z!UmD&s>TpAvjCSI9h(q>sgH6dgf~HvAqr=lYL&G#caOrhE{8-v0|PqukB{$}Cghaa zT;*Bi8|s-xICL4`YjbO9FtW>yu^_&$8cR8LOah0-Q>q)%SaN?FJ2VOVXp8_MCYFcn^?Box5#mr>AkIOzGNm*= zXRdaOjPM-pveS<6S*76RlZk9TCwR6kvjW0FAn?V0noEtPU`LCE#&pRq~Dsu)*I^|Rd510iZYu<(B=RGX7 zt`-y714^h8g=EH(6U9ihS`lH6nR&i4BXetYg}9<5&B{{K!_Cgw=C@sJQ6g9Y4h-$5gnHZByZCv4r2K?qUu;ag zB+t)0v?R8!U@+!0_i9tem6}9r0dA|PMKipi9jv3Up~PZzxGASvQ+PjM;?TB}Wiq#GkU#O|I1xK76L_#3ST zc)0ntngo3KkwCc(x%^pm`{yo%Augjq6Y5ESsJ}qaQm&lvV=^>gNq37OPRSP>CDr>r1JplJD@RaS4?Yp06=6rAB|1nL(wG8ZLpUx zgc)$BwZopXw6$?ce&)DSj^|5}ALwWi!twzD6@>dXU8klHfdp)N&J(G}?;D={iw zZbHWLyr7c&BT)pZ{p-06PUNr6^Px54wQ!x>u5&%)#Pf%Q-XnXCK-ufHsL6#Ur%wZc z=;`8pwMUWrA1XL3n^_PURfQBY5baYZ=0okQwCp+u-{Ycp#KGijMHenxOUD_v_ou_& zoNU$>xEK1DbMT8It+=Zv7c9v8MxUL=;*HQ?O9v6&axo2wqSPiWjvJEF;YNA(&r)c1$`!0s@K-LZcs_}550i7vV-fJP!QRD^+ljYPBO&;J*QP?!DZ z$WH2pCLqxE6WkSM#A#f%Gh>W|4p$M0pWZuwy#KzXFh0FvL?7{Ru2ni2;su7+(sI>D za*Kwzl8L2hXWHPceTJCN=JPF8y%~;QUS>et5(@+h8iEX?vc@Bm-4p3xO+(Sh(cTTx zvR@qO+T%TqO=K;3#TLLUr98u4q*OA1IQ7a3qJU=(*JrmfqNY8!Va4f<;g?kE8$|Ca zd!Wdwx)fGGNJc>Rqt=c{36kzCaF-U+=r`XqK};D`{Go|H0+)HDrdv;yYIgB12VA<- zpv<*TJZ{|3FyERqtyPU7X!H%)I7G2*M-yyJb#Y`E5+`37k55L7%B6V4!tq7RGLOm( zOD78RErKfgLUY_Ou$^Ch21`8n{^9Zja_vv2PvP)X&?#r&syY`7gOpU@3Xuj0Z7`ff z5p%lK9wE^LkgFPk6an(?)1Qyxoyh)%!LcX*=)>`JI&{7Md6byJBi(94)^n8bgk@3o z3Qr7>;=_yHCM*Qc{@7PF0{U|s&3a;2q6D`rEo9Jq0%H9=E4#~6L-X5V*`+CTf_`6M zVPM&y;`8rsdi*hse~ka~-iMmv-wph|C;bQT=a>%V%3nIvzXJc>r~50g1KRxl|99>F zVdt0T&7YRepzWMrS~*2=I3>gG@@LYYn} z@5pNJH${yNRGzWVvnV^$=yTRqwB80+JB+drQDok0SndWz%A{Iqv?i*p&JU^bT`n{& z?DIyZU>huv1Sz@*bix?WFJg0%X^a8)FJ6@YlPk2~ghQ1%pfbcltezt*#5i z9F36PZgW&bp<(coH+fWsram~iLDNw=r${+fYz?5g&)>}7rc29t(ztiV(v`QA=E;w4 zl1t5ANYtWCF=^t#q8AZ_VhN@P>I^ApZOYWSMjL2X zsYu!Vt~c@N+Vr&+<6T({1Ia#&BYxUpHgd?g$MVP?Ex zH{c;e?YJ_ip}CnPgVtj2wN-re7AhymlX&_H7gkmP(GxKJeU6m=zE|=V=Cb3-Lcw#= zrnle$VUM6(o~Vpd&?nTDBy2>Q*q=7RWiLOyUHxp!RqSz$wtMxGVf^~cVs~_SeiIbw z|AdU;poe5AH~^3e+WjDdMt~smSFlv5>niQ=q4*gdK84Yo4LN1#l8=?8zKepOhtU5Z zVZV(QMKM`RsDrff>NjMsX4J0Zwk$c=obL0}VjutD`l`4plbQ+>dt!u=I{QZr6*i;; zk<@APAOU4n0f!x&PDE(T+WX?QsrRoweERS|TQIh#-vmg-tf~Is8r0gOEdR`h%4pM5 zhgMgN*L3-&T!T%gG=WuLbLp-FW-*ET6pJ)@*eHBH^1znrvdSW6BS=O=1@T&aRCm5$ z{wJ#=XL785l(R}RpmrB?aSChB?MOh--!i0d9IkI8?(4G5z2v!)0M65kf)U>b^$8c+ zB&|dAfFa!(=nHgY^h+j4U_Svt7>3bkcFX6j?Br;n5xNyq$!ep4M^g^4nK~U=Qcq$D z_`5ZqyJOcudiVC^Dp1hqD;0|97C)EHX!ml`OW7mn)>@n)nJp^v83ZPO5W3L!X-d6H zc7o80iJOt(| zTEQ|SP$cgr>2~N^hu3-!4ElEAawrYHDPJ>!YnSqll_8QK7;$BVAzmAdN`tN0IhcZO zdNh-^LVI28j^)WP;u^1ennTbwcT<^jLZOUh*BXDy{n>PbgdzQQtaD~2@aE?+Seb`- zPHswWn#tnPL9tfwBULJGtAF|@Q-7QXRJc)}fP9|wuLJlf9E3^q4g#n* za{0EA7X7w(W*0oTi34@L4&Rq-uq~IDt!&HJ(#q3%p6xAwMvL*EJ;MZ;!x@cUwG_(< zbo$>eum%p{;vqLpW6Q1Lz8gjC?7<)2U=cCkf>DV>@J)gMCsqhf&-v)YHBOcHIp(Kr zq5sZh!$kb;CV78dp(-=Q1o0NV`-BGici52vM40<7N`6>$6eMMYeX~zsY!IXh`GAl0 zKS`oTZ8JVzY1UCa%|lmLL9CYG&J#A|99p(3P4H}Hog!#s0`ICImw5i=)3N4gsUk8IqgLu;h{v|nu28hE1 zCds`E>pX7WMNzT_-MVWNH2S`fW~|Ap$-tsj#w^0;rpy5I^K}P0!Mf@?8znnIHYARU zc3I@8La2WyZ3nJYC21X*o@~aJpxuS%%W+r(r?+Bta}pi|O5McyLYl?aqg`uJP{aGO zJhZMkId~JJGH$gjcoiWC36y8+#ScYjh^)2~JnWA|@Z>5? z*$#y>W)p$d*h7U9`aJWfN7ucw4$nQYMnxKxWfQpH9?Gh;WrYj*?+%1$m1F{*I0{)V zg1;*sXZew-9@}`0SNQNK*-$D($u%ZY^(Xemm7LhP%JYoi8w)GR4<<`Ny$hoFxIh+> zYd_fDz0vXbNNiIcVdQgGVZWO49c#{ACUJ@BsFA1%4l2D_YO~H!PqFG?b7^X&?q?s% zW89c@=(6;xSWEPk=@B{5i)EK~t~5^vi$%2m-eOZ^o3bHsT%-Fmdmdlm);KAe>%OyJ zK%ZV;^pQvcxwoEukG7;Su)(rvd8JteQDAWlhvq_{5@(r4wnhE8#uP_{hjQ;}rRjeB z0m)sHycPW;%Tzf+zp-(n5CUnO@%QsJ3MJV0__U=VD|st5GbtvpYDZ&5<$=9RIafl? znIg)u9?@+Ns0gRuhOfdT++71?X3w9e_-l*XzEAP5Y9YftCEAl4onM4(>@397ieSt4 zO)w$Bbxhfmb9f`lS5h?TotIEPJitC6t#zql5gYU?Hx}%-&hibTN#Ax%oOe5HzT6!U zI|32RBNzCG5EH8`?R1Ge02cFEc&s%C0kzF~oB7hrUt=*VM?V39JHrS$L&DRMClI@4 zS~u|R>cwFr=T=XB+1Oii54l;M(|y`V4h^#m{oVSE{+ZQPkt0CJ@pNn@%k7E8y)pUi z@jYwxFTpe|Prj~;e!U8-uXrLmVHf#(j>JDmmja(|MpL$?2d2-I`ramKT@Ofgxx3pezT^$kBlO%)FlP zbv*Wayv1?+f^!uxA!S`3EA}a?pupvmP!@07{bW{}mvkC@UqXsum-3}xX7o}T#cYbd z3l9y4kCIGMG^#nQ{ZDv?Cy%C0zaL~P38b>99&sTy#PV=H9NDiAgOzK2J0I``ny2BMXHJ`+M_B=k!TL zU|1VT{OrnjqZ@?BFT_9}q7}}xw}G0^NL-9xzeCzZ?i))+x)qgTI3w)m^k{-=XD;^6 zK8|x2>VZ0O9AYR8Q;<4`sDcVHh^4eq^MNy9%uTO*qx!~%-WM-Gid@UeI;yhX;1CLT z0Z>Ft*yep>&}h4b_&m=ET+MrEJjY8qLS8rz7RBzc+~TAe470hfnLWO?ea1`Lwz0YQ z`n(Q*l;VdNNh6E2__e4;Fy}n59&cZI2hO&Nx}$IucfXw^%P~+HMFM zpezzeAuG_DwgrnP%I1Me37z?&-3(83dqTMn0wI;INc1z@>%2p*@L_m9V>E)5Ikpzr zLTHj;HjWcpBI7HYFWpQ*OEkJVS(0HeXeE=^U;%BGwZSEo0}4U|BJx8 z17yROn{V~5N3$P{w{v_RrxSDheKWkYoK7Q$bNpX-htG(GpLSy%h!!Y}qV!8f1zXFR3NVO}H|Oq4NnfOu zqsZ8XkXP~@Mq+iZIQ11{=st}{eIKMycqeqh7tW*4MggrswkO6<5!%9_2PIa@xOu#O z$rs(GvEDDuh1LX_kF*n(reog`vX3^%lq`;+HoMg>BgI61hwWpw{U9b*Sdzw8Ej%RF zVpip(RgHR0-@|fRb2cq$OWsNm#+20_&WgTl8nzBI$X+r*5%Ko8eH%p;!*em+wf8Qq zQ+$qn;0+~yzi*Tg{n`{&5KyG5Sk^PB)M0-ObzDAe`O!&~8m3*Vc)~PFe>Ib*zJNVP z=W}NitK_g{bgjpgei$SnSG`X*zArDB#&U1fKwI>&p86Cl*3zuo1fS$7nsL#wP<9>S z?J=Vw>J%6y>o$sqA1A)O3;>~&^BuQUClTq7W7Vm? ztZI1s!86ZWoa6CT<~C+gsO}mBlt-6Gi||rT(QFoR2*}VVd(i562k(~AU-->q+XHxmgN7^b)#b7jlImQ_dX_^TkyAJYIh3kQd1M!O ze{_`bR$D*4`l{DkG3P9@9W;LNWZdz3^kQs$YtSWPp6KD}Iz_#)beWDKcefX`t)JH} z1h!#01dcOzLbINz*9drz>hlHfEtdUQe98q+;u$LLiL@nDWnP`Vk)mX=7hMM~`d8MO z{p|A*+&lJq`VdxsPC@J{=fULwWL)YOa;p9%mXyZo;l*?G^IiNykw* z#&0d~Rork?43WezvMoXkmeaMSUA3l_k5cn=q@4EPSnMz{KkVIplkD6J6I~GvwIE13 zFvTosNbiWQf}2d2@=9d?oc>1HgqfWr)>Un}&B@c+&8kdLx{#4P+Ou>*-=`X<;9}Q>NhI;m|1eUB^+$RwNGGQmb%`Cf*GotFtv=Tirj?!{Q zZ{80kfwVI4+hnEn_r&vKE>Kqc?~4cNj^o@&(+Zt**_~XI#<~+wIZR- zd&&vy%=vx-?`2`k`NF24-J#eYSzoWR+Q7~LgP=6Wb#Lb;#Upr8M=d;(HL2q!!tg^b z>;7SXtz+H-x;aq5%h8{wMD@FJUlymd@^uk+9IagsRx5cu9;%S1i=xiUy6O^ATXKyq z4QHPpX(?2Z7`d$`{0U1%%A;}X+>dvsy}e`wEd<5Qx8N;(xlzZb2C>!ao5SO}8Svy- z*y&w4%+~!k&%UkOZQaCWK+jEb5k3!_WYPMS<2kqqYj0&3uPmv&>X!TUlNcuS|8 zq}hho?}~g+sY@cN#`GkJl1ZVbr5@|@9=@e&=O_w+Bp#~Ue|(LXN_;HP=q0L1ES`kL zm!JsHNfTQDX8OD~!AVm%W-mf&7KdhL_QXFPK6)eu*XnA%xd*^Dj?3=(vKP-qz;Sqh z&x*wO%b9ZC-CJ?babkU#rnHjWp&2XPucHz)wskO0t4jh)kvis>>k89SUMuj|939HC z(pd|dU9%zI7i3o z?VhU}O=S{+TqyED?g?s|XRxHPQE39pM>o4KO%pynyr^<5Oa;l``o;yX>xpY!@5<2! zQ7{iBFWr1n4BZG{mZ5A;jg04%wC~{rzsb_&?(*+-r}aa}0mpVM7r;KMq{U2$jNi@~ z*I$r#=>mP-2@s`Z2`Cwenhwt9nn;-Pmv|knwYP9@5JVRRA%oc|2`T*5lz03wFb-+8 z+2oi>Zjt%dExuMI34_4!`uh9Bc19Pjp9BT+YvqV-Pg+J7&c~+1$69FEK)*&xxwdCK z9hu<^C8jWy{m7S%rabpv?c;2RXi z+I=y#+O7P;{QdlVlkvf%NJ4B#Om8NOcLXhIYEHXiDo)!bqJzBbdsS~ z;t7o#Y-YAfL))gYJ4*L|zTiNjiVX;y!`+e7^^G!h=;OVxT6JKd6sKr7*H#~5(12|z z)b!&+OVl{t^rqf;$Hp}?$Sjt^Q!{>1wEFBa@ZR{|ZSCDW>(4FirL^xD6+E2_vx>?T z+He2}1XsoJ;L7w>N1Ce5MP;JzaTuKwg~!WhbjlkhBXkxm00QUdheF+Sy5N}|WpknG zp1xjpZn~ukI@~tw@piy!TU=cSYPiZ|ArwRFRsuliBLaDjdoYos!H<`=2Ei^wbmq4( zKSXm_-+=E@v+TkKIT6gFibWeuZY~d{$mLX)6CjHYWh;F;P#UgstJD56+}Fw7y+;8o zazSx>F_Xv=|K=~3zU}vrmH;6E!TLjtgAB%Nd8;dyMx$q;MwL2{wl^@N$fIDNg9L?M0?P3>x~bTI31V_%#GFdD(~$% z=_&_MVY`-JIGdVbBmG$B%ZuWTQOF64_3@;Fb0^&in5MJU9kw8BK*6|r?OlxZ=^T@z z0gdV0vJjnFSDY8NXHRw%+hY%r^!tKqD_t3@(C*)(_G={qqG4p;2qc^uf>t67&TG5I{`XUOMbf$)m?%!@RPd>B+Epea8JYVcIQjY8QYNEi!gWXwi&b3m zsh}P>)&ajBxaNwl-*bsc2l~pL-rq1~NOS~VICgD$@B4P=OwUUDp%QGzc_!gFau$gO zL?{r(Vj#41H`mv}zS+TYsvx6W3NZg-w4!G)gS#WLk7gl~dSwJpYSF&btQ85BHUbnt z^o<%f)e>QVsoi72Ha*3j&M3Ac*}Q~-;9^8&+SCurmFNgYoBRS^JSm`dU@;Rhu1}90D){G^uLzXv%qZ!6571voVq#l-^Tlv({0D3x>Uc3n zT0H3NGYX6)5sqe5W}fvh>@Jrgb=5IfcJF2NItHeL(RQm65OOeqlvR)0ms~_0l$!cC z3sY7X-jr6aWi6)WP!>B&XEDI(Ag!^s0bj-{F3j>S;JOChkfN#23n%5&P@tO^@57S1B72ss*X*h<@U$vDDWyH1CjYrI5KU(NyaKIj!?Teq&TVTPRa1goz-yE1I9Q1Z88dOQtcDVZ6{jo zFX8V+Sa-XgNC&z9ae+9AD7?)B!4VGxR+PWO(b&=Pe`o|v*}qz5La+VmZ}mFxTcps_ zm`r<`ND&RzLVPc+T7azgff*kbm3-&`-e|UYDk0Q0qWj8fMGui_Lrn4H%B($w|MmfO zJ8>{NoQX-vSe#{ar*=o0F+KZ%S=q8*v`HtfAY=Y-B_N0`6NVa%lMB5PW&0*%E8WcmqJTQXq6F zs>gu#VmQZS<|QKYzU=Di=V*EZZ}o6%fr$eLr{SJke%OvF@kd3KVOaccNqWO z|HDNYW%<85_;ol3Je>Lm?-4^)&mpOm+^J}&1Pfur{`qwYD zuV0OS)!qLz&PVuT8}NTB{I90J3Wx|)@?L6jk~+M28Td!x5k1y1a}Ax!6mr6h5*6c9fC^;9w4{{C+OFidGF0k zX1-tW-tDz+tzKQ{RM)v@-+gMIQjv#&#Rk9w5C8xG1;FGu%gPW60C)!r0AK+Sp!Fmj z9bC;FT#YomUYolZuzK3tk>|rg)8_)9A?N>Z{1-={Jn@rKFFWwX^~-z7Z!B`lHDU-n zN8mwBMpconzJ!4?Q@w0!>!xv*@Q|hxk09O|q?5YX={xA~#{p`-qD&(7-!ercPgdU++%(n8*v2+ZJ9N zDaZUv{qgkPU1PJzOw?>*!zcQor*#C>Z-W-ZyV@643&DealR z77b=gDGTs1g8fT|hMV}il#fh_eI1IYo_<ar1{J(!TvLFc4jBSVBWH6v7dFTL(d(lNsEJlO>4gLQ-y>+@3MgIwwmzm9O_7csv7xn_Y)>E0_Ky8 z_fSQ{1QZUZ({P4dOe_~FeTVEQE`Jbes#^(ISD9ox3sQO+nb~!nOJ#IoJ@~RHWe%xP za%0|bPRI<>bOroWqI}SA@+V4`aTSfBr9EoFx1~c5-5_Gat?_|3j`#D00byc zJGS3(;^F9QYvSl=`;)!?jTtBig+aXhcaL&4MfpB-EEd`hV#TpS~i-FE(c{Uiw5U-Q5LHBDOv<(=n%WkI92uxUSEW4BEaE@c=V=xFN zM-`-C88|G=!~KH~N62XDE?CVX@k7kn*!IInEouy#9>3(sAaMpHj zq3AVk+g)&rvPJwsp0tcd*f+$LJakBgEPx^2Wh+0eRr6%sRpM@hp>yes`P0RT#kb+X zxi!d2|2<_Q+^&$bVE_O>NcDpbIRlb1e@>QiO>xI1UhEF^F9ASLZF;#xJ498aPYqQ7 zSqy3iXB7PKhTxa^GOE=0G?lxkz70Fn9r~(CIo+22c6W}N_im^Z=YyJ7SVZV<^89Xp_x0Kl zTX0(k=Efr6O&OYM@vKl+lM<(4jkofs7hLlDcQoo%iCp>qNi>)KFx2&3=Q^37?~bT@Q-RR4$A=W3WZyfuivE{&m-@x>GlxBiAevbs*k}$Yl+6*x zDTEnW2Anj!c`R9b^1eGyiOuP3^9p!cUwQZ#u>5p(HKE+O4!?X#HCJ6p*x$7O5W-sWzD*X!=aP|2&AFurca8mE6%5((K|w z>0$`~3?6Hsp8kY=d>jRgUx<)`-qGf9bubPh_nT~x4FBYn+pO>V7@N8A+;-w!3>#GM zWMxI$!zS9JJF!^71_9V{(?&khXe#Y_nNDKx(`JO79{KL7V7mE0VfxEt{odC7L}eBsc$%cKOTgAVs%TA`wT+i-J(M>Co4;nTIR z?OV|<@IQL+Wz5VCv?wWiv+-hRyaB%0jAes@Wa*W4?FrhLl+r|~| z&7hT&9;$EyVUGbS`ahRQnAv4!A`mDqAOZlWzeCx@)yvM@<>#(nr!^Kc!w&4AIz>x9 zS-9tBuun33u_rA79b6aLH>a5Sxkof^vr-Mn$0~$Kcg(~pgbE&fm04(yi~EJ_YI;%q zF2ivI$rqu5=JR4^&ZsxHZcTLX8Z4a8IXNO}~$ z%|l^GbuMTqRZ6`;(c>H+ZRqcL`K`8u={>iubME2|#jxsTtlJyHc9&^Tz&VvprDlbk z3v5#*qkQtr+Md$Nd-*dMO!76f;aGjPCTyPq$8hYH^lUCBGNny5S@i;a`$F08rF?}> z54c%aXlL+W6gTyqisGP>$Ynd~Us76mY~1a{u66p>H^d_hG-lPeHqf@f*FP^*wL-54 zU2DSyjp`NH>bmNqRhC>IT3RnGbByv9-iH`BOnnBjZu;ijb`_JNSflXRlqM6{sdS1g zX}$AAJeoECAbcG=!=%{Y4@6~cs_5l5WHb=FJC?fccCnN*Rg#CXp0cZNSd-TL-mEbo z7GC1iJtQaFQ(@9d^Z?y6%&=3^&~KEIQk8)@96c251)|0ec{}Ib7%X;c9y)J7o%*^V z9xPW%Qil4;o?*MH$e@{3e5Fm8@ZT z)K(ikr%QGMZY(xt{VZ48_)a(j^#k^!i~BL~?jZqzTvMCrN#36m;fnN}bM|mCv-0!d z6PNt~5u^8{PRgQ<(c{qg25dlU)6MA~OdGc!V!>93Q?Hr!WP|xNAE4F>O>uvL!1xgP zOT)=3@`a7Ke9ii{-IFO8{a~l=6%kbUfpWXq&PW{)QKUR)L3bmz0(~*nl4sQGMw)00 z^ZwZE(W-@i7}|1Uj82I^KaMMU8<`F%XB02eZXa$n;X26U2JPePyGE!GFxj{bnq&QO z(9v)XsU0Gzo#5>eHr%4*@yx43;TYeEVd?CXU{g*f+D+AOW^rW91MBzHBl%F*yU8Ex zcW(W!kM+)`M%l#$5}sGMBcWE1W7<7o2DhsB2;bAm2S5s`zjJ1O)f2kmGXP+W{HH|l z8)v#&ncJJQ{Wf#_gxjI6UL+A8wm;j2D9ZWq4e#bBOvh86B{M!mUAm%Jklq#j2Lb}w z)o8@Pc#$NDym!fM*m6dJReqTmg<+29@B^9o-lJj7|A2Ba z9wVmu*;)7(2^P_K=LSvY2n-Y|r@*n-&})kLv2npNA}lcig7EDeELw6!AP#Bj({|YY z+^a=13F$9)4^wObzL(z{(M3W;1AGKza(bk~v29*S{pd)C(gPDd7hz@!)(Kc>l4CO^VA(o{OuHjx#zhI{Y?p8-W=BrQIdhV^#qs1I{I8B2M$_1_5- z_t8Ip4-&^~vs&k&?+>-TshT;wuzMg(+_1I1@qWL8yr1lk8bL3MHa}TZrFF!r-3nH~ zUvcWmFlH~+=G+oMsxeM7u!%sF|5mIYIb}ca44@(wK`krPoVt!kD9-7DLGvurv(=1H zd}B!z@;cR>-&IFbxPOD4Qp-q`srQ;`0 z=ckecbWx$^Cj;I_E%V`{Tr-=Ihs(>7fPmA$>m78X$IGv^uKP2d#v3`lcT)*D0e}MDM@FI+F6&H3|r_uS;T@t{e0ZuA<`hLX%u=!aSN__99@&L%+yf z%4=bxLiThEEJdF0Za{W)dPRWuefqw^{J_6&C!XJ;1G69Zbu7oC@$%jqCL7k?-d-)wB+|~;eAexx6bFZD6FpfUksbO^ysq#MKP|Dfu!+NWR-pjz zT<$EwN_oVy*(G7&LLwc+`;Y=y^T#z=cs)iG4;3>)p^&IPtcJ^;jc)8{EMPr8H;-+7 zBM<~0thmvX%Ti3NaVhIs2);v4<`L#rt~BMB{VZ_MRwnS#=Kje~v(}m|XP$GXe$P?hWc^Y-sifoS)t$jC9*Rs}fd6>k*NPJ@j3nEoFd{rYf3G8f(V8ewpohetH zDV6<{Jdm{0781KX4vy#6)z=s8TcP5M;vp8qi94n^C3R_SQI$yJY0};aT<_DMR7}{o z$YWjA7h0UWY}~BNgk=g@sG?kJquHm~!FcP;&&QJQbO&0}LT5RZn@yUDoVoU+=P_(w zP?j_W($cC=_C0s_QE!8g!bHVw`vYqizjx3RzL{!P4=-Q6$&Wxb+u zP(MLw1fC-<)hDr!#fT*!36*<_INzR0`wehaXER-l$uZpt==yxfle3%3I%EZ3HCu@+ za9zFfwf~2HBhqr}1>&A|vgp`$7{MJrV4;o<8`HCx@Hde`&ii8=ff-deaa%t;_Fs)y z&9_BN3na!+2}Y{8=ck%w_wa;Qj!r2vk>M*Zq<7~XunMJ?!CWONuUL}Jjkv&DZ(A1k zGw#p2ChQ$N|AzYEOmen{vB=>;iN(3nmbDoTbIwanXmi#t2Bw>xC1;dVQMXO0H=?#y zb*YYv839FkmiNZqO`2c$w1!S}C~F+#CdPaqmvwWyz1%iu@vJdW$oq8we~H?zN>3J# zjLJn3-#Z5TF5G6ST0#sFFBc`y3aPiVQa@cHoFCYAxH(EN$$}+sxsKf+4 zY?H)m7Y}EV#%z345WKUb^A;-HklI0wt3($q#svaRJ3cam#C0E5az8s_+iW>B0%zq$ z?xky0&>555RUG+l#qSKY2D(PnZ-;!Lj_2I%A*$d%&~bl;=Z+ENmwt$Fi2pks|9pgX zu`)Mzbz%GM@Ea#*CJe_c{nUrxZqdNUF>QQ^8z{6yf{pnHx^a6tVP?(oHF>+ODekms zP|zsTP;HZSHxGqUN~AHMIAI++MfFK5pc-^EdyyS4+{39K4hn-)HXn_4aUeLmYbtba z5IrHNT2C9s6C6B%Q+L%t!nG}hn%){{o!2dzRGsc~Uq`eJLT)<@sxxuI-&Sy0nB+3H zJ26fzi-*RQKIGZ?y`j$rCsDK(x2@z+F;c1RoBxre- zwB73&wzM&~NOyMFH=h8;7*|L_>x1(Lx zL?{~GL4hDY(jQMd-M8_NzlmXxm4AAwVHkg@$l@f+EL&I0Fv6xyU!%#Xp+?X8a+C?a zrhGK<&@Nsa97C?EM`KF!7E`atB-bh}D=^wWEi2leQ{ydGF_NSP&Uh+gl!|7Pm_{o3 zf=N_a56%n>MOH?+lG4pqTV}Kv&i(@@aFE`^q`3cjbue}s%lU1D%O_D&1h0@+0+k}m5gy> zh<58Bdy-aQ($EoKN`5Gx7m9syB3&zHJPBkwr}!d7f2c5I=SD_RuNj^8vxewJTx(51B8Zl3NuW)*Y^Dxaf+p#Kpnn zpdw5=6FMuZ)cNq{!PiMQ(9LI-H=-RmszIO1u<>}`ou~Hrvq2eioHzG6A?;WW`h=2) zp>7vnSgEIiz=5kPs#44*cnojB5IHWsrbjEpyQ2Z0jf#x1mm{RQP*5j9-IztYNa$sJ zM|qSaFwOCu`fiZbq$1(-`w39oOvL5>c|n#a(=cf61^26zEO?HyAv&)Mxb7K2n>;e_ z?^9cgasb&-Qt$L$Hjmu-Fzi!tPz6PG`A zw!c4Bga6#ueRg=G+RjMeB)J>gV?d(t|{K5 z7oMmfxhx=5NsVRebH}~d)SEGEkhYT5GrzmW{Q~6T*^~EyY(>p{irS$qXCmT6d1fIm zft2Xf>iR_a=i&!s8&o}o-x#lJ?SV%7511> zs7sQHieHJR!;7YQ>W|Ajn(_{_jsY^p{DV9?BkzlvU~b1=O3_hUQq}yJ)I|S^A_>Ry z(v}?&;qeMIS^uLxSDSVm^u=`3La;xpMu4bq%8%WIyTP8Q&WWsdi|)ql{QGk3KJvm* z1k{N7IA(DmsOUWU6^K9+bW+U>2jgffR^qv_+n}pz%I9!e1n~E{|FAo~D|i`%di2JJ z)rJ$rUCVHz_0A;vj6RAo2Pldb`8xZZu5+#Lf%ARoNj^IICBxWL-xqW||2ldESqpg& z+CDCF<S0 z1AT;;7DpkGw@x)c)@R2|5SLayq>peo+dPE?{sPr~ajCqE)U+4a`hlJ>&1b#4US}F9CL9Li!Y;; z;w)>SLh%6nbd>po9F7T0pY`&Hist99=IkD*-ti?~{z^SQ1j?6wQx`X&xH!-YjF$ZngG)k)#36sh#P(goEQi7vT`LSNA5IzNOJnpfu$5PJ z7F{&p{_*l?;@XEsm&{g@-#%;Lsv;X5m6(|C3ZCj2@?bELJlaf&4P2ZdAX_=`SqO$l zpYD7Z_ju+RD%-Z~lP8Z3G zKHbk>8+Amkgz#>enn;ulCMGO?WMp=_si~jsm0lV_7U=gK77CgL!ax7~a>u`y@!!XP zc-2Ei{;v-H+J^qO;m>0V1S@}PO8;v3*Vf#h4cj1P>;K=V`_<2{{hB{Koj^J`zjSYY zHU3p0{=>Kh<#&qyk6!$%>93mCAEs2m-%Nj1zXwXFEV ziwx24ckpYG@vDcw((E7h0Du=U0Pr``{nh-hTjQV2Vafhv{;!QvMIH{ax&QzYq`_rzx^NC+a=`y literal 0 HcmV?d00001 From 1a63ee670255ef27ce2eaa2b98d815b050e4e058 Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Thu, 11 Jul 2024 10:14:07 +0800 Subject: [PATCH 29/64] fix(statefulRoute): fix list column can't reach (#2953) --- .../odc/server/web/controller/v1/DBTableColumnController.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/DBTableColumnController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/DBTableColumnController.java index ada90e85d1..f6b719e9d1 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/DBTableColumnController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/DBTableColumnController.java @@ -52,6 +52,7 @@ public class DBTableColumnController { @ApiOperation(value = "list", notes = "查询表的所有列，sid示例：sid:1000-1:d:db1:t:tb1") @RequestMapping(value = "/list/{sid:.*}", method = RequestMethod.GET) + @StatefulRoute(stateName = StateName.DB_SESSION, stateIdExpression = "#sid") public OdcResult> list(@PathVariable String sid) { ResourceIdentifier i = ResourceIDParser.parse(sid); ConnectionSession session = sessionService.nullSafeGet(i.getSid(), true); From 94f2735b76aa83cadc61692e958c84122474827a Mon Sep 17 00:00:00 2001 From: LuckyLeo Date: Thu, 11 Jul 2024 11:58:27 +0800 Subject: [PATCH 30/64] fix(permission): fail to submit ticket if lack of database permission (#2946) * fix fail to start task if lack of database permission * response to CR comments --- .../odc/service/permission/DBResourcePermissionHelper.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java index 674cbce4f4..19e152fecb 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java @@ -423,10 +423,9 @@ private Map> getInnerDBPermissionTypes(Collect })); Map> typesFromTable = userTablePermissionRepository .findNotExpiredByUserIdAndDatabaseIdIn(authenticationFacade.currentUserId(), databaseIds) - .stream().collect(Collectors.toMap( - UserTablePermissionEntity::getDatabaseId, - e -> Collections.singleton(DatabasePermissionType.ACCESS), - (e1, e2) -> e1)); + .stream() + .collect(Collectors.groupingBy(UserTablePermissionEntity::getDatabaseId, + Collectors.mapping(e -> DatabasePermissionType.from(e.getAction()), Collectors.toSet()))); typesFromTable.forEach((k, v) -> typesFromDatabase.merge(k, v, (v1, v2) -> { v1.addAll(v2); return v1; From 50c64c53a1d489e17694a84965a54ffb7338e6a2 Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Thu, 11 Jul 2024 14:06:30 +0800 Subject: [PATCH 31/64] fix(statefulRoute): failed to list built-in snippets (#2935) * fix listBuiltinSnippets can't reach * fix listBuiltinSnippets can't reach --- .../odc/core/alarm/AlarmEventNames.java | 1 + .../v2/BuiltinSnippetController.java | 3 ++ .../web/controller/v2/ScheduleController.java | 1 - .../odc/service/state/RouteHealthManager.java | 5 ++-- .../odc/service/state/StateRouteAspect.java | 27 ++++++++++++++--- .../odc/service/state/model/RouteInfo.java | 29 +++++++++++++------ 6 files changed, 50 insertions(+), 16 deletions(-) diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/alarm/AlarmEventNames.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/alarm/AlarmEventNames.java index db43c3db1b..265897e2c2 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/alarm/AlarmEventNames.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/alarm/AlarmEventNames.java @@ -25,6 +25,7 @@ private AlarmEventNames() {} */ public static final String SYSTEM_CONFIG_CHANGED = "SYSTEM_CONFIG_CHANGED"; public static final String REST_API_CALL_FAILED = "REST_API_CALL_FAILED"; + public static final String STATEFUL_ROUTE_NOT_HEALTHY = "STATEFUL_ROUTE_NOT_HEALTHY"; /** * warn diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/BuiltinSnippetController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/BuiltinSnippetController.java index ac101e433d..fc63dfda76 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/BuiltinSnippetController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/BuiltinSnippetController.java @@ -27,6 +27,8 @@ import com.oceanbase.odc.service.common.response.Responses; import com.oceanbase.odc.service.snippet.BuiltinSnippet; import com.oceanbase.odc.service.snippet.BuiltinSnippetService; +import com.oceanbase.odc.service.state.model.StateName; +import com.oceanbase.odc.service.state.model.StatefulRoute; import io.swagger.annotations.ApiOperation; @@ -39,6 +41,7 @@ public class BuiltinSnippetController { @ApiOperation(value = "listBuiltinSnippets", notes = "查询内建的 snippets") @RequestMapping(value = "builtinSnippets", method = RequestMethod.GET) + @StatefulRoute(stateName = StateName.DB_SESSION, stateIdExpression = "#sessionId") public ListResponse listBuiltinSnippets( @RequestParam(value = "sessionId", required = false) String sessionId) { if (Objects.isNull(sessionId)) { diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ScheduleController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ScheduleController.java index 6a660d2b06..34088eda51 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ScheduleController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ScheduleController.java @@ -155,5 +155,4 @@ public SuccessResponse getScheduleTaskLog(@PathVariable Long scheduleId, @RequestParam OdcTaskLogLevel logType) { return Responses.single(scheduleService.getLog(scheduleId, taskId, logType)); } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/state/RouteHealthManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/state/RouteHealthManager.java index ffbc670cc9..056895a39f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/state/RouteHealthManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/state/RouteHealthManager.java @@ -52,7 +52,7 @@ public class RouteHealthManager implements InitializingBean { public boolean isHealthy(RouteInfo routeInfo) { RouteManageInfo routeManageInfo = ROUTE_HEALTHY_MAP.computeIfAbsent(routeInfo, - r -> new RouteManageInfo(r.isHealthyHost(), LocalDateTime.now())); + r -> new RouteManageInfo(r.isHealthyHost(3), LocalDateTime.now())); routeManageInfo.now(); return routeManageInfo.isHealthy(); } @@ -79,9 +79,10 @@ private void doTest() { Map.Entry item = it.next(); RouteManageInfo manageInfo = item.getValue(); if (manageInfo.expired(expireSeconds)) { + log.info("route info has expired, route={}", manageInfo); it.remove(); } - manageInfo.setHealthy(item.getKey().isHealthyHost()); + manageInfo.setHealthy(item.getKey().isHealthyHost(3)); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/state/StateRouteAspect.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/state/StateRouteAspect.java index acf369d2f6..d9c7b71262 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/state/StateRouteAspect.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/state/StateRouteAspect.java @@ -49,12 +49,15 @@ import com.oceanbase.odc.common.trace.TraceContextHolder; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.common.util.SystemUtils; +import com.oceanbase.odc.core.alarm.AlarmEventNames; +import com.oceanbase.odc.core.alarm.AlarmUtils; import com.oceanbase.odc.core.shared.Verify; import com.oceanbase.odc.service.common.model.HostProperties; import com.oceanbase.odc.service.common.util.SpringContextUtil; import com.oceanbase.odc.service.dispatch.DispatchResponse; import com.oceanbase.odc.service.dispatch.HttpRequestProvider; import com.oceanbase.odc.service.dispatch.RequestDispatcher; +import com.oceanbase.odc.service.session.factory.StateHostGenerator; import com.oceanbase.odc.service.state.model.RouteInfo; import com.oceanbase.odc.service.state.model.SingleNodeStateResponse; import com.oceanbase.odc.service.state.model.StateManager; @@ -83,6 +86,9 @@ public class StateRouteAspect { @Autowired private ThreadPoolExecutor statefulRouteThreadPoolExecutor; + @Autowired + private StateHostGenerator stateHostGenerator; + @Pointcut("@annotation(com.oceanbase.odc.service.state.model.StatefulRoute)") public void stateRouteMethods() {} @@ -102,13 +108,23 @@ public Object aroundStateRouteExecution(ProceedingJoinPoint proceedingJoinPoint) } else { routeInfo = stateManager.getRouteInfo(stateIdBySePL); Verify.notNull(routeInfo, "routeInfo"); - if (!routeInfo.isCurrentNode(properties) && routeHealthManager.isHealthy(routeInfo)) { + boolean healthyNode = routeHealthManager.isHealthy(routeInfo); + boolean notCurrentNode = + !routeInfo.isCurrentNode(properties.getRequestPort(), stateHostGenerator.getHost()); + log.debug("sate routeInfo={}, host={},port={}", routeInfo, stateHostGenerator.getHost(), + properties.getRequestPort()); + log.debug("healthyNode={}, notCurrentNode={}", healthyNode, notCurrentNode); + if (notCurrentNode && healthyNode) { DispatchResponse dispatchResponse = requestDispatcher.forward(routeInfo.getHostName(), routeInfo.getPort()); logTrace(method, stateIdBySePL, routeInfo); StateRouteFilter.getContext().setDispatchResponse(dispatchResponse); return null; } + if (!healthyNode) { + AlarmUtils.alarm(AlarmEventNames.STATEFUL_ROUTE_NOT_HEALTHY, + "can't arrive route info " + routeInfo); + } } } return proceedingJoinPoint.proceed(); @@ -119,13 +135,16 @@ private Object handleMultiState(StateManager stateManager, Object stateIdBySePL, Set allRoutes = stateManager.getAllRoutes(stateIdBySePL); List allResponse = new ArrayList<>(); - RouteInfo currentNode = allRoutes.stream().filter(r -> r.isCurrentNode(properties)).findFirst().orElse(null); + RouteInfo currentNode = allRoutes.stream() + .filter(r -> r.isCurrentNode(properties.getRequestPort(), stateHostGenerator.getHost())).findFirst() + .orElse(null); Object proceed = null; if (currentNode != null) { proceed = proceedingJoinPoint.proceed(); } - Set otherNodeRoute = allRoutes.stream().filter(r -> !r.isCurrentNode(properties)).collect( - Collectors.toSet()); + Set otherNodeRoute = allRoutes.stream() + .filter(r -> !r.isCurrentNode(properties.getRequestPort(), stateHostGenerator.getHost())).collect( + Collectors.toSet()); if (CollectionUtils.isEmpty(otherNodeRoute)) { return proceed; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/state/model/RouteInfo.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/state/model/RouteInfo.java index 8790f85704..5f1d9e53dd 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/state/model/RouteInfo.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/state/model/RouteInfo.java @@ -27,14 +27,15 @@ import lombok.Data; import lombok.NoArgsConstructor; -import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; @Data @NoArgsConstructor +@Slf4j public class RouteInfo { - private static final RestTemplate restTemplate = new RestTemplateBuilder().setConnectTimeout(Duration.ofSeconds(5)) - .setReadTimeout(Duration.ofSeconds(5)).build(); + private static final RestTemplate restTemplate = new RestTemplateBuilder().setConnectTimeout(Duration.ofSeconds(3)) + .setReadTimeout(Duration.ofSeconds(3)).build(); private String hostName; private Integer port; @@ -45,14 +46,11 @@ public RouteInfo(String hostName, Integer port) { this.port = port; } - public boolean isCurrentNode(@NonNull HostProperties properties) { - Integer currentNodePort = properties.getRequestPort(); - Verify.notNull(properties.getPort(), "Port"); - if (!Objects.equals(port, currentNodePort)) { + public boolean isCurrentNode(Integer currentPort, String currentHost) { + if (!Objects.equals(port, currentPort)) { return false; } - return hostName.equalsIgnoreCase(currentNodeHostName()) - || hostName.equalsIgnoreCase(currentNodeIpAddress(properties)); + return hostName.equalsIgnoreCase(currentHost); } public static String currentNodeHostName() { @@ -69,10 +67,23 @@ public boolean isHealthyHost() { String response = restTemplate.getForObject(url, String.class); return response.contains("true"); } catch (Exception e) { + log.warn("test route health check failed, hostName={},port={}", hostName, port, e); return false; } } + public boolean isHealthyHost(int retry) { + Verify.verify(retry > 0, "retry"); + for (int i = 0; i < retry; i++) { + boolean healthyHost = isHealthyHost(); + if (healthyHost) { + return true; + } + } + log.info("retry {} times, test route health check failed, hostName={},port={}", retry, hostName, port); + return false; + } + public String toString() { return hostName + ":" + port; } From 579fc8fdaec8d27363c0b763f5051253c156d6e3 Mon Sep 17 00:00:00 2001 From: LuckyLeo Date: Thu, 11 Jul 2024 19:24:21 +0800 Subject: [PATCH 32/64] security: modify annotations on some service classes (#2955) * add annotations for service * format code --- .../odc/service/connection/database/DatabaseService.java | 1 + .../logicaldatabase/LogicalDatabaseSyncManager.java | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java index abd8797b83..00a815b843 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java @@ -778,6 +778,7 @@ public boolean modifyDatabasesOwners(@NotNull Long projectId, @NotNull @Valid Mo return true; } + @SkipAuthorize("odc internal usage") @Transactional(rollbackFor = Exception.class) public void updateObjectSyncStatus(@NotNull Collection databaseIds, @NotNull DBObjectSyncStatus status) { if (CollectionUtils.isEmpty(databaseIds)) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalDatabaseSyncManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalDatabaseSyncManager.java index 63eb861d51..ef550c1f81 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalDatabaseSyncManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalDatabaseSyncManager.java @@ -24,7 +24,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.integration.jdbc.lock.JdbcLockRegistry; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; -import org.springframework.stereotype.Service; +import org.springframework.stereotype.Component; import com.oceanbase.odc.core.shared.exception.BadRequestException; import com.oceanbase.odc.metadb.connection.DatabaseRepository; @@ -42,7 +42,7 @@ * @Description: [] */ -@Service +@Component @Slf4j public class LogicalDatabaseSyncManager { @Autowired From a1aa043601e91d9c75f1f7c19ef701fcaa029a0f Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Mon, 15 Jul 2024 11:23:01 +0800 Subject: [PATCH 33/64] refactor(statefulRoute): generalization plDebugsession to UUID stateId (#2960) * temp * fix --- .../web/controller/v2/PLDebugController.java | 22 ++++----- .../odc/service/pldebug/PLDebugService.java | 19 ++------ .../pldebug/model/PLDebugSessionId.java | 24 ---------- .../state/StatefulUuidStateIdGenerator.java | 48 +++++++++++++++++++ .../StatefulUuidStateIdManager.java} | 16 +++---- .../odc/service/state/model/StateName.java | 4 +- .../state/model/StatefulUuidStateId.java | 41 ++++++++++++++++ 7 files changed, 113 insertions(+), 61 deletions(-) delete mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/model/PLDebugSessionId.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/state/StatefulUuidStateIdGenerator.java rename server/odc-service/src/main/java/com/oceanbase/odc/service/{pldebug/PLDebugStateManager.java => state/StatefulUuidStateIdManager.java} (67%) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/state/model/StatefulUuidStateId.java diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/PLDebugController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/PLDebugController.java index 520bdc825a..803e958cdc 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/PLDebugController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/PLDebugController.java @@ -65,14 +65,14 @@ public SuccessResponse start(@RequestBody StartPLDebugReq request) @ApiOperation(value = "endDebug", notes = "结束调试") @RequestMapping(value = "/{id}", method = RequestMethod.DELETE) - @StatefulRoute(stateName = StateName.PL_DEBUG_SESSION, stateIdExpression = "#id") + @StatefulRoute(stateName = StateName.UUID_STATEFUL_ID, stateIdExpression = "#id") public SuccessResponse end(@PathVariable String id) { return Responses.success(plDebugService.end(id)); } @ApiOperation(value = "setBreakpoints", notes = "设置断点") @RequestMapping(value = "/{id}/breakpoints/batchCreate", method = RequestMethod.POST) - @StatefulRoute(stateName = StateName.PL_DEBUG_SESSION, stateIdExpression = "#id") + @StatefulRoute(stateName = StateName.UUID_STATEFUL_ID, stateIdExpression = "#id") public SuccessResponse> setBreakpoints(@PathVariable String id, @RequestBody List breakpoints) { return Responses.success(plDebugService.setBreakpoints(id, breakpoints)); @@ -80,7 +80,7 @@ public SuccessResponse> setBreakpoints(@PathVariable Str @ApiOperation(value = "deleteBreakpoints", notes = "删除断点") @RequestMapping(value = "/{id}/breakpoints/batchDelete", method = RequestMethod.DELETE) - @StatefulRoute(stateName = StateName.PL_DEBUG_SESSION, stateIdExpression = "#id") + @StatefulRoute(stateName = StateName.UUID_STATEFUL_ID, stateIdExpression = "#id") public SuccessResponse deleteBreakpoints(@PathVariable String id, @RequestBody List breakpoints) { return Responses.success(plDebugService.deleteBreakpoints(id, breakpoints)); @@ -88,56 +88,56 @@ public SuccessResponse deleteBreakpoints(@PathVariable String id, @ApiOperation(value = "listBreakpoints", notes = "获取断点列表") @RequestMapping(value = "/{id}/breakpoints", method = RequestMethod.GET) - @StatefulRoute(stateName = StateName.PL_DEBUG_SESSION, stateIdExpression = "#id") + @StatefulRoute(stateName = StateName.UUID_STATEFUL_ID, stateIdExpression = "#id") public SuccessResponse> listBreakpoints(@PathVariable String id) { return Responses.success(plDebugService.listBreakpoints(id)); } @ApiOperation(value = "getVariables", notes = "获取堆栈变量") @RequestMapping(value = "/{id}/variables", method = RequestMethod.GET) - @StatefulRoute(stateName = StateName.PL_DEBUG_SESSION, stateIdExpression = "#id") + @StatefulRoute(stateName = StateName.UUID_STATEFUL_ID, stateIdExpression = "#id") public SuccessResponse> getVariables(@PathVariable String id) { return Responses.success(plDebugService.getVariables(id)); } @ApiOperation(value = "getContext", notes = "获取堆栈上下文") @RequestMapping(value = "/{id}/context", method = RequestMethod.GET) - @StatefulRoute(stateName = StateName.PL_DEBUG_SESSION, stateIdExpression = "#id") + @StatefulRoute(stateName = StateName.UUID_STATEFUL_ID, stateIdExpression = "#id") public SuccessResponse getContext(@PathVariable String id) { return Responses.success(plDebugService.getContext(id)); } @ApiOperation(value = "stepOver", notes = "单步执行") @RequestMapping(value = "/{id}/stepOver", method = RequestMethod.POST) - @StatefulRoute(stateName = StateName.PL_DEBUG_SESSION, stateIdExpression = "#id") + @StatefulRoute(stateName = StateName.UUID_STATEFUL_ID, stateIdExpression = "#id") public SuccessResponse stepOver(@PathVariable String id) { return Responses.success(plDebugService.stepOver(id)); } @ApiOperation(value = "resume", notes = "恢复执行") @RequestMapping(value = "/{id}/resume", method = RequestMethod.POST) - @StatefulRoute(stateName = StateName.PL_DEBUG_SESSION, stateIdExpression = "#id") + @StatefulRoute(stateName = StateName.UUID_STATEFUL_ID, stateIdExpression = "#id") public SuccessResponse resume(@PathVariable String id) { return Responses.success(plDebugService.resume(id)); } @ApiOperation(value = "stepIn", notes = "步入") @RequestMapping(value = "/{id}/stepIn", method = RequestMethod.POST) - @StatefulRoute(stateName = StateName.PL_DEBUG_SESSION, stateIdExpression = "#id") + @StatefulRoute(stateName = StateName.UUID_STATEFUL_ID, stateIdExpression = "#id") public SuccessResponse stepIn(@PathVariable String id) { return Responses.success(plDebugService.stepIn(id)); } @ApiOperation(value = "stepOut", notes = "跳出") @RequestMapping(value = "/{id}/stepOut", method = RequestMethod.POST) - @StatefulRoute(stateName = StateName.PL_DEBUG_SESSION, stateIdExpression = "#id") + @StatefulRoute(stateName = StateName.UUID_STATEFUL_ID, stateIdExpression = "#id") public SuccessResponse stepOut(@PathVariable String id) { return Responses.success(plDebugService.stepOut(id)); } @ApiOperation(value = "resumeIgnoreBreakpoints", notes = "恢复执行并忽略断点") @RequestMapping(value = "/{id}/resumeIgnoreBreakpoints", method = RequestMethod.POST) - @StatefulRoute(stateName = StateName.PL_DEBUG_SESSION, stateIdExpression = "#id") + @StatefulRoute(stateName = StateName.UUID_STATEFUL_ID, stateIdExpression = "#id") public SuccessResponse resumeIgnoreBreakpoints(@PathVariable String id) { return Responses.success(plDebugService.resumeIgnoreBreakpoints(id)); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/PLDebugService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/PLDebugService.java index f6a43c07f3..750d61e7cc 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/PLDebugService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/PLDebugService.java @@ -15,8 +15,6 @@ */ package com.oceanbase.odc.service.pldebug; -import java.nio.charset.StandardCharsets; -import java.util.Base64; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -40,8 +38,6 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.oceanbase.odc.common.concurrent.ExecutorUtils; -import com.oceanbase.odc.common.json.JsonUtils; -import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.common.util.SystemUtils; import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.session.ConnectionSession; @@ -55,7 +51,6 @@ import com.oceanbase.odc.service.pldebug.model.PLDebugBreakpoint; import com.oceanbase.odc.service.pldebug.model.PLDebugConstants; import com.oceanbase.odc.service.pldebug.model.PLDebugContextResp; -import com.oceanbase.odc.service.pldebug.model.PLDebugSessionId; import com.oceanbase.odc.service.pldebug.model.PLDebugVariable; import com.oceanbase.odc.service.pldebug.model.StartPLDebugReq; import com.oceanbase.odc.service.pldebug.operator.DBPLOperators; @@ -63,7 +58,7 @@ import com.oceanbase.odc.service.regulation.ruleset.SqlConsoleRuleService; import com.oceanbase.odc.service.regulation.ruleset.model.SqlConsoleRules; import com.oceanbase.odc.service.session.SessionProperties; -import com.oceanbase.odc.service.session.factory.StateHostGenerator; +import com.oceanbase.odc.service.state.StatefulUuidStateIdGenerator; import com.oceanbase.tools.dbbrowser.model.DBObjectType; import lombok.NonNull; @@ -100,7 +95,7 @@ public class PLDebugService { private SqlConsoleRuleService sqlConsoleRuleService; @Autowired - private StateHostGenerator stateHostGenerator;; + private StatefulUuidStateIdGenerator statefulUuidStateIdGenerator; @PostConstruct public void init() { @@ -171,20 +166,14 @@ public String start(@NonNull ConnectionSession connectionSession, StartPLDebugRe throw OBException.executeFailed(ErrorCodes.DebugPackageCreateFailed, e.getMessage()); } PLDebugSession plDebugSession = - new PLDebugSession(authenticationFacade.currentUserId(), this::generateSessionId); + new PLDebugSession(authenticationFacade.currentUserId(), + () -> statefulUuidStateIdGenerator.generateStateId("PLDebugSession")); plDebugSession.setDbmsoutputMaxRows(sessionProperties.getDbmsOutputMaxRows()); plDebugSession.start(connectionSession, debugSessionExecutor, req, sessionTimeoutSeconds, syncEnabled); debugId2Session.put(plDebugSession.getSessionId(), plDebugSession); return plDebugSession.getSessionId(); } - private String generateSessionId() { - PLDebugSessionId id = new PLDebugSessionId(); - id.setUuid(StringUtils.uuidNoHyphen()); - id.setFrom(stateHostGenerator.getHost()); - return Base64.getEncoder().encodeToString(JsonUtils.toJson(id).getBytes(StandardCharsets.UTF_8)); - } - private void validate(StartPLDebugReq req) { Validate.notNull(req.getDebugType(), "DebugType can not be null for StartPLDebugReq"); if (DBObjectType.ANONYMOUS_BLOCK == req.getDebugType()) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/model/PLDebugSessionId.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/model/PLDebugSessionId.java deleted file mode 100644 index c895df61d0..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/model/PLDebugSessionId.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.pldebug.model; - -import lombok.Data; - -@Data -public class PLDebugSessionId { - private String uuid; - private String from; -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/state/StatefulUuidStateIdGenerator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/state/StatefulUuidStateIdGenerator.java new file mode 100644 index 0000000000..9730146149 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/state/StatefulUuidStateIdGenerator.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.state; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.service.session.factory.StateHostGenerator; +import com.oceanbase.odc.service.state.model.StatefulUuidStateId; + +@Component +@ConditionalOnProperty(value = {"odc.web.stateful-route.enabled"}, havingValue = "true") +public class StatefulUuidStateIdGenerator { + + @Autowired + private StateHostGenerator stateHostGenerator; + + public String generateStateId(String type) { + StatefulUuidStateId uuidStateId = StatefulUuidStateId.createTypeUuidStateId(type, StringUtils.uuidNoHyphen(), + stateHostGenerator.getHost()); + return Base64.getEncoder().encodeToString(JsonUtils.toJson(uuidStateId).getBytes(StandardCharsets.UTF_8)); + } + + public static StatefulUuidStateId parseStateId(String stateId) { + return JsonUtils.fromJson(new String(Base64.getDecoder().decode(stateId)), + StatefulUuidStateId.class); + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/PLDebugStateManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/state/StatefulUuidStateIdManager.java similarity index 67% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/PLDebugStateManager.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/state/StatefulUuidStateIdManager.java index 0b22defb40..cb05e699d2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/PLDebugStateManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/state/StatefulUuidStateIdManager.java @@ -13,22 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.pldebug; - -import java.util.Base64; +package com.oceanbase.odc.service.state; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; -import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.service.common.model.HostProperties; -import com.oceanbase.odc.service.pldebug.model.PLDebugSessionId; import com.oceanbase.odc.service.state.model.RouteInfo; import com.oceanbase.odc.service.state.model.StateManager; +import com.oceanbase.odc.service.state.model.StatefulUuidStateId; @Component -public class PLDebugStateManager implements StateManager { +@ConditionalOnProperty(value = {"odc.web.stateful-route.enabled"}, havingValue = "true") +public class StatefulUuidStateIdManager implements StateManager { @Autowired private HostProperties hostProperties; @@ -36,8 +35,7 @@ public class PLDebugStateManager implements StateManager { @Override public RouteInfo getRouteInfo(Object stateId) { Preconditions.checkArgument(stateId instanceof String, "stateId"); - PLDebugSessionId plDebugSessionId = JsonUtils.fromJson(new String(Base64.getDecoder().decode((String) stateId)), - PLDebugSessionId.class); - return new RouteInfo(plDebugSessionId.getFrom(), hostProperties.getRequestPort()); + StatefulUuidStateId statefulUuidStateId = StatefulUuidStateIdGenerator.parseStateId((String) stateId); + return new RouteInfo(statefulUuidStateId.getFrom(), hostProperties.getRequestPort()); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/state/model/StateName.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/state/model/StateName.java index 421462658f..186e780ae5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/state/model/StateName.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/state/model/StateName.java @@ -15,15 +15,15 @@ */ package com.oceanbase.odc.service.state.model; -import com.oceanbase.odc.service.pldebug.PLDebugStateManager; import com.oceanbase.odc.service.session.ConnectSessionStateManager; +import com.oceanbase.odc.service.state.StatefulUuidStateIdManager; import lombok.Getter; @Getter public enum StateName { NONE(null), - PL_DEBUG_SESSION(PLDebugStateManager.class), + UUID_STATEFUL_ID(StatefulUuidStateIdManager.class), DB_SESSION(ConnectSessionStateManager.class); private final Class stateManagerClass; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/state/model/StatefulUuidStateId.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/state/model/StatefulUuidStateId.java new file mode 100644 index 0000000000..8acba4c3a1 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/state/model/StatefulUuidStateId.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.state.model; + +import javax.annotation.Nullable; + +import lombok.Data; + +@Data +public class StatefulUuidStateId { + @Nullable + private String type; + private String uuid; + private String from; + + public static StatefulUuidStateId createUuidStateId(String uuid, String from) { + StatefulUuidStateId statefulUuidStateId = new StatefulUuidStateId(); + statefulUuidStateId.uuid = uuid; + statefulUuidStateId.from = from; + return statefulUuidStateId; + } + + public static StatefulUuidStateId createTypeUuidStateId(String type, String uuid, String from) { + StatefulUuidStateId uuidStateId = createUuidStateId(uuid, from); + uuidStateId.type = type; + return uuidStateId; + } +} From 600276e2044626950f82ffef37d2a800072b758f Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Mon, 15 Jul 2024 11:25:03 +0800 Subject: [PATCH 34/64] fix(web): modify tomcat keepAliveTimeout to 70 seconds (#2964) * fix tomcat keepAliveTimeout * fix tomcat keepAliveTimeout --- server/odc-server/src/main/resources/config/application.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/server/odc-server/src/main/resources/config/application.yml b/server/odc-server/src/main/resources/config/application.yml index 4c0682092b..eccab8a789 100644 --- a/server/odc-server/src/main/resources/config/application.yml +++ b/server/odc-server/src/main/resources/config/application.yml @@ -58,6 +58,7 @@ server: tomcat: max-threads: 500 min-spare-threads: 18 + keep-alive-timeout: 70000 swagger: enabled: false From 82da905fd727a9c90d3aa3ddadef0eb165b301f1 Mon Sep 17 00:00:00 2001 From: LioRoger Date: Mon, 15 Jul 2024 13:52:08 +0800 Subject: [PATCH 35/64] fix(osc): the online schema change blocked when rate limiter modified before swap table action (#2908) * Fix issue of online schema change module. when rate limitor is changed, scheduler task save the wrong value of rate limitor, that will cause data process's loop restart which block the following actions * fix typo --- .../DefaultOnlineSchemaChangeTaskHandler.java | 46 +++++++++++++++---- .../onlineschemachange/OscService.java | 6 +++ .../ScheduleCheckOmsProjectValve.java | 35 ++++++++++++-- 3 files changed, 73 insertions(+), 14 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/DefaultOnlineSchemaChangeTaskHandler.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/DefaultOnlineSchemaChangeTaskHandler.java index 4ffaba9fed..afdff98764 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/DefaultOnlineSchemaChangeTaskHandler.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/DefaultOnlineSchemaChangeTaskHandler.java @@ -69,6 +69,7 @@ import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeScheduleTaskParameters; import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeScheduleTaskResult; import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeSqlType; +import com.oceanbase.odc.service.onlineschemachange.model.RateLimiterConfig; import com.oceanbase.odc.service.onlineschemachange.oms.openapi.OmsProjectOpenApiService; import com.oceanbase.odc.service.onlineschemachange.oms.request.OmsProjectControlRequest; import com.oceanbase.odc.service.onlineschemachange.pipeline.BaseCreateOmsProjectValve; @@ -195,8 +196,8 @@ private void doStart(Long scheduleTaskId, OscValveContext valveContext) { @Override public void complete(@NonNull Long scheduleId, @NonNull Long scheduleTaskId) { ScheduleTaskEntity scheduleTask = scheduleTaskService.nullSafeGetById(scheduleTaskId); - OnlineSchemaChangeScheduleTaskParameters parameters = JsonUtils.fromJson(scheduleTask.getParametersJson(), - OnlineSchemaChangeScheduleTaskParameters.class); + OnlineSchemaChangeScheduleTaskParameters parameters = + parseOnlineSchemaChangeScheduleTaskParameters(scheduleTask.getParametersJson()); // check task is expired Duration between = Duration.between(scheduleTask.getCreateTime().toInstant(), Instant.now()); log.info("Schedule id {} to check schedule task status with schedule task id {}", scheduleId, scheduleTaskId); @@ -216,8 +217,8 @@ public void complete(@NonNull Long scheduleId, @NonNull Long scheduleTaskId) { @Override public void terminate(@NonNull Long scheduleId, @NonNull Long scheduleTaskId) { ScheduleTaskEntity scheduleTask = scheduleTaskService.nullSafeGetById(scheduleTaskId); - OnlineSchemaChangeScheduleTaskParameters parameters = JsonUtils.fromJson(scheduleTask.getParametersJson(), - OnlineSchemaChangeScheduleTaskParameters.class); + OnlineSchemaChangeScheduleTaskParameters parameters = + parseOnlineSchemaChangeScheduleTaskParameters(scheduleTask.getParametersJson()); completeHandler.onOscScheduleTaskCancel(parameters.getOmsProjectId(), parameters.getUid(), scheduleId, scheduleTaskId); } @@ -246,8 +247,8 @@ private void doComplete(Long scheduleId, Long scheduleTaskId, ScheduleTaskEntity } ScheduleEntity scheduleEntity = scheduleService.nullSafeGetById(scheduleId); - OnlineSchemaChangeParameters onlineSchemaChangeParameters = JsonUtils.fromJson( - scheduleEntity.getJobParametersJson(), OnlineSchemaChangeParameters.class); + OnlineSchemaChangeParameters onlineSchemaChangeParameters = + parseOnlineSchemaChangeParameters(scheduleEntity.getJobParametersJson()); if (onlineSchemaChangeParameters.getErrorStrategy() == TaskErrorStrategy.CONTINUE) { log.info("Because error strategy is continue, so schedule next task"); // schedule next task @@ -372,11 +373,11 @@ private OscValveContext getOscValveContext(Long scheduleId, Long scheduleTaskId) private OscValveContext createValveContext(Long scheduleId, Long scheduleTaskId) { ScheduleEntity scheduleEntity = scheduleService.nullSafeGetById(scheduleId); - OnlineSchemaChangeParameters onlineSchemaChangeParameters = JsonUtils.fromJson( - scheduleEntity.getJobParametersJson(), OnlineSchemaChangeParameters.class); + OnlineSchemaChangeParameters onlineSchemaChangeParameters = + parseOnlineSchemaChangeParameters(scheduleEntity.getJobParametersJson()); ScheduleTaskEntity scheduleTaskEntity = scheduleTaskService.nullSafeGetById(scheduleTaskId); - OnlineSchemaChangeScheduleTaskParameters oscScheduleTaskParameters = JsonUtils.fromJson( - scheduleTaskEntity.getParametersJson(), OnlineSchemaChangeScheduleTaskParameters.class); + OnlineSchemaChangeScheduleTaskParameters oscScheduleTaskParameters = + parseOnlineSchemaChangeScheduleTaskParameters(scheduleTaskEntity.getParametersJson()); OscValveContext valveContext = new OscValveContext(); valveContext.setSchedule(scheduleEntity); @@ -468,4 +469,29 @@ private void dropNewTableIfExits(OnlineSchemaChangeScheduleTaskParameters taskPa } } + /** + * parse and set compatible fields + * + * @param jsonStr + * @return + */ + private OnlineSchemaChangeParameters parseOnlineSchemaChangeParameters(String jsonStr) { + OnlineSchemaChangeParameters onlineSchemaChangeParameters = JsonUtils.fromJson( + jsonStr, OnlineSchemaChangeParameters.class); + // correct null value to default RateLimiterConfig object + if (null == onlineSchemaChangeParameters.getRateLimitConfig()) { + onlineSchemaChangeParameters.setRateLimitConfig(new RateLimiterConfig()); + } + return onlineSchemaChangeParameters; + } + + private OnlineSchemaChangeScheduleTaskParameters parseOnlineSchemaChangeScheduleTaskParameters(String jsonStr) { + OnlineSchemaChangeScheduleTaskParameters onlineSchemaChangeScheduleTaskParameters = JsonUtils.fromJson( + jsonStr, OnlineSchemaChangeScheduleTaskParameters.class); + // correct null value to default RateLimiterConfig object + if (null == onlineSchemaChangeScheduleTaskParameters.getRateLimitConfig()) { + onlineSchemaChangeScheduleTaskParameters.setRateLimitConfig(new RateLimiterConfig()); + } + return onlineSchemaChangeScheduleTaskParameters; + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OscService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OscService.java index 48498b5427..b43284f4b0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OscService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OscService.java @@ -19,6 +19,7 @@ import java.util.Objects; import java.util.Optional; +import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -178,6 +179,11 @@ public boolean updateRateLimiterConfig(UpdateRateLimiterConfigRequest req) { "modify task rate limiter is unsupported when task is terminated"); OnlineSchemaChangeTaskResult taskResult = JsonUtils.fromJson(task.getResultJson(), new TypeReference() {}); + // Task result may be empty cause task may hasn't been scheduled. Remind user raise this request + // latter. + PreConditions.validArgumentState(null != taskResult && CollectionUtils.isNotEmpty(taskResult.getTasks()), + ErrorCodes.BadRequest, new Object[] {req.getFlowInstanceId()}, + "Task has not been scheduled, please retry update rate limiter latter"); Long scheduleId = Long.parseLong(taskResult.getTasks().get(0).getJobName()); ScheduleEntity scheduleEntity = scheduleService.nullSafeGetById(scheduleId); OnlineSchemaChangeParameters parameters = JsonUtils.fromJson( diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/pipeline/ScheduleCheckOmsProjectValve.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/pipeline/ScheduleCheckOmsProjectValve.java index 2f05969b67..45c76999d4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/pipeline/ScheduleCheckOmsProjectValve.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/pipeline/ScheduleCheckOmsProjectValve.java @@ -23,12 +23,14 @@ import com.google.common.collect.Lists; import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.common.util.tableformat.Table; import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.metadb.schedule.ScheduleTaskEntity; import com.oceanbase.odc.metadb.schedule.ScheduleTaskRepository; import com.oceanbase.odc.service.onlineschemachange.configuration.OnlineSchemaChangeProperties; +import com.oceanbase.odc.service.onlineschemachange.exception.OmsException; import com.oceanbase.odc.service.onlineschemachange.exception.OscException; import com.oceanbase.odc.service.onlineschemachange.logger.DefaultTableFactory; import com.oceanbase.odc.service.onlineschemachange.model.FullVerificationResult; @@ -102,7 +104,11 @@ public void invoke(ValveContext valveContext) { return null; }) .getCheckerResult(); - updateOmsProjectConfig(scheduleTask.getId(), taskParameter, inputParameters, progress.getStatus()); + // If config update is in processing, wait until the process done + // This behavior may cause process blocked if OMS update failed or resume failed + if (updateOmsProjectConfig(scheduleTask.getId(), taskParameter, inputParameters, progress.getStatus())) { + return; + } OnlineSchemaChangeScheduleTaskResult result = new OnlineSchemaChangeScheduleTaskResult(taskParameter); @@ -119,11 +125,13 @@ public void invoke(ValveContext valveContext) { context.getParameter().getSwapTableType(), scheduleTask); } - private void updateOmsProjectConfig(Long scheduleTaskId, OnlineSchemaChangeScheduleTaskParameters taskParameters, + private boolean updateOmsProjectConfig(Long scheduleTaskId, OnlineSchemaChangeScheduleTaskParameters taskParameters, OnlineSchemaChangeParameters inputParameters, OmsProjectStatusEnum omsProjectStatus) { // if rate limiter parameters is changed, try to stop and restart project if (Objects.equals(inputParameters.getRateLimitConfig(), taskParameters.getRateLimitConfig())) { - return; + log.info("Rate limiter not changed,rateLimiterConfig = {}, update oms project not required", + inputParameters.getRateLimitConfig()); + return false; } log.info("Input rate limiter has changed, currentOmsProjectStatus={}, rateLimiterConfig={}, " + "oldRateLimiterConfig={}.", @@ -153,6 +161,7 @@ private void updateOmsProjectConfig(Long scheduleTaskId, OnlineSchemaChangeSched taskParameters.getOmsProjectId(), scheduleTaskId, e); } } + return true; } private void doUpdateOmsProjectConfig(Long scheduleTaskId, OnlineSchemaChangeScheduleTaskParameters taskParameters, @@ -173,7 +182,12 @@ private void doUpdateOmsProjectConfig(Long scheduleTaskId, OnlineSchemaChangeSch log.info("Try to update oms project, omsProjectId={}, scheduleTaskId={}," + " request={}.", taskParameters.getOmsProjectId(), scheduleTaskId, JsonUtils.toJson(request)); - projectOpenApiService.updateProjectConfig(request); + try { + projectOpenApiService.updateProjectConfig(request); + } catch (OmsException omsException) { + throwOrIgnore(omsException); + } + log.info("Update oms project completed, Try to resume project, omsProjectId={}," + " scheduleTaskId={}", taskParameters.getOmsProjectId(), scheduleTaskId); @@ -188,6 +202,19 @@ private void doUpdateOmsProjectConfig(Long scheduleTaskId, OnlineSchemaChangeSch } } + /** + * Lower version OMS may not support updateConfig api Ignore this exception and continue resume + * migrate project + * + * @param omsException + */ + private void throwOrIgnore(OmsException omsException) { + if (!StringUtils.containsIgnoreCase(omsException.getMessage(), + "Unsupported action: UpdateProjectConfig")) { + throw omsException; + } + } + private void handleOmsProjectStepResult(ValveContext valveContext, ProjectStepResult projectStepResult, OnlineSchemaChangeScheduleTaskResult result, SwapTableType swapTableType, ScheduleTaskEntity scheduleTask) { From 3edc63b29fca3898c690d37b5fb8006bc3efe4f0 Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Wed, 17 Jul 2024 17:24:57 +0800 Subject: [PATCH 36/64] fix(stateful): remove wrong condition (#2975) * remove wrong condition * remove wrong condition --- .../odc/service/state/StatefulUuidStateIdGenerator.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/state/StatefulUuidStateIdGenerator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/state/StatefulUuidStateIdGenerator.java index 9730146149..068eb49685 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/state/StatefulUuidStateIdGenerator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/state/StatefulUuidStateIdGenerator.java @@ -19,7 +19,6 @@ import java.util.Base64; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import com.oceanbase.odc.common.json.JsonUtils; @@ -28,7 +27,6 @@ import com.oceanbase.odc.service.state.model.StatefulUuidStateId; @Component -@ConditionalOnProperty(value = {"odc.web.stateful-route.enabled"}, havingValue = "true") public class StatefulUuidStateIdGenerator { @Autowired From b7215e26cb4386bdf5439b920fa525b74bbf9f6b Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Fri, 19 Jul 2024 18:08:54 +0800 Subject: [PATCH 37/64] fix(security): update oauth2 client version (#2981) * fix listBuiltinSnippets can't reach * upgrade oauth2 version --- pom.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index a21ba229d5..9b62de6863 100644 --- a/pom.xml +++ b/pom.xml @@ -45,7 +45,6 @@ 5.3.26 5.7.12 - 5.7.5 5.5.18 2.7.12 5.6.3.Final @@ -426,7 +425,7 @@ org.springframework.security spring-security-oauth2-client - ${spring-security-oauth2-client.version} + ${spring-security.version} org.springframework.security From 50f1f1ecbca89a1f6d9e412c362a9b3619fe4e8d Mon Sep 17 00:00:00 2001 From: IL MARE Date: Mon, 22 Jul 2024 11:53:56 +0800 Subject: [PATCH 38/64] fix(pre-check): failed to get sql check result when the check result file is not on this machine (#2943) --- .../com/oceanbase/odc/service/flow/FlowTaskInstanceService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceService.java index 3e21e4e3d9..5c8b8f882d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceService.java @@ -313,7 +313,7 @@ public List getResult( if (!this.dispatchChecker.isThisMachine(info)) { DispatchResponse response = requestDispatcher.forward(info.getHost(), info.getPort()); return response.getContentByType( - new TypeReference>() {}).getData().getContents(); + new TypeReference>() {}).getData().getContents(); } String dir = FileManager.generateDir(FileBucket.PRE_CHECK) + File.separator + taskId; String fileName; From 28e54d51f155e31b33bd2d24047717eae1bc201b Mon Sep 17 00:00:00 2001 From: IL MARE Date: Mon, 22 Jul 2024 14:09:57 +0800 Subject: [PATCH 39/64] fix(global-search): unable to stop data object synchronization (#2928) * database sync process can not stop * response to cr comment --- .../connection/DatabaseRepositoryTest.java | 85 +++++++++++++++++++ .../metadb/connection/DatabaseRepository.java | 9 ++ .../connection/database/DatabaseService.java | 14 +++ .../database/DatabaseSyncManager.java | 1 + .../db/schema/DBSchemaIndexService.java | 1 + .../db/schema/DBSchemaSyncScheduler.java | 12 ++- .../db/schema/DBSchemaSyncTaskManager.java | 7 +- .../db/schema/GlobalSearchProperties.java | 51 +++++++++++ 8 files changed, 172 insertions(+), 8 deletions(-) create mode 100644 server/integration-test/src/test/java/com/oceanbase/odc/metadb/connection/DatabaseRepositoryTest.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/GlobalSearchProperties.java diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/connection/DatabaseRepositoryTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/connection/DatabaseRepositoryTest.java new file mode 100644 index 0000000000..2d34beb4da --- /dev/null +++ b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/connection/DatabaseRepositoryTest.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.metadb.connection; + +import java.util.Arrays; +import java.util.Date; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import com.oceanbase.odc.ServiceTestEnv; +import com.oceanbase.odc.service.db.schema.model.DBObjectSyncStatus; +import com.oceanbase.odc.test.tool.TestRandom; + +/** + * Test cases for {@link DatabaseRepository} + * + * @author yh263208 + * @date 2024-07-08 14:29 + * @since ODC_release_4.3.1 + */ +public class DatabaseRepositoryTest extends ServiceTestEnv { + + @Autowired + private DatabaseRepository databaseRepository; + + @Before + public void setUp() { + this.databaseRepository.deleteAll(); + } + + @Test + public void setObjectSyncStatusByObjectLastSyncTimeBefore_noObjectMatched_setNothing() { + Date syncTime = new Date(System.currentTimeMillis() - 86400); + DatabaseEntity d1 = TestRandom.nextObject(DatabaseEntity.class); + d1.setObjectLastSyncTime(syncTime); + d1.setObjectSyncStatus(DBObjectSyncStatus.SYNCED); + DatabaseEntity d2 = TestRandom.nextObject(DatabaseEntity.class); + d2.setObjectLastSyncTime(syncTime); + d2.setObjectSyncStatus(DBObjectSyncStatus.SYNCED); + this.databaseRepository.saveAll(Arrays.asList(d1, d2)); + int affectRows = this.databaseRepository.setObjectSyncStatusByObjectSyncStatusAndObjectLastSyncTimeBefore( + DBObjectSyncStatus.SYNCING, DBObjectSyncStatus.SYNCING, + new Date(System.currentTimeMillis() - 86400 * 2)); + Assert.assertEquals(0, affectRows); + } + + @Test + public void setObjectSyncStatusByObjectLastSyncTimeBefore_oneObjectMatched_setSucceed() { + DatabaseEntity d1 = TestRandom.nextObject(DatabaseEntity.class); + d1.setObjectLastSyncTime(new Date()); + d1.setObjectSyncStatus(DBObjectSyncStatus.SYNCED); + DatabaseEntity d2 = TestRandom.nextObject(DatabaseEntity.class); + d2.setObjectLastSyncTime(new Date(System.currentTimeMillis() - 86400)); + d2.setObjectSyncStatus(DBObjectSyncStatus.SYNCED); + this.databaseRepository.saveAll(Arrays.asList(d1, d2)); + this.databaseRepository.setObjectSyncStatusByObjectSyncStatusAndObjectLastSyncTimeBefore( + DBObjectSyncStatus.INITIALIZED, DBObjectSyncStatus.SYNCED, + new Date(System.currentTimeMillis() - 86400 / 2)); + Set actual = this.databaseRepository.findAll().stream() + .map(DatabaseEntity::getObjectSyncStatus).collect(Collectors.toSet()); + Set expect = new HashSet<>( + Arrays.asList(DBObjectSyncStatus.SYNCED, DBObjectSyncStatus.INITIALIZED)); + Assert.assertEquals(expect, actual); + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseRepository.java index e4b09a1127..038ecd01a0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseRepository.java @@ -71,6 +71,14 @@ public interface DatabaseRepository extends JpaRepository, nativeQuery = true) int setObjectSyncStatusByIdIn(@Param("ids") Collection ids, @Param("status") DBObjectSyncStatus status); + @Modifying + @Transactional + @Query(value = "update connect_database t set t.object_sync_status = :#{#status.name()} " + + "where t.object_sync_status = :#{#originalStatus.name()} and t.object_last_sync_time < :syncTime", + nativeQuery = true) + int setObjectSyncStatusByObjectSyncStatusAndObjectLastSyncTimeBefore(@Param("status") DBObjectSyncStatus status, + @Param("originalStatus") DBObjectSyncStatus originalStatus, @Param("syncTime") Date syncTime); + @Modifying @Transactional @Query(value = "update connect_database t set t.object_sync_status = :#{#status.name()}, t.object_last_sync_time = :syncTime where t.id = :id", @@ -95,4 +103,5 @@ int setObjectLastSyncTimeAndStatusById(@Param("id") Long id, @Param("syncTime") @Query(value = "update connect_database t set t.project_id = null where t.project_id = :projectId", nativeQuery = true) int setProjectIdToNull(@Param("projectId") Long projectId); + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java index 00a815b843..dd4e693135 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java @@ -100,6 +100,7 @@ import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.db.DBSchemaService; import com.oceanbase.odc.service.db.schema.DBSchemaSyncTaskManager; +import com.oceanbase.odc.service.db.schema.GlobalSearchProperties; import com.oceanbase.odc.service.db.schema.model.DBObjectSyncStatus; import com.oceanbase.odc.service.db.schema.syncer.DBSchemaSyncProperties; import com.oceanbase.odc.service.iam.HorizontalDataPermissionValidator; @@ -208,6 +209,9 @@ public class DatabaseService { @Autowired private DBSchemaSyncProperties dbSchemaSyncProperties; + @Autowired + private GlobalSearchProperties globalSearchProperties; + @Transactional(rollbackFor = Exception.class) @SkipAuthorize("internal authenticated") public Database detail(@NonNull Long id) { @@ -462,6 +466,7 @@ public boolean deleteDatabases(@NonNull DeleteDatabasesReq req) { public Boolean syncDataSourceSchemas(@NonNull Long dataSourceId) throws InterruptedException { Boolean res = internalSyncDataSourceSchemas(dataSourceId); try { + refreshExpiredPendingDBObjectStatus(); dbSchemaSyncTaskManager .submitTaskByDataSource(connectionService.getBasicWithoutPermissionCheck(dataSourceId)); } catch (Exception e) { @@ -794,6 +799,15 @@ public void updateObjectLastSyncTimeAndStatus(@NotNull Long databaseId, databaseRepository.setObjectLastSyncTimeAndStatusById(databaseId, new Date(), status); } + @SkipAuthorize("odc internal usage") + @Transactional(rollbackFor = Exception.class) + public void refreshExpiredPendingDBObjectStatus() { + Date syncDate = new Date(System.currentTimeMillis() - this.globalSearchProperties.getMaxPendingMillis()); + int affectRows = this.databaseRepository.setObjectSyncStatusByObjectSyncStatusAndObjectLastSyncTimeBefore( + DBObjectSyncStatus.INITIALIZED, DBObjectSyncStatus.PENDING, syncDate); + log.info("Refresh outdated pending objects status, syncDate={}, affectRows={}", syncDate, affectRows); + } + private void checkPermission(Long projectId, Long dataSourceId) { if (Objects.isNull(projectId) && Objects.isNull(dataSourceId)) { throw new AccessDeniedException("invalid projectId or dataSourceId"); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseSyncManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseSyncManager.java index 6d9f71d19c..332c3fd493 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseSyncManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseSyncManager.java @@ -79,6 +79,7 @@ public Future submitSyncDataSourceAndDBSchemaTask(@NonNull ConnectionCo return doExecute(() -> executor.submit(() -> { Boolean res = syncDBForDataSource(connection); try { + databaseService.refreshExpiredPendingDBObjectStatus(); dbSchemaSyncTaskManager.submitTaskByDataSource(connection); } catch (Exception e) { log.warn("Failed to submit sync database schema task for datasource id={}", connection.getId(), e); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaIndexService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaIndexService.java index 897b72f412..9fc1f21177 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaIndexService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaIndexService.java @@ -198,6 +198,7 @@ public QueryDBObjectResp listDatabaseObjects(@NonNull @Valid QueryDBObjectParams } public Boolean syncDatabaseObjects(@NonNull @Valid SyncDBObjectReq req) { + this.databaseService.refreshExpiredPendingDBObjectStatus(); Set databases = new HashSet<>(); if (req.getResourceType() == ResourceType.ODC_CONNECTION) { Set dbs = new HashSet<>(databaseService.listExistDatabasesByConnectionId(req.getResourceId())); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncScheduler.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncScheduler.java index 134598fdc7..bc50c4e668 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncScheduler.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncScheduler.java @@ -25,7 +25,6 @@ import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.integration.jdbc.lock.JdbcLockRegistry; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @@ -38,6 +37,7 @@ import com.oceanbase.odc.service.config.UserConfigService; import com.oceanbase.odc.service.config.model.Configuration; import com.oceanbase.odc.service.connection.ConnectionService; +import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import lombok.extern.slf4j.Slf4j; @@ -57,6 +57,9 @@ public class DBSchemaSyncScheduler { @Autowired private ConnectionService connectionService; + @Autowired + private DatabaseService databaseService; + @Autowired private OrganizationRepository organizationRepository; @@ -66,15 +69,15 @@ public class DBSchemaSyncScheduler { @Autowired private JdbcLockRegistry jdbcLockRegistry; - @Value("${odc.database.schema.global-search.enabled:true}") - private boolean enableGlobalSearch; + @Autowired + private GlobalSearchProperties globalSearchProperties; private static final String LOCK_KEY = "db-schema-sync-schedule-lock"; private static final long LOCK_HOLD_TIME_SECONDS = 10; @Scheduled(cron = "${odc.database.schema.sync.cron-expression:0 0 2 * * ?}") public void sync() throws InterruptedException { - if (!enableGlobalSearch) { + if (!globalSearchProperties.isEnableGlobalSearch()) { log.info("Skip syncing database schema due to global search is disabled"); return; } @@ -93,6 +96,7 @@ public void sync() throws InterruptedException { } private void doSync() { + this.databaseService.refreshExpiredPendingDBObjectStatus(); List dataSources = new ArrayList<>(); Map> orgMap = organizationRepository.findAll().stream() .collect(Collectors.groupingBy(OrganizationEntity::getType)); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncTaskManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncTaskManager.java index e09b242366..7112cde3af 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncTaskManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncTaskManager.java @@ -25,7 +25,6 @@ import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.stereotype.Component; @@ -62,11 +61,11 @@ public class DBSchemaSyncTaskManager { @Autowired private DBSchemaSyncProperties syncProperties; - @Value("${odc.database.schema.global-search.enabled:true}") - private boolean enableGlobalSearch; + @Autowired + private GlobalSearchProperties globalSearchProperties; public void submitTaskByDatabases(@NonNull Collection databases) { - if (CollectionUtils.isEmpty(databases) || !enableGlobalSearch) { + if (CollectionUtils.isEmpty(databases) || !globalSearchProperties.isEnableGlobalSearch()) { return; } Set databaseIds = databases.stream().map(Database::getId).collect(Collectors.toSet()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/GlobalSearchProperties.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/GlobalSearchProperties.java new file mode 100644 index 0000000000..1024d4df08 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/GlobalSearchProperties.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.db.schema; + +import java.util.concurrent.TimeUnit; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.cloud.context.config.annotation.RefreshScope; +import org.springframework.context.annotation.Configuration; + +import lombok.Data; + +/** + * {@link GlobalSearchProperties} + * + * @author yh263208 + * @date 2024-07-08 14:51 + * @since ODC_release_4.3.1 + */ +@Data +@RefreshScope +@Configuration +public class GlobalSearchProperties { + + @Value("${odc.database.schema.global-search.enabled:true}") + private boolean enableGlobalSearch; + @Value("${odc.database.schema.global-search.max-pending-hours:1}") + private long maxPendingHours; + + public long getMaxPendingMillis() { + long maxPendingHours = this.maxPendingHours; + if (this.maxPendingHours <= 0) { + maxPendingHours = 1; + } + return TimeUnit.MILLISECONDS.convert(this.maxPendingHours, TimeUnit.HOURS); + } + +} From 5a5febf28e84ce8f2507e301851f22a1edfefbec Mon Sep 17 00:00:00 2001 From: IL MARE Date: Mon, 22 Jul 2024 14:12:28 +0800 Subject: [PATCH 40/64] fix(db-browser): failed to recognize the commit and rollback statement (#2985) * supports recognize commit/rollback statement * several bug fix --- .../tools/dbbrowser/parser/CacheElement.java | 38 +++++++++++++++++-- .../listener/MysqlModeSqlParserListener.java | 15 ++++++++ .../listener/OracleModeParserListener.java | 14 +++++++ .../listener/OracleModeSqlParserListener.java | 15 ++++++++ .../tools/dbbrowser/parser/PLParserTest.java | 15 ++++++++ .../tools/dbbrowser/parser/SqlParserTest.java | 28 ++++++++++++++ .../schema/obmysql/OBMySQLTableExtension.java | 6 ++- .../oboracle/OBOracleTableExtension.java | 6 ++- 8 files changed, 132 insertions(+), 5 deletions(-) diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/CacheElement.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/CacheElement.java index a99efcd38b..90faf841e6 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/CacheElement.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/CacheElement.java @@ -15,32 +15,64 @@ */ package com.oceanbase.tools.dbbrowser.parser; +import java.util.Objects; + import lombok.NonNull; class CacheElement { private final T value; private final Exception exception; + private final StackTraceElement[] traceElements; public CacheElement(@NonNull T value) { this.exception = null; this.value = value; + this.traceElements = null; } public CacheElement(@NonNull Exception exception) { this.value = null; this.exception = exception; + this.traceElements = getRealStackTraceElements(exception); } - public T get() { + public synchronized T get() { if (this.exception != null) { + Exception target = setStackTraceElements(this.exception); if (this.exception instanceof RuntimeException) { - throw (RuntimeException) this.exception; + throw (RuntimeException) target; } else { - throw new RuntimeException(this.exception); + throw new RuntimeException(target); } } return this.value; } + private StackTraceElement[] getRealStackTraceElements(Exception e) { + StackTraceElement elt = Thread.currentThread().getStackTrace()[3]; + int i; + for (i = 0; i < e.getStackTrace().length; i++) { + StackTraceElement item = e.getStackTrace()[i]; + if (Objects.equals(item.getClassName(), elt.getClassName()) + && Objects.equals(item.getMethodName(), elt.getMethodName())) { + i++; + break; + } + } + StackTraceElement[] target = new StackTraceElement[i]; + System.arraycopy(e.getStackTrace(), 0, target, 0, i); + return target; + } + + private Exception setStackTraceElements(Exception e) { + StackTraceElement[] elts = Thread.currentThread().getStackTrace(); + int length = this.traceElements.length; + StackTraceElement[] target = new StackTraceElement[length + elts.length - 4]; + System.arraycopy(this.traceElements, 0, target, 0, length); + System.arraycopy(elts, 4, target, length, elts.length - 4); + e.setStackTrace(target); + return e; + } + } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/MysqlModeSqlParserListener.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/MysqlModeSqlParserListener.java index 2f597d0ceb..40f3745470 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/MysqlModeSqlParserListener.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/MysqlModeSqlParserListener.java @@ -46,6 +46,7 @@ import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_tablegroup_stmtContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_tablespace_stmtContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_tenant_stmtContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Commit_stmtContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Create_database_stmtContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Create_function_stmtContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Create_table_stmtContext; @@ -71,6 +72,7 @@ import com.oceanbase.tools.sqlparser.obmysql.OBParser.No_table_select_with_order_and_limitContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Opt_for_update_waitContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Relation_factorContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Rollback_stmtContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Scope_or_scope_aliasContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Select_clause_set_with_order_and_limitContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Select_stmtContext; @@ -245,6 +247,18 @@ public void enterSet_charset_stmt(Set_charset_stmtContext ctx) { setSqlType(SqlType.SET); } + @Override + public void enterCommit_stmt(Commit_stmtContext ctx) { + setSqlType(SqlType.COMMIT); + this.dbObjectType = DBObjectType.OTHERS; + } + + @Override + public void enterRollback_stmt(Rollback_stmtContext ctx) { + setSqlType(SqlType.ROLLBACK); + this.dbObjectType = DBObjectType.OTHERS; + } + @Override public void enterSet_transaction_stmt(Set_transaction_stmtContext ctx) { setSqlType(SqlType.SET); @@ -537,4 +551,5 @@ public static class ColumnDefinition { private String name; private Boolean isStored; } + } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeParserListener.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeParserListener.java index cc371ef747..d0dc419617 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeParserListener.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeParserListener.java @@ -37,12 +37,14 @@ import com.oceanbase.tools.dbbrowser.parser.constant.SqlType; import com.oceanbase.tools.dbbrowser.util.StringUtils; import com.oceanbase.tools.sqlparser.oracle.PlSqlLexer; +import com.oceanbase.tools.sqlparser.oracle.PlSqlParser; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Alter_functionContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Alter_indexContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Alter_packageContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Alter_procedureContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Alter_userContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Anonymous_blockContext; +import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Commit_statementContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Create_function_bodyContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Create_indexContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Create_packageContext; @@ -66,6 +68,7 @@ import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Procedure_bodyContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Procedure_nameContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Procedure_specContext; +import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Rollback_statementContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Type_declarationContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Variable_declarationContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParserBaseListener; @@ -405,6 +408,16 @@ public void enterData_manipulation_language_statements(Data_manipulation_languag } } + @Override + public void enterCommit_statement(Commit_statementContext ctx) { + doRecord(SqlType.COMMIT, DBObjectType.OTHERS, null); + } + + @Override + public void enterRollback_statement(PlSqlParser.Rollback_statementContext ctx) { + doRecord(SqlType.ROLLBACK, DBObjectType.OTHERS, null); + } + private void doRecord(SqlType sqlType, DBObjectType dbObjectType, String rawPlName) { if (Objects.nonNull(sqlType) && Objects.isNull(this.sqlType)) { this.sqlType = sqlType; @@ -426,4 +439,5 @@ private String getDdl(ParserRuleContext ctx) { CharStream stream = start.getTokenSource().getInputStream(); return stream.getText(Interval.of(start.getStartIndex(), end)); } + } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeSqlParserListener.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeSqlParserListener.java index 70eaf38299..0ef7c736ac 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeSqlParserListener.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeSqlParserListener.java @@ -52,6 +52,7 @@ import com.oceanbase.tools.sqlparser.oboracle.OBParser.Alter_user_stmtContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Column_nameContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Column_name_listContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Commit_stmtContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Create_dblink_stmtContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Create_index_stmtContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Create_keystore_stmtContext; @@ -99,6 +100,7 @@ import com.oceanbase.tools.sqlparser.oboracle.OBParser.Out_of_line_constraintContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.References_clauseContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Relation_nameContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Rollback_stmtContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Scope_or_scope_aliasContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Select_stmtContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Select_with_hierarchical_queryContext; @@ -685,4 +687,17 @@ public void enterCreate_profile_stmt(Create_profile_stmtContext ctx) { setSqlType(SqlType.CREATE); this.dbObjectType = DBObjectType.OTHERS; } + + @Override + public void enterCommit_stmt(Commit_stmtContext ctx) { + setSqlType(SqlType.COMMIT); + this.dbObjectType = DBObjectType.OTHERS; + } + + @Override + public void enterRollback_stmt(Rollback_stmtContext ctx) { + setSqlType(SqlType.ROLLBACK); + this.dbObjectType = DBObjectType.OTHERS; + } + } diff --git a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/PLParserTest.java b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/PLParserTest.java index 1a2a96918c..5aded51d8a 100644 --- a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/PLParserTest.java +++ b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/PLParserTest.java @@ -23,6 +23,7 @@ import com.oceanbase.tools.dbbrowser.model.DBFunction; import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBProcedure; +import com.oceanbase.tools.dbbrowser.parser.constant.SqlType; import com.oceanbase.tools.dbbrowser.parser.result.ParseMysqlPLResult; import com.oceanbase.tools.dbbrowser.parser.result.ParseOraclePLResult; import com.oceanbase.tools.sqlparser.SyntaxErrorException; @@ -1168,4 +1169,18 @@ public void parseOracle_packageWithChineseChar_getNameSucceed() { Assert.assertEquals(1, actual.getFunctionList().size()); } + @Test + public void parseOracle_commitStmt_getSqlTypeSucceed() { + ParseOraclePLResult actual = PLParser.parseOracle("commit"); + Assert.assertEquals(DBObjectType.OTHERS, actual.getDbObjectType()); + Assert.assertEquals(SqlType.COMMIT, actual.getSqlType()); + } + + @Test + public void parseOracle_rollbackStmt_getSqlTypeSucceed() { + ParseOraclePLResult actual = PLParser.parseOracle("rollback"); + Assert.assertEquals(DBObjectType.OTHERS, actual.getDbObjectType()); + Assert.assertEquals(SqlType.ROLLBACK, actual.getSqlType()); + } + } diff --git a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/SqlParserTest.java b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/SqlParserTest.java index 9719888727..006996da61 100644 --- a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/SqlParserTest.java +++ b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/SqlParserTest.java @@ -256,4 +256,32 @@ public void testParseMysqlReference() { Assert.assertEquals("sys", parseSqlResult.getForeignConstraint().get(1).getReferenceSchemaName()); } + @Test + public void parseMysql_commitStmt_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseMysql("commit"); + Assert.assertEquals(DBObjectType.OTHERS, actual.getDbObjectType()); + Assert.assertEquals(SqlType.COMMIT, actual.getSqlType()); + } + + @Test + public void parseMysql_rollbackStmt_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseMysql("rollback"); + Assert.assertEquals(DBObjectType.OTHERS, actual.getDbObjectType()); + Assert.assertEquals(SqlType.ROLLBACK, actual.getSqlType()); + } + + @Test + public void parseOracle_commitStmt_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseOracle("commit"); + Assert.assertEquals(DBObjectType.OTHERS, actual.getDbObjectType()); + Assert.assertEquals(SqlType.COMMIT, actual.getSqlType()); + } + + @Test + public void parseOracle_rollbackStmt_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseOracle("rollback"); + Assert.assertEquals(DBObjectType.OTHERS, actual.getDbObjectType()); + Assert.assertEquals(SqlType.ROLLBACK, actual.getSqlType()); + } + } diff --git a/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/OBMySQLTableExtension.java b/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/OBMySQLTableExtension.java index e7574511c6..fec24ae055 100644 --- a/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/OBMySQLTableExtension.java +++ b/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/OBMySQLTableExtension.java @@ -80,7 +80,11 @@ public DBTable getDetail(@NonNull Connection connection, @NonNull String schemaN table.setDDL(ddl); table.setTableOptions(schemaAccessor.getTableOptions(schemaName, tableName)); table.setStats(getTableStats(connection, schemaName, tableName)); - table.setColumnGroups(schemaAccessor.listTableColumnGroups(schemaName, tableName)); + try { + table.setColumnGroups(schemaAccessor.listTableColumnGroups(schemaName, tableName)); + } catch (Exception e) { + // eat the exception + } return table; } diff --git a/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/OBOracleTableExtension.java b/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/OBOracleTableExtension.java index d4f850d070..bc9e78218a 100644 --- a/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/OBOracleTableExtension.java +++ b/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/OBOracleTableExtension.java @@ -80,7 +80,11 @@ public DBTable getDetail(@NonNull Connection connection, @NonNull String schemaN table.setTableOptions(tableOptions); table.setDDL(getTableDDL(connection, schemaName, tableName, parser, columns, tableOptions)); table.setStats(getTableStats(connection, schemaName, tableName)); - table.setColumnGroups(accessor.listTableColumnGroups(schemaName, tableName)); + try { + table.setColumnGroups(accessor.listTableColumnGroups(schemaName, tableName)); + } catch (Exception e) { + // eat the exception + } return table; } From 502bf7e105f76871689cbad1b25fe193d82b51e7 Mon Sep 17 00:00:00 2001 From: LuckyLeo Date: Mon, 22 Jul 2024 14:26:42 +0800 Subject: [PATCH 41/64] enhancement(result-set): only query table constraints when it's necessary (#2958) * lazy loading constraints * fix UT failed * add @NonNull annotation * rename class to Lazy --- .../com/oceanbase/odc/common/util/Lazy.java | 48 ++++++++++++++++++ .../oceanbase/odc/common/util/LazyTest.java | 49 +++++++++++++++++++ .../odc/service/dml/BaseDMLBuilder.java | 17 ++++--- .../odc/service/dml/MySQLDMLBuilder.java | 3 +- .../odc/service/dml/OracleDMLBuilder.java | 5 +- .../odc/service/dml/TableDataService.java | 4 +- .../odc/service/dml/UpdateGenerator.java | 2 +- 7 files changed, 117 insertions(+), 11 deletions(-) create mode 100644 server/odc-common/src/main/java/com/oceanbase/odc/common/util/Lazy.java create mode 100644 server/odc-common/src/test/java/com/oceanbase/odc/common/util/LazyTest.java diff --git a/server/odc-common/src/main/java/com/oceanbase/odc/common/util/Lazy.java b/server/odc-common/src/main/java/com/oceanbase/odc/common/util/Lazy.java new file mode 100644 index 0000000000..c68eb17312 --- /dev/null +++ b/server/odc-common/src/main/java/com/oceanbase/odc/common/util/Lazy.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.common.util; + +import java.util.function.Supplier; + +import lombok.NonNull; + +/** + * @author: liuyizhuo.lyz + * @date: 2024/7/11 + */ +public class Lazy { + private static final Object NO_INIT = new Object(); + + private volatile T target; + private final Supplier supplier; + + public Lazy(@NonNull Supplier supplier) { + target = (T) NO_INIT; + this.supplier = supplier; + } + + public T get() { + if (target == NO_INIT) { + synchronized (this) { + if (target == NO_INIT) { + target = supplier.get(); + } + } + } + return target; + } + +} diff --git a/server/odc-common/src/test/java/com/oceanbase/odc/common/util/LazyTest.java b/server/odc-common/src/test/java/com/oceanbase/odc/common/util/LazyTest.java new file mode 100644 index 0000000000..76c758f386 --- /dev/null +++ b/server/odc-common/src/test/java/com/oceanbase/odc/common/util/LazyTest.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.common.util; + +import org.junit.Assert; +import org.junit.Test; + +/** + * @author: liuyizhuo.lyz + * @date: 2024/7/11 + */ +public class LazyTest { + + @Test + public void test_LazyInitialize_Get_Success() { + DummyCounter counter = new DummyCounter(); + Assert.assertEquals(0, counter.cnt); + Lazy o = new Lazy<>(() -> { + counter.count(); + return counter; + }); + Assert.assertEquals(1, o.get().cnt); + Assert.assertEquals(1, o.get().cnt); + } + + private static class DummyCounter { + + private int cnt = 0; + + private void count() { + cnt++; + } + + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dml/BaseDMLBuilder.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dml/BaseDMLBuilder.java index 1f047dfa6e..b99dadf91a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dml/BaseDMLBuilder.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dml/BaseDMLBuilder.java @@ -27,6 +27,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import com.oceanbase.odc.common.util.Lazy; import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.session.ConnectionSessionUtil; import com.oceanbase.odc.service.db.browser.DBSchemaAccessors; @@ -52,12 +53,12 @@ abstract class BaseDMLBuilder implements DMLBuilder { private final String tableName; private final String schema; private final List modifyUnits; - private final List constraints; + private final Lazy> constraints; private final List whereColumns; protected final ConnectionSession connectionSession; public BaseDMLBuilder(@NonNull List modifyUnits, List whereColumns, - @NonNull ConnectionSession connectionSession, List constraints) { + @NonNull ConnectionSession connectionSession, Lazy> constraints) { Set schemas = modifyUnits.stream() .map(DataModifyUnit::getSchemaName) .filter(Objects::nonNull).collect(Collectors.toSet()); @@ -78,7 +79,9 @@ public BaseDMLBuilder(@NonNull List modifyUnits, List wh this.schema = null; } this.connectionSession = connectionSession; - this.constraints = constraints == null ? getConstraints(schema, tableName, connectionSession) : constraints; + this.constraints = + constraints == null ? new Lazy<>(() -> getConstraints(schema, tableName, connectionSession)) + : constraints; this.whereColumns = whereColumns; } @@ -135,7 +138,8 @@ public boolean containsPrimaryKeys() { public boolean containsUniqueKeys() { final Set columnNames = this.modifyUnits.stream() .map(DataModifyUnit::getColumnName).collect(Collectors.toSet()); - return this.constraints.stream() + return this.constraints.get() + .stream() .filter(c -> c.getType() == DBConstraintType.UNIQUE_KEY && c.getColumnNames() != null) .anyMatch(c -> columnNames.containsAll(c.getColumnNames())); } @@ -174,7 +178,8 @@ private boolean isFloat(String type) { } private boolean matchUniqueConstraint(DataModifyUnit u) { - Optional optional = this.constraints.stream() + Optional optional = this.constraints.get() + .stream() .filter(c -> c.getType() == DBConstraintType.UNIQUE_KEY).findFirst(); return optional.isPresent() && optional.get().getColumnNames().contains(u.getColumnName()); } @@ -182,7 +187,7 @@ private boolean matchUniqueConstraint(DataModifyUnit u) { private DBTableConstraint getPrimaryConstraint() { Map name2Units = this.modifyUnits.stream() .collect(Collectors.toMap(DataModifyUnit::getColumnName, unit -> unit)); - for (DBTableConstraint constraint : this.constraints) { + for (DBTableConstraint constraint : this.constraints.get()) { if (DBConstraintType.PRIMARY_KEY != constraint.getType()) { continue; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dml/MySQLDMLBuilder.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dml/MySQLDMLBuilder.java index 1df2228d72..808a8dbd7f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dml/MySQLDMLBuilder.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dml/MySQLDMLBuilder.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Set; +import com.oceanbase.odc.common.util.Lazy; import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.service.dml.model.DataModifyUnit; import com.oceanbase.tools.dbbrowser.model.DBTableConstraint; @@ -39,7 +40,7 @@ public class MySQLDMLBuilder extends BaseDMLBuilder { public MySQLDMLBuilder(@NonNull List modifyUnits, List whereColumns, - ConnectionSession connectionSession, List constraints) { + ConnectionSession connectionSession, Lazy> constraints) { super(modifyUnits, whereColumns, connectionSession, constraints); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dml/OracleDMLBuilder.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dml/OracleDMLBuilder.java index d4f9635b00..36f162abb8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dml/OracleDMLBuilder.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dml/OracleDMLBuilder.java @@ -21,6 +21,7 @@ import java.util.Objects; import java.util.Set; +import com.oceanbase.odc.common.util.Lazy; import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.shared.constant.OdcConstants; import com.oceanbase.odc.service.dml.model.DataModifyUnit; @@ -41,7 +42,7 @@ public class OracleDMLBuilder extends BaseDMLBuilder { public OracleDMLBuilder(@NonNull List modifyUnits, List whereColumns, - ConnectionSession connectionSession, List constraints) { + ConnectionSession connectionSession, Lazy> constraints) { super(modifyUnits, whereColumns, connectionSession, constraints); } @@ -88,7 +89,7 @@ private boolean containsODCInternalRowId() { @Override public boolean containsPrimaryKeyOrRowId() { - return super.containsPrimaryKeyOrRowId() || containsRowId() || containsODCInternalRowId(); + return containsRowId() || containsODCInternalRowId() || super.containsPrimaryKeyOrRowId(); } @Override diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dml/TableDataService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dml/TableDataService.java index 9c1c045081..1d41f631e1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dml/TableDataService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dml/TableDataService.java @@ -27,6 +27,7 @@ import org.springframework.stereotype.Service; import org.springframework.validation.annotation.Validated; +import com.oceanbase.odc.common.util.Lazy; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.session.ConnectionSession; @@ -73,7 +74,8 @@ public BatchDataModifyResp batchGetModifySql(@NotNull ConnectionSession connecti BatchDataModifyResp resp = new BatchDataModifyResp(); resp.setTableName(tableName); resp.setSchemaName(schemaName); - List constraints = BaseDMLBuilder.getConstraints(schemaName, tableName, connectionSession); + Lazy> constraints = new Lazy<>( + () -> BaseDMLBuilder.getConstraints(schemaName, tableName, connectionSession)); Map columnName2Column = getColumnName2Column(connectionSession, schemaName, tableName); StringBuilder sqlBuilder = new StringBuilder(); for (Row row : sortedRows) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dml/UpdateGenerator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dml/UpdateGenerator.java index 75343f13c0..65651a19de 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dml/UpdateGenerator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dml/UpdateGenerator.java @@ -90,7 +90,7 @@ public String generate() { .append(" where ") .append(oldSql) .append(";"); - this.affectMultiRows = !this.dmlBuilder.containsUniqueKeys() && !this.dmlBuilder.containsPrimaryKeyOrRowId(); + this.affectMultiRows = !this.dmlBuilder.containsPrimaryKeyOrRowId() && !this.dmlBuilder.containsUniqueKeys(); return sqlBuilder.toString(); } From 9b2e973f0c1b751c9b816d5201b9d8e1cc0b1cad Mon Sep 17 00:00:00 2001 From: LuckyLeo Date: Mon, 22 Jul 2024 17:17:54 +0800 Subject: [PATCH 42/64] fix(diagnose): failed to view query profile for distributed OB (#2945) * fix some bugs in cluster case * fix failed to explain some sql * fix UT failed * support DML to view query profile * format code * response to CR comments * catch kill query exception * 1. eat exception when there is no records in plan monitor 2. query sql audit without session ids --- .../shared/model/OBExecutionServerInfo.java | 32 ++++++ .../odc/core/shared/model/SqlPlanMonitor.java | 1 + .../oceanbase/odc/core/sql/util/OBUtils.java | 67 +++++++----- .../queryprofile/OBQueryProfileManager.java | 20 +++- .../session/ConnectConsoleService.java | 18 +++- ...a => OBQueryProfileExecutionListener.java} | 34 ++++-- .../session/model/AsyncExecuteContext.java | 2 +- .../session/model/SqlExecuteResult.java | 1 + .../api/SqlDiagnoseExtensionPoint.java | 4 +- .../mysql/MySQLDiagnoseExtensionPoint.java | 4 +- .../obmysql/OBMySQLDiagnoseExtension.java | 101 ++++++------------ .../obmysql/diagnose/DiagnoseUtil.java | 94 ++++++++++++++++ .../obmysql/diagnose/QueryProfileHelper.java | 7 +- .../oboracle/OBOracleDiagnoseExtension.java | 18 ++-- 14 files changed, 285 insertions(+), 118 deletions(-) create mode 100644 server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/OBExecutionServerInfo.java rename server/odc-service/src/main/java/com/oceanbase/odc/service/session/{OBExecutionListener.java => OBQueryProfileExecutionListener.java} (73%) diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/OBExecutionServerInfo.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/OBExecutionServerInfo.java new file mode 100644 index 0000000000..94a659b769 --- /dev/null +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/OBExecutionServerInfo.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.core.shared.model; + +import lombok.Data; + +/** + * @author: liuyizhuo.lyz + * @date: 2024/7/4 + */ +@Data +public class OBExecutionServerInfo { + + private String ip; + private String port; + private String tenantId; + private String planId; + +} diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/SqlPlanMonitor.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/SqlPlanMonitor.java index 79f826d18b..23bd38e9f6 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/SqlPlanMonitor.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/SqlPlanMonitor.java @@ -27,6 +27,7 @@ @Data public class SqlPlanMonitor { + private String conId; private String svrIp; private String svrPort; private String processName; diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/util/OBUtils.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/util/OBUtils.java index f5941bc07f..78bcfd7d59 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/util/OBUtils.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/util/OBUtils.java @@ -41,6 +41,7 @@ import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.exception.BadRequestException; import com.oceanbase.odc.core.shared.exception.UnexpectedException; +import com.oceanbase.odc.core.shared.model.OBExecutionServerInfo; import com.oceanbase.odc.core.shared.model.OBSqlPlan; import com.oceanbase.odc.core.shared.model.SqlExecDetail; import com.oceanbase.odc.core.shared.model.SqlPlanMonitor; @@ -391,7 +392,7 @@ public static String queryOBProxySessId(@NonNull Statement statement, @NonNull D String proxySessId = null; String sql = "select proxy_sessid from " + (dialectType.isMysql() ? "oceanbase" : "sys") - + ".v$ob_processlist where id = " + + ".gv$ob_processlist where id = " + connectionId; try (ResultSet rs = statement.executeQuery(sql)) { if (rs.next()) { @@ -407,7 +408,7 @@ public static List querySessionIdsByProxySessId(@NonNull Statement state SqlBuilder sqlBuilder = getBuilder(connectType) .append("select id from ") .append(dialectType.isMysql() ? "oceanbase" : "sys") - .append(".v$ob_processlist where proxy_sessid = ") + .append(".gv$ob_processlist where proxy_sessid = ") .append(proxySessId); List ids = new ArrayList<>(); try (ResultSet rs = statement.executeQuery(sqlBuilder.toString())) { @@ -428,7 +429,7 @@ public static String queryTraceIdFromASH(@NonNull Statement statement, SqlBuilder sqlBuilder = getBuilder(connectType) .append("select trace_id from ") .append(dialectType.isMysql() ? "oceanbase" : "sys") - .append(".v$active_session_history where session_id in (") + .append(".gv$active_session_history where session_id in (") .append(String.join(",", sessionIds)) .append(")") .append(dialectType.isMysql() ? " limit 1" : " and rownum=1"); @@ -444,51 +445,68 @@ public static String queryTraceIdFromASH(@NonNull Statement statement, * OceanBase only supports ASH views in versions higher than 4.0. Therefore, this method is not * applicable to earlier versions, please use sql_audit instead. */ - public static String queryPlanIdByTraceIdFromASH(@NonNull Statement statement, String traceId, - ConnectType connectType) - throws SQLException { + public static OBExecutionServerInfo queryPlanIdByTraceIdFromASH(@NonNull Statement statement, String traceId, + List sessionIds, ConnectType connectType) throws SQLException { DialectType dialectType = connectType.getDialectType(); SqlBuilder sqlBuilder = getBuilder(connectType) - .append("select plan_id from ") + .append("select svr_ip,svr_port,plan_id,con_id from ") .append(dialectType.isMysql() ? "oceanbase" : "sys") - .append(".v$active_session_history where trace_id=") - .value(traceId); + .append(".gv$active_session_history where trace_id=") + .value(traceId) + .append(" and session_id in (") + .append(String.join(",", sessionIds)) + .append(") and session_type='FOREGROUND' and plan_id!=0 ") + .append(dialectType.isMysql() ? "limit 1" : "and rownum<=1"); try (ResultSet rs = statement.executeQuery(sqlBuilder.toString())) { if (!rs.next()) { throw new SQLException("No result found in ASH."); } - return rs.getString(1); + OBExecutionServerInfo executorInfo = new OBExecutionServerInfo(); + executorInfo.setIp(rs.getString("svr_ip")); + executorInfo.setPort(rs.getString("svr_port")); + executorInfo.setPlanId(rs.getString("plan_id")); + executorInfo.setTenantId(rs.getString("con_id")); + return executorInfo; } } - public static String queryPlanIdByTraceIdFromAudit(@NonNull Statement statement, String traceId, - ConnectType connectType) - throws SQLException { + public static OBExecutionServerInfo queryPlanIdByTraceIdFromAudit(@NonNull Statement statement, String traceId, + ConnectType connectType) throws SQLException { DialectType dialectType = connectType.getDialectType(); SqlBuilder sqlBuilder = getBuilder(connectType) - .append("select plan_id from ") + .append("select svr_ip,svr_port,tenant_id,plan_id from ") .append(dialectType.isMysql() ? "oceanbase" : "sys") - .append(".v$ob_sql_audit where trace_id=") + .append(".gv$ob_sql_audit where trace_id=") .value(traceId) - .append(" and is_inner_sql=0"); + .append(" and length(query_sql)>0 and is_inner_sql=0 ") + .append(dialectType.isMysql() ? "limit 1" : "and rownum<=1"); try (ResultSet rs = statement.executeQuery(sqlBuilder.toString())) { if (!rs.next()) { throw new SQLException("No result found in sql_audit."); } - return rs.getString(1); + OBExecutionServerInfo executorInfo = new OBExecutionServerInfo(); + executorInfo.setIp(rs.getString("svr_ip")); + executorInfo.setPort(rs.getString("svr_port")); + executorInfo.setPlanId(rs.getString("plan_id")); + executorInfo.setTenantId(rs.getString("tenant_id")); + return executorInfo; } } - public static List queryOBSqlPlanByPlanId(@NonNull Statement statement, @NonNull String planId, - ConnectType connectType) throws SQLException { + public static List queryOBSqlPlanByPlanId(@NonNull Statement statement, + @NonNull OBExecutionServerInfo executorInfo, ConnectType connectType) throws SQLException { DialectType dialectType = connectType.getDialectType(); SqlBuilder sqlBuilder = getBuilder(connectType) .append("select id, parent_id, operator, object_owner, object_name, object_alias, ") .append("depth, cost, cardinality, ") .append("other, access_predicates, filter_predicates, projection, special_predicates from ") .append(dialectType.isMysql() ? "oceanbase" : "sys") - .append(".v$ob_sql_plan where plan_id=") - .value(planId) + .append(".gv$ob_sql_plan where plan_id=") + .value(executorInfo.getPlanId()) + .append(" and svr_ip=") + .value(executorInfo.getIp()) + .append(" and svr_port=") + .value(executorInfo.getPort()) .append(" order by id asc"); List records = new ArrayList<>(); try (ResultSet rs = statement.executeQuery(sqlBuilder.toString())) { @@ -511,7 +529,7 @@ public static List queryOBSqlPlanByPlanId(@NonNull Statement statemen records.add(plan); } } - Verify.notEmpty(records, "plan records"); + Verify.verify(!records.isEmpty(), "plan records expected not empty, plan id=" + executorInfo.getPlanId()); return records; } @@ -519,7 +537,7 @@ public static List querySqlPlanMonitorStats(@NonNull Statement s ConnectType connectType, Map statId2Name) throws SQLException { DialectType dialectType = connectType.getDialectType(); SqlBuilder sqlBuilder = getBuilder(connectType) - .append("select svr_ip,svr_port,first_refresh_time,last_refresh_time,first_change_time,") + .append("select con_id,svr_ip,svr_port,first_refresh_time,last_refresh_time,first_change_time,") .append("last_change_time,plan_line_id,starts,output_rows,db_time,user_io_wait_time,workarea_max_mem,") .append("workarea_max_tempseg,process_name,"); for (int i = 1; i <= 10; i++) { @@ -528,12 +546,13 @@ public static List querySqlPlanMonitorStats(@NonNull Statement s } sqlBuilder.append("current_timestamp(6) current_ts from ") .append(dialectType.isMysql() ? "oceanbase" : "sys") - .append(".v$sql_plan_monitor where trace_id=") + .append(".gv$sql_plan_monitor where trace_id=") .value(traceId); List records = new ArrayList<>(); try (ResultSet rs = statement.executeQuery(sqlBuilder.toString())) { while (rs.next()) { SqlPlanMonitor record = new SqlPlanMonitor(); + record.setConId(rs.getString("con_id")); record.setSvrIp(rs.getString("svr_ip")); record.setSvrPort(rs.getString("svr_port")); record.setProcessName(rs.getString("process_name")); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/OBQueryProfileManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/OBQueryProfileManager.java index 37d4bd2153..672de5a4ed 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/OBQueryProfileManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/queryprofile/OBQueryProfileManager.java @@ -20,6 +20,8 @@ import java.io.ByteArrayInputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -30,6 +32,7 @@ import org.springframework.util.StreamUtils; import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.common.util.VersionUtils; import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.session.ConnectionSessionUtil; @@ -38,6 +41,7 @@ import com.oceanbase.odc.core.shared.exception.UnexpectedException; import com.oceanbase.odc.core.sql.execute.cache.BinaryDataManager; import com.oceanbase.odc.core.sql.execute.cache.model.BinaryContentMetaData; +import com.oceanbase.odc.core.sql.util.OBUtils; import com.oceanbase.odc.plugin.connect.model.diagnose.SqlExplain; import com.oceanbase.odc.service.plugin.ConnectionPluginUtil; @@ -58,7 +62,7 @@ public class OBQueryProfileManager { @Qualifier("queryProfileMonitorExecutor") private ThreadPoolTaskExecutor executor; - public void submit(ConnectionSession session, @NonNull String traceId) { + public void submit(ConnectionSession session, @NonNull String traceId, List sessionIds) { executor.execute(() -> { if (session.isExpired() || ConnectionSessionUtil.getBinaryContentMetadata(session, PROFILE_KEY_PREFIX + traceId) != null) { @@ -68,7 +72,7 @@ public void submit(ConnectionSession session, @NonNull String traceId) { SqlExplain profile = session.getSyncJdbcExecutor(BACKEND_DS_KEY).execute( (ConnectionCallback) conn -> ConnectionPluginUtil .getDiagnoseExtension(session.getConnectType().getDialectType()) - .getQueryProfileByTraceId(conn, traceId)); + .getQueryProfileByTraceIdAndSessIds(conn, traceId, sessionIds)); BinaryDataManager binaryDataManager = ConnectionSessionUtil.getBinaryDataManager(session); BinaryContentMetaData metaData = binaryDataManager.write( new ByteArrayInputStream(JsonUtils.toJson(profile).getBytes())); @@ -94,10 +98,11 @@ public SqlExplain getProfile(@NonNull String traceId, ConnectionSession session) InputStream stream = ConnectionSessionUtil.getBinaryDataManager(session).read(metadata); return JsonUtils.fromJson(StreamUtils.copyToString(stream, StandardCharsets.UTF_8), SqlExplain.class); } + List sessionIds = getSessionIds(session); return session.getSyncJdbcExecutor(BACKEND_DS_KEY).execute( (StatementCallback) stmt -> ConnectionPluginUtil .getDiagnoseExtension(session.getConnectType().getDialectType()) - .getQueryProfileByTraceId(stmt.getConnection(), traceId)); + .getQueryProfileByTraceIdAndSessIds(stmt.getConnection(), traceId, sessionIds)); } catch (Exception e) { log.warn("Failed to get profile with OB trace_id={}.", traceId, e); throw new UnexpectedException( @@ -105,4 +110,13 @@ public SqlExplain getProfile(@NonNull String traceId, ConnectionSession session) } } + private List getSessionIds(ConnectionSession session) { + String proxySessId = ConnectionSessionUtil.getConsoleConnectionProxySessId(session); + if (StringUtils.isEmpty(proxySessId)) { + return Collections.singletonList(ConnectionSessionUtil.getConsoleConnectionId(session)); + } + return session.getSyncJdbcExecutor(BACKEND_DS_KEY).execute((StatementCallback>) stmt -> OBUtils + .querySessionIdsByProxySessId(stmt, proxySessId, session.getConnectType())); + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java index 6ed7f89265..4790a8580b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java @@ -316,7 +316,8 @@ public SqlAsyncExecuteResp streamExecute(@NotNull String sessionId, statementCallBack.setMaxCachedLines(sessionProperties.getResultSetMaxCachedLines()); statementCallBack.setLocale(LocaleContextHolder.getLocale()); if (connectionSession.getDialectType().isOceanbase() && sqlTuples.size() <= 10) { - statementCallBack.getListeners().add(new OBExecutionListener(connectionSession, profileManager)); + statementCallBack.getListeners() + .add(new OBQueryProfileExecutionListener(connectionSession, profileManager)); } Future> futureResult = connectionSession.getAsyncJdbcExecutor( @@ -354,6 +355,13 @@ public AsyncExecuteResultResp getMoreResults(@NotNull String sessionId, String r return new AsyncExecuteResultResp(shouldRemoveContext, context, results); } catch (Exception e) { shouldRemoveContext = true; + // Front-end would stop getting more results if there is an exception. In this case the left queries + // should be killed. + try { + killCurrentQuery(sessionId); + } catch (Exception ex) { + log.warn("Failed to kill query. Session id={}. Request id={}", sessionId, requestId); + } throw e; } finally { if (shouldRemoveContext) { @@ -589,6 +597,14 @@ private SqlExecuteResult generateResult(@NonNull ConnectionSession connectionSes } catch (Exception e) { log.warn("Failed to init warning message", e); } + try { + String version = ConnectionSessionUtil.getVersion(connectionSession); + result.setWithQueryProfile(OBQueryProfileExecutionListener + .isSqlTypeSupportProfile(generalResult.getSqlTuple()) && + VersionUtils.isGreaterThanOrEqualsTo(version, OBQueryProfileManager.ENABLE_QUERY_PROFILE_VERSION)); + } catch (Exception e) { + result.setWithQueryProfile(false); + } return result; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/OBExecutionListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/OBQueryProfileExecutionListener.java similarity index 73% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/session/OBExecutionListener.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/session/OBQueryProfileExecutionListener.java index 46179c450d..a68f93b744 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/OBExecutionListener.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/OBQueryProfileExecutionListener.java @@ -31,24 +31,29 @@ import com.oceanbase.odc.core.session.ConnectionSessionUtil; import com.oceanbase.odc.core.sql.execute.model.JdbcGeneralResult; import com.oceanbase.odc.core.sql.execute.model.SqlTuple; -import com.oceanbase.odc.core.sql.parser.AbstractSyntaxTree; import com.oceanbase.odc.core.sql.util.OBUtils; import com.oceanbase.odc.service.queryprofile.OBQueryProfileManager; import com.oceanbase.odc.service.session.model.AsyncExecuteContext; +import com.oceanbase.tools.dbbrowser.parser.ParserUtil; +import com.oceanbase.tools.dbbrowser.parser.constant.GeneralSqlType; import com.oceanbase.tools.dbbrowser.parser.constant.SqlType; +import com.oceanbase.tools.dbbrowser.parser.result.BasicResult; + +import lombok.extern.slf4j.Slf4j; /** * @author: liuyizhuo.lyz * @date: 2024/4/23 */ -public class OBExecutionListener implements SqlExecutionListener { +@Slf4j +public class OBQueryProfileExecutionListener implements SqlExecutionListener { private static final Long DEFAULT_QUERY_TRACE_ID_WAIT_MILLIS = 1100L; private final ConnectionSession session; private final List sessionIds; private final OBQueryProfileManager profileManager; - public OBExecutionListener(ConnectionSession session, OBQueryProfileManager profileManager) { + public OBQueryProfileExecutionListener(ConnectionSession session, OBQueryProfileManager profileManager) { this.session = session; this.profileManager = profileManager; sessionIds = getSessionIds(); @@ -60,8 +65,9 @@ public void onExecutionStart(SqlTuple sqlTuple, AsyncExecuteContext context) {} @Override public void onExecutionEnd(SqlTuple sqlTuple, List results, AsyncExecuteContext context) { JdbcGeneralResult firstResult = results.get(0); - if (StringUtils.isNotEmpty(firstResult.getTraceId()) && isSelect(sqlTuple)) { - profileManager.submit(session, firstResult.getTraceId()); + if (StringUtils.isNotEmpty(firstResult.getTraceId()) && isSqlTypeSupportProfile(sqlTuple) + && CollectionUtils.isNotEmpty(sessionIds)) { + profileManager.submit(session, firstResult.getTraceId(), sessionIds); } } @@ -69,7 +75,7 @@ public void onExecutionEnd(SqlTuple sqlTuple, List results, A public void onExecutionCancelled(SqlTuple sqlTuple, List results, AsyncExecuteContext context) {} public void onExecutionStartAfter(SqlTuple sqlTuple, AsyncExecuteContext context) { - if (CollectionUtils.isEmpty(sessionIds)) { + if (CollectionUtils.isEmpty(sessionIds) || !isSqlTypeSupportProfile(sqlTuple)) { return; } String traceId = session.getSyncJdbcExecutor(BACKEND_DS_KEY).execute((StatementCallback) stmt -> OBUtils @@ -92,14 +98,20 @@ private List getSessionIds() { if (StringUtils.isEmpty(proxySessId)) { return Collections.singletonList(ConnectionSessionUtil.getConsoleConnectionId(session)); } - return session.getSyncJdbcExecutor(CONSOLE_DS_KEY).execute((StatementCallback>) stmt -> OBUtils - .querySessionIdsByProxySessId(stmt, proxySessId, session.getConnectType())); + try { + return session.getSyncJdbcExecutor(CONSOLE_DS_KEY).execute((StatementCallback>) stmt -> OBUtils + .querySessionIdsByProxySessId(stmt, proxySessId, session.getConnectType())); + } catch (Exception e) { + log.warn("Failed to init session ids. Reason:{}", e.getMessage()); + return Collections.emptyList(); + } } - private boolean isSelect(SqlTuple sqlTuple) { + public static boolean isSqlTypeSupportProfile(SqlTuple sqlTuple) { try { - AbstractSyntaxTree ast = sqlTuple.getAst(); - return ast.getParseResult().getSqlType() == SqlType.SELECT; + BasicResult parseResult = sqlTuple.getAst().getParseResult(); + SqlType sqlType = parseResult.getSqlType(); + return ParserUtil.getGeneralSqlType(parseResult) == GeneralSqlType.DML || ParserUtil.isSelectType(sqlType); } catch (Exception e) { // eat exception return false; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/AsyncExecuteContext.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/AsyncExecuteContext.java index 048857d6b8..7bb07a021e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/AsyncExecuteContext.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/AsyncExecuteContext.java @@ -75,7 +75,7 @@ public List getMoreSqlExecutionResults(long timeoutMillis) { List copiedResults = new ArrayList<>(); long expect = System.currentTimeMillis() + timeoutMillis; - while (System.currentTimeMillis() <= expect && results.isEmpty()) { + while (!isFinished() && System.currentTimeMillis() <= expect && results.isEmpty()) { } while (!results.isEmpty()) { copiedResults.add(results.poll()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlExecuteResult.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlExecuteResult.java index 3ff582d7e4..4f0b27ef34 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlExecuteResult.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlExecuteResult.java @@ -106,6 +106,7 @@ public class SqlExecuteResult { private List whereColumns; private boolean withFullLinkTrace = false; private String traceEmptyReason; + private boolean withQueryProfile; @JsonIgnore private SqlTuple sqlTuple; diff --git a/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/SqlDiagnoseExtensionPoint.java b/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/SqlDiagnoseExtensionPoint.java index 0d2f75c10f..0b843f7545 100644 --- a/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/SqlDiagnoseExtensionPoint.java +++ b/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/SqlDiagnoseExtensionPoint.java @@ -18,6 +18,7 @@ import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; +import java.util.List; import org.pf4j.ExtensionPoint; @@ -43,6 +44,7 @@ public interface SqlDiagnoseExtensionPoint extends ExtensionPoint { SqlExecDetail getExecutionDetailBySql(Connection connection, @NonNull String sql) throws SQLException; - SqlExplain getQueryProfileByTraceId(Connection connection, @NonNull String traceId) throws SQLException; + SqlExplain getQueryProfileByTraceIdAndSessIds(Connection connection, @NonNull String traceId, + @NonNull List sessionIds) throws SQLException; } diff --git a/server/plugins/connect-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/connect/mysql/MySQLDiagnoseExtensionPoint.java b/server/plugins/connect-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/connect/mysql/MySQLDiagnoseExtensionPoint.java index 8d11debb5f..2df515953f 100644 --- a/server/plugins/connect-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/connect/mysql/MySQLDiagnoseExtensionPoint.java +++ b/server/plugins/connect-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/connect/mysql/MySQLDiagnoseExtensionPoint.java @@ -20,6 +20,7 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; +import java.util.List; import org.pf4j.Extension; @@ -94,7 +95,8 @@ public SqlExecDetail getExecutionDetailBySql(Connection connection, @NonNull Str } @Override - public SqlExplain getQueryProfileByTraceId(Connection connection, @NonNull String traceId) throws SQLException { + public SqlExplain getQueryProfileByTraceIdAndSessIds(Connection connection, @NonNull String traceId, + @NonNull List sessionIds) throws SQLException { throw new UnsupportedOperationException("Not supported for mysql mode"); } diff --git a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLDiagnoseExtension.java b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLDiagnoseExtension.java index be188bec58..bee42de80b 100644 --- a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLDiagnoseExtension.java +++ b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLDiagnoseExtension.java @@ -44,6 +44,7 @@ import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.exception.OBException; import com.oceanbase.odc.core.shared.exception.UnexpectedException; +import com.oceanbase.odc.core.shared.model.OBExecutionServerInfo; import com.oceanbase.odc.core.shared.model.OBSqlPlan; import com.oceanbase.odc.core.shared.model.PlanNode; import com.oceanbase.odc.core.shared.model.SqlExecDetail; @@ -67,6 +68,7 @@ @Slf4j @Extension public class OBMySQLDiagnoseExtension implements SqlDiagnoseExtensionPoint { + private static final String OB_EXPLAIN_COMPATIBLE_VERSION = "4.0"; private static final Pattern OPERATOR_PATTERN = Pattern.compile(".+\\|(OPERATOR +)\\|.+"); @Override @@ -88,24 +90,12 @@ public SqlExplain getExplain(Statement statement, @NonNull String sql) throws SQ } SqlExplain explain = new SqlExplain(); try { - String[] segs = text.split("Outputs & filters"); - // 层次计划信息 - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("|0 "); - String temp = segs[0].split("--\n\\|0")[1].split("\\|\n==")[0]; - stringBuilder.append(temp).append("|"); - String formatText = stringBuilder.toString(); - // output & filter - String[] outputSegs = segs[1].split("Used Hint")[0].split("[0-9]+ - output"); - Map outputFilters = new HashMap<>(); - for (int i = 1; i < outputSegs.length; i++) { - // 去除内存地址 - String seg = outputSegs[i].replaceAll("\\(0x[A-Za-z0-9]+\\)", ""); - String tmp = "output" + seg; - outputFilters.put(i - 1, tmp); + String version = OBUtils.getObVersion(statement.getConnection()); + if (VersionUtils.isGreaterThanOrEqualsTo(version, OB_EXPLAIN_COMPATIBLE_VERSION)) { + explain.setExpTree(DiagnoseUtil.getOB4xExplainTree(text)); + } else { + explain.setExpTree(DiagnoseUtil.getExplainTree(text)); } - String jsonTree = getJsonTreeText(formatText, outputFilters); - explain.setExpTree(jsonTree); // outline String outline = text.split("BEGIN_OUTLINE_DATA")[1].split("END_OUTLINE_DATA")[0]; explain.setOutline(outline); @@ -121,43 +111,6 @@ public SqlExplain getExplain(Statement statement, @NonNull String sql) throws SQ return explain; } - /** - * - * |0 |HASH JOIN | |98011 |258722| |1 | SUBPLAN SCAN |SUBQUERY_TABLE |101 |99182 | |2 | HASH - * DISTINCT| |101 |99169 | |3 | TABLE SCAN |t_test_get_explain2|100000 |66272 | |4 | TABLE SCAN |t1 - * |100000 |68478 | - * - * @param jsonText - * @return - */ - private String getJsonTreeText(String jsonText, Map outputFilters) { - String[] segs = jsonText.split("\\|"); - PlanNode planTree = null; - for (int i = 0; i < segs.length; i++) { - if ((i - 1) % 6 == 0) { - String id = segs[i].trim(); - String operator = segs[i + 1]; - String name = segs[i + 2].trim(); - String rowCount = segs[i + 3].trim(); - String cost = segs[i + 4].trim(); - // 构建计划树 - int depth = DiagnoseUtil.recognizeNodeDepth(operator); - PlanNode node = new PlanNode(); - node.setId(Integer.parseInt(id.trim())); - node.setName(name); - node.setCost(cost); - node.setDepth(depth); - node.setOperator(operator); - node.setRowCount(rowCount); - node.setOutputFilter(outputFilters.get(Integer.parseInt(id))); - PlanNode tmpPlanNode = DiagnoseUtil.buildPlanTree(planTree, node); - if (planTree == null) { - planTree = tmpPlanNode; - } - } - } - return JSON.toJSONString(planTree); - } @Override public SqlExplain getPhysicalPlanBySqlId(Connection connection, @NonNull String sqlId) throws SQLException { @@ -270,11 +223,12 @@ public SqlExecDetail getExecutionDetailBySql(Connection connection, @NonNull Str } @Override - public SqlExplain getQueryProfileByTraceId(Connection connection, @NonNull String traceId) throws SQLException { + public SqlExplain getQueryProfileByTraceIdAndSessIds(Connection connection, @NonNull String traceId, + @NonNull List sessionIds) throws SQLException { try (Statement stmt = connection.createStatement()) { - String planId = getPlanIdByTraceId(stmt, traceId); - Verify.notEmpty(planId, "plan id"); - PlanGraph graph = getPlanGraph(stmt, planId); + OBExecutionServerInfo executorInfo = getPlanIdByTraceIdAndSessIds(stmt, traceId, sessionIds); + Verify.notEmpty(executorInfo.getPlanId(), "plan id"); + PlanGraph graph = getPlanGraph(stmt, executorInfo); graph.setTraceId(traceId); QueryProfileHelper.refreshGraph(graph, getSqlPlanMonitorRecords(stmt, traceId)); @@ -293,22 +247,34 @@ public SqlExplain getQueryProfileByTraceId(Connection connection, @NonNull Strin } catch (Exception e) { log.warn("Failed to query sql audit with OB trace_id={}.", traceId, e); } - SqlExplain explain = innerGetSqlExplainByDbmsXplan(stmt, planId); + SqlExplain explain; + try { + explain = innerGetSqlExplainByDbmsXplan(stmt, executorInfo); + } catch (Exception e) { + log.warn("Failed to query plan by dbms_xplan.display_cursor.", e); + explain = new SqlExplain(); + explain.setShowFormatInfo(false); + } explain.setGraph(graph); return explain; } } - protected String getPlanIdByTraceId(Statement stmt, String traceId) throws SQLException { + protected OBExecutionServerInfo getPlanIdByTraceIdAndSessIds(Statement stmt, String traceId, + List sessionIds) + throws SQLException { try { - return OBUtils.queryPlanIdByTraceIdFromASH(stmt, traceId, ConnectType.OB_MYSQL); + return OBUtils.queryPlanIdByTraceIdFromASH(stmt, traceId, sessionIds, ConnectType.OB_MYSQL); } catch (SQLException e) { return OBUtils.queryPlanIdByTraceIdFromAudit(stmt, traceId, ConnectType.OB_MYSQL); } } - protected String getPhysicalPlanByDbmsXplan(Statement stmt, String planId) throws SQLException { - try (ResultSet rs = stmt.executeQuery("select dbms_xplan.display_cursor(" + planId + ")")) { + protected String getPhysicalPlanByDbmsXplan(Statement stmt, OBExecutionServerInfo executorInfo) + throws SQLException { + String sql = String.format("select dbms_xplan.display_cursor(%s,'ALL','%s',%s,%s)", + executorInfo.getPlanId(), executorInfo.getIp(), executorInfo.getPort(), executorInfo.getTenantId()); + try (ResultSet rs = stmt.executeQuery(sql)) { if (!rs.next()) { throw new SQLException("Failed to query plan by dbms_xplan.display_cursor"); } @@ -316,8 +282,8 @@ protected String getPhysicalPlanByDbmsXplan(Statement stmt, String planId) throw } } - protected PlanGraph getPlanGraph(Statement stmt, String planId) throws SQLException { - List planRecords = OBUtils.queryOBSqlPlanByPlanId(stmt, planId, ConnectType.OB_MYSQL); + protected PlanGraph getPlanGraph(Statement stmt, OBExecutionServerInfo executorInfo) throws SQLException { + List planRecords = OBUtils.queryOBSqlPlanByPlanId(stmt, executorInfo, ConnectType.OB_MYSQL); return PlanGraphBuilder.buildPlanGraph(planRecords); } @@ -342,11 +308,12 @@ protected List getSqlPlanMonitorRecords(Statement stmt, String t * 0 - output... * */ - private SqlExplain innerGetSqlExplainByDbmsXplan(Statement stmt, String planId) throws SQLException { + private SqlExplain innerGetSqlExplainByDbmsXplan(Statement stmt, OBExecutionServerInfo executorInfo) + throws SQLException { SqlExplain sqlExplain = new SqlExplain(); sqlExplain.setShowFormatInfo(true); - String planText = getPhysicalPlanByDbmsXplan(stmt, planId); + String planText = getPhysicalPlanByDbmsXplan(stmt, executorInfo); sqlExplain.setOriginalText(planText); String[] segs = planText.split("Outputs & filters")[0].split("\n"); String headerLine = segs[1]; diff --git a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/DiagnoseUtil.java b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/DiagnoseUtil.java index 312f1423b4..f0a65285f6 100644 --- a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/DiagnoseUtil.java +++ b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/DiagnoseUtil.java @@ -17,9 +17,14 @@ import java.sql.ResultSet; import java.sql.SQLException; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.core.shared.exception.UnexpectedException; import com.oceanbase.odc.core.shared.model.PlanNode; import com.oceanbase.odc.core.shared.model.SqlExecDetail; @@ -29,6 +34,7 @@ * @since ODC_release_4.2.0 */ public class DiagnoseUtil { + private static final Pattern OPERATOR_PATTERN = Pattern.compile(".+\\|(OPERATOR +)\\|.+"); public static PlanNode buildPlanTree(PlanNode planTree, PlanNode node) { if (planTree == null) { @@ -139,4 +145,92 @@ public static String getPlanTypeName(int code) { return "UNCERTAIN"; } } + + /** + * + * |0 |HASH JOIN | |98011 |258722| |1 | SUBPLAN SCAN |SUBQUERY_TABLE |101 |99182 | |2 | HASH + * DISTINCT| |101 |99169 | |3 | TABLE SCAN |t_test_get_explain2|100000 |66272 | |4 | TABLE SCAN |t1 + * |100000 |68478 | + */ + public static String getExplainTree(String text) { + String[] splits = text.split("Outputs & filters"); + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("|0 "); + String temp = splits[0].split("--\n\\|0")[1].split("\\|\n==")[0]; + stringBuilder.append(temp).append("|"); + String formatText = stringBuilder.toString(); + // output & filter + String[] outputSegs = splits[1].split("Used Hint")[0].split("[0-9]+ - output"); + Map outputFilters = new HashMap<>(); + for (int i = 1; i < outputSegs.length; i++) { + String seg = outputSegs[i].replaceAll("\\(0x[A-Za-z0-9]+\\)", ""); + String tmp = "output" + seg; + outputFilters.put(i - 1, tmp); + } + + String[] segs = formatText.split("\\|"); + PlanNode planTree = null; + for (int i = 0; i < segs.length; i++) { + if ((i - 1) % 6 == 0) { + String id = segs[i].trim(); + String operator = segs[i + 1]; + String name = segs[i + 2].trim(); + String rowCount = segs[i + 3].trim(); + String cost = segs[i + 4].trim(); + + int depth = DiagnoseUtil.recognizeNodeDepth(operator); + PlanNode node = new PlanNode(); + node.setId(Integer.parseInt(id.trim())); + node.setName(name); + node.setCost(cost); + node.setDepth(depth); + node.setOperator(operator); + node.setRowCount(rowCount); + node.setOutputFilter(outputFilters.get(Integer.parseInt(id))); + PlanNode tmpPlanNode = DiagnoseUtil.buildPlanTree(planTree, node); + if (planTree == null) { + planTree = tmpPlanNode; + } + } + } + return JsonUtils.toJson(planTree); + } + + public static String getOB4xExplainTree(String text) { + String[] segs = text.split("Outputs & filters"); + + String[] lines = segs[0].split("\n"); + String headerLine = lines[1]; + Matcher matcher = OPERATOR_PATTERN.matcher(headerLine); + if (!matcher.matches()) { + throw new UnexpectedException("Invalid explain:" + text); + } + int operatorStartIndex = matcher.start(1); + int operatorStrLen = matcher.end(1) - operatorStartIndex; + + PlanNode tree = null; + for (int i = 0; i < segs.length - 4; i++) { + PlanNode node = new PlanNode(); + node.setId(i); + + String line = segs[i + 3]; + String temp = line.substring(operatorStartIndex); + String operatorStr = temp.substring(0, operatorStrLen); + DiagnoseUtil.recognizeNodeDepthAndOperator(node, operatorStr); + + String[] others = temp.substring(operatorStrLen).split("\\|"); + node.setName(others[1]); + node.setRowCount(others[2]); + node.setCost(others[3]); + node.setRealRowCount(others[4]); + node.setRealCost(others[5]); + + PlanNode tmpPlanNode = DiagnoseUtil.buildPlanTree(tree, node); + if (tree == null) { + tree = tmpPlanNode; + } + } + return JsonUtils.toJson(tree); + } + } diff --git a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/QueryProfileHelper.java b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/QueryProfileHelper.java index 6a22824472..1f484c84f7 100644 --- a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/QueryProfileHelper.java +++ b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/QueryProfileHelper.java @@ -69,6 +69,7 @@ public static void refreshGraph(PlanGraph graph, List spmStats) replaceSPMStatsIntoProfile(spmStats, graph); } catch (Exception e) { log.warn("Failed to get runtime statistics with OB trace_id={}.", traceId, e); + return; } Map> topNodes = new HashMap<>(); @@ -147,7 +148,7 @@ private static void mergeSPMRecords(List spms, PlanGraphOperator long totalCPUTs = 0; long totalIOWaitTs = 0; long starts = 0; - float outputs = 0; + long outputs = 0; long maxMem = 0; long maxDisk = 0; long ioBytes = 0; @@ -211,7 +212,7 @@ private static void mergeSPMRecords(List spms, PlanGraphOperator } outputs += spm.getOutputRows(); - child.putStatistics(OUTPUT_ROWS, (float) spm.getOutputRows() + ""); + child.putStatistics(OUTPUT_ROWS, spm.getOutputRows() + ""); if (spm.getWorkareaMaxMem() != 0) { child.putStatistics(WORKAREA_MAX_MEN, spm.getWorkareaMaxMem() + ""); maxMem = Math.max(spm.getWorkareaMaxMem(), maxMem); @@ -285,7 +286,7 @@ private static void mergeSPMRecords(List spms, PlanGraphOperator // output rows List inEdges = operator.getInEdges(); if (inEdges != null && inEdges.size() == 1) { - inEdges.get(0).setWeight(outputs); + inEdges.get(0).setWeight((float) outputs); } subOperators.sort(Comparator.comparingLong(PlanGraphOperator::getDuration).reversed()); diff --git a/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleDiagnoseExtension.java b/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleDiagnoseExtension.java index cbd43392f6..6cf2624d3f 100644 --- a/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleDiagnoseExtension.java +++ b/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleDiagnoseExtension.java @@ -30,6 +30,7 @@ import com.oceanbase.odc.core.shared.constant.ConnectType; import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.exception.OBException; +import com.oceanbase.odc.core.shared.model.OBExecutionServerInfo; import com.oceanbase.odc.core.shared.model.OBSqlPlan; import com.oceanbase.odc.core.shared.model.PlanNode; import com.oceanbase.odc.core.shared.model.SqlExecDetail; @@ -182,24 +183,29 @@ protected SqlExecDetail innerGetExecutionDetail(Connection connection, String ap } @Override - protected String getPlanIdByTraceId(Statement stmt, String traceId) throws SQLException { + protected OBExecutionServerInfo getPlanIdByTraceIdAndSessIds(Statement stmt, String traceId, + List sessionIds) + throws SQLException { try { - return OBUtils.queryPlanIdByTraceIdFromASH(stmt, traceId, ConnectType.OB_ORACLE); + return OBUtils.queryPlanIdByTraceIdFromASH(stmt, traceId, sessionIds, ConnectType.OB_ORACLE); } catch (SQLException e) { return OBUtils.queryPlanIdByTraceIdFromAudit(stmt, traceId, ConnectType.OB_ORACLE); } } @Override - protected String getPhysicalPlanByDbmsXplan(Statement stmt, String planId) throws SQLException { + protected String getPhysicalPlanByDbmsXplan(Statement stmt, OBExecutionServerInfo executorInfo) + throws SQLException { try (ResultSet rs = stmt.executeQuery("select VALUE from V$NLS_PARAMETERS where PARAMETER='NLS_CHARACTERSET'")) { if (rs.next() && StringUtils.containsIgnoreCase(rs.getString(1), "GBK")) { stmt.execute("set names gbk"); } } + String sql = String.format("select * from TABLE(DBMS_XPLAN.DISPLAY_CURSOR(%s,'ALL','%s',%s,%s))", + executorInfo.getPlanId(), executorInfo.getIp(), executorInfo.getPort(), executorInfo.getTenantId()); StringBuilder builder = new StringBuilder(); - try (ResultSet rs = stmt.executeQuery("select * from TABLE(DBMS_XPLAN.DISPLAY_CURSOR(" + planId + "))")) { + try (ResultSet rs = stmt.executeQuery(sql)) { while (rs.next()) { builder.append(rs.getString(1)).append("\n"); } @@ -210,8 +216,8 @@ protected String getPhysicalPlanByDbmsXplan(Statement stmt, String planId) throw return builder.toString(); } - protected PlanGraph getPlanGraph(Statement stmt, String planId) throws SQLException { - List planRecords = OBUtils.queryOBSqlPlanByPlanId(stmt, planId, ConnectType.OB_ORACLE); + protected PlanGraph getPlanGraph(Statement stmt, OBExecutionServerInfo executorInfo) throws SQLException { + List planRecords = OBUtils.queryOBSqlPlanByPlanId(stmt, executorInfo, ConnectType.OB_ORACLE); return PlanGraphBuilder.buildPlanGraph(planRecords); } From 660047a59cc1c20d0fcaa77902c8fb9ab7cd7e74 Mon Sep 17 00:00:00 2001 From: LuckyLeo Date: Wed, 24 Jul 2024 10:36:27 +0800 Subject: [PATCH 43/64] fix(query-profile): modified the version supporting query profile (#3002) * set supported version to [4.2.4, 4.3.0) || [4.3.3, ) * fix explain parse failed --- .../service/session/ConnectConsoleService.java | 2 +- .../session/OBQueryProfileExecutionListener.java | 13 +++++++++++-- .../connect/obmysql/diagnose/DiagnoseUtil.java | 16 ++++++++++++---- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java index 4790a8580b..e3b5cccd89 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java @@ -601,7 +601,7 @@ private SqlExecuteResult generateResult(@NonNull ConnectionSession connectionSes String version = ConnectionSessionUtil.getVersion(connectionSession); result.setWithQueryProfile(OBQueryProfileExecutionListener .isSqlTypeSupportProfile(generalResult.getSqlTuple()) && - VersionUtils.isGreaterThanOrEqualsTo(version, OBQueryProfileManager.ENABLE_QUERY_PROFILE_VERSION)); + OBQueryProfileExecutionListener.isObVersionSupportQueryProfile(version)); } catch (Exception e) { result.setWithQueryProfile(false); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/OBQueryProfileExecutionListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/OBQueryProfileExecutionListener.java index a68f93b744..a6ed74c7ca 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/OBQueryProfileExecutionListener.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/OBQueryProfileExecutionListener.java @@ -17,7 +17,6 @@ import static com.oceanbase.odc.core.session.ConnectionSessionConstants.BACKEND_DS_KEY; import static com.oceanbase.odc.core.session.ConnectionSessionConstants.CONSOLE_DS_KEY; -import static com.oceanbase.odc.service.queryprofile.OBQueryProfileManager.ENABLE_QUERY_PROFILE_VERSION; import java.util.Collections; import java.util.List; @@ -91,7 +90,7 @@ public Long getOnExecutionStartAfterMillis() { } private List getSessionIds() { - if (VersionUtils.isLessThan(ConnectionSessionUtil.getVersion(session), ENABLE_QUERY_PROFILE_VERSION)) { + if (!isObVersionSupportQueryProfile(ConnectionSessionUtil.getVersion(session))) { return Collections.emptyList(); } String proxySessId = ConnectionSessionUtil.getConsoleConnectionProxySessId(session); @@ -117,4 +116,14 @@ public static boolean isSqlTypeSupportProfile(SqlTuple sqlTuple) { return false; } } + + /** + * [4.2.4, 4.3.0) || [4.3.3, ) + */ + public static boolean isObVersionSupportQueryProfile(String version) { + return VersionUtils.isGreaterThanOrEqualsTo(version, "4.2.4") + && VersionUtils.isLessThan(version, "4.3.0") + || VersionUtils.isGreaterThanOrEqualsTo(version, "4.3.3"); + + } } diff --git a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/DiagnoseUtil.java b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/DiagnoseUtil.java index f0a65285f6..f2e9cfd6c3 100644 --- a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/DiagnoseUtil.java +++ b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/DiagnoseUtil.java @@ -199,6 +199,15 @@ public static String getExplainTree(String text) { public static String getOB4xExplainTree(String text) { String[] segs = text.split("Outputs & filters"); + // output & filter + String[] outputSegs = segs[1].split("Used Hint")[0].split("[0-9]+ - output"); + Map outputFilters = new HashMap<>(); + for (int i = 1; i < outputSegs.length; i++) { + String seg = outputSegs[i].replaceAll("\\(0x[A-Za-z0-9]+\\)", ""); + String tmp = "output" + seg; + outputFilters.put(i - 1, tmp); + } + String[] lines = segs[0].split("\n"); String headerLine = lines[1]; Matcher matcher = OPERATOR_PATTERN.matcher(headerLine); @@ -209,11 +218,11 @@ public static String getOB4xExplainTree(String text) { int operatorStrLen = matcher.end(1) - operatorStartIndex; PlanNode tree = null; - for (int i = 0; i < segs.length - 4; i++) { + for (int i = 0; i < lines.length - 4; i++) { PlanNode node = new PlanNode(); node.setId(i); - String line = segs[i + 3]; + String line = lines[i + 3]; String temp = line.substring(operatorStartIndex); String operatorStr = temp.substring(0, operatorStrLen); DiagnoseUtil.recognizeNodeDepthAndOperator(node, operatorStr); @@ -222,8 +231,7 @@ public static String getOB4xExplainTree(String text) { node.setName(others[1]); node.setRowCount(others[2]); node.setCost(others[3]); - node.setRealRowCount(others[4]); - node.setRealCost(others[5]); + node.setOutputFilter(outputFilters.get(i)); PlanNode tmpPlanNode = DiagnoseUtil.buildPlanTree(tree, node); if (tree == null) { From 3d51ba419ff915d23b033885eb274514bdfa0487 Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Wed, 24 Jul 2024 20:57:02 +0800 Subject: [PATCH 44/64] fix(alarm): task alarm add exception message (#3004) * fix * fix --- .../service/task/schedule/daemon/DestroyExecutorJob.java | 4 ++-- .../odc/service/task/schedule/daemon/DoCancelingJob.java | 3 ++- .../odc/service/task/schedule/daemon/StartPreparingJob.java | 3 ++- .../odc/service/task/service/StdTaskFrameworkService.java | 6 ++++-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyExecutorJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyExecutorJob.java index d0d5ba91c7..72c3bebc42 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyExecutorJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyExecutorJob.java @@ -82,8 +82,8 @@ private void destroyExecutor(TaskFrameworkService taskFrameworkService, JobEntit if (e.getMessage() != null && !e.getMessage().startsWith(JobConstants.ODC_EXECUTOR_CANNOT_BE_DESTROYED)) { AlarmUtils.alarm(AlarmEventNames.TASK_EXECUTOR_DESTROY_FAILED, - MessageFormat.format("Job executor destroy failed, jobId={0}", - lockedEntity.getId())); + MessageFormat.format("Job executor destroy failed, jobId={0}, message={1}", + lockedEntity.getId(), e.getMessage())); } throw new TaskRuntimeException(e); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DoCancelingJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DoCancelingJob.java index 50d9c30fe9..d509789ff0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DoCancelingJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DoCancelingJob.java @@ -87,7 +87,8 @@ private void cancelJob(TaskFrameworkService taskFrameworkService, JobEntity jobE } catch (JobException e) { log.warn("Stop job occur error: ", e); AlarmUtils.alarm(AlarmEventNames.TASK_CANCELED_FAILED, - MessageFormat.format("Cancel job failed, jobId={0}", lockedEntity.getId())); + MessageFormat.format("Cancel job failed, jobId={0}, message={1}", lockedEntity.getId(), + e.getMessage())); throw new TaskRuntimeException(e); } log.info("Job be cancelled successfully, jobId={}, oldStatus={}.", lockedEntity.getId(), diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/StartPreparingJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/StartPreparingJob.java index cf0bed56ef..be5fe316be 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/StartPreparingJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/StartPreparingJob.java @@ -113,7 +113,8 @@ private void startJob(TaskFrameworkService taskFrameworkService, JobEntity jobEn getConfiguration().getJobDispatcher().start(jc); } catch (JobException e) { AlarmUtils.alarm(AlarmEventNames.TASK_START_FAILED, - MessageFormat.format("Start job failed, jobId={0}", lockedEntity.getId())); + MessageFormat.format("Start job failed, jobId={0}, message={1}", lockedEntity.getId(), + e.getMessage())); throw new TaskRuntimeException(e); } } else { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java index ba551ec73a..f3e2f4728f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java @@ -47,6 +47,7 @@ import com.oceanbase.odc.common.event.EventPublisher; import com.oceanbase.odc.common.jpa.SpecificationUtil; import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.common.security.SensitiveDataUtils; import com.oceanbase.odc.common.trace.TraceContextHolder; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.common.util.SystemUtils; @@ -329,8 +330,9 @@ public void handleResult(TaskResult taskResult) { .publishEvent(new JobTerminateEvent(taskResult.getJobIdentity(), taskResult.getStatus()))); if (taskResult.getStatus() == JobStatus.FAILED) { AlarmUtils.alarm(AlarmEventNames.TASK_EXECUTION_FAILED, - MessageFormat.format("Job execution failed, jobId={0}", - taskResult.getJobIdentity().getId())); + MessageFormat.format("Job execution failed, jobId={0}, resultJson={1}", + taskResult.getJobIdentity().getId(), + SensitiveDataUtils.mask(taskResult.getResultJson()))); } } } From ad64de5dc9908ccb0374e2039a006a72867266fa Mon Sep 17 00:00:00 2001 From: LuckyLeo Date: Thu, 25 Jul 2024 11:08:07 +0800 Subject: [PATCH 45/64] fix(data-transfer): clean work directory before import (#3006) --- .../odc/service/datatransfer/task/DataTransferTask.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/datatransfer/task/DataTransferTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/datatransfer/task/DataTransferTask.java index 5cbdafd026..c42895c633 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/datatransfer/task/DataTransferTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/datatransfer/task/DataTransferTask.java @@ -332,6 +332,8 @@ private List copyImportScripts(List fileNames, DataTransferFormat f LOGGER.warn("Multiple files for CSV format is invalid, importFileNames={}", fileNames); throw new IllegalArgumentException("Multiple files isn't accepted for CSV format"); } + // There maybe some dirty files generated before 4.2.3. We should clean them first. + FileUtils.cleanDirectory(destDir); LocalFileManager fileManager = SpringContextUtil.getBean(LocalFileManager.class); List inputs = new ArrayList<>(); for (String fileName : fileNames) { From 5f0902e29d203233c89faf8c81cdafe4d4bcf953 Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Thu, 25 Jul 2024 13:59:15 +0800 Subject: [PATCH 46/64] fix(login): set max_login_record_time_minutes default value to 0 in web mode (#3003) * fix * fix * fix --- .../V_4_3_1_6__init_max_login_record_time.sql | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 server/odc-migrate/src/main/resources/migrate/web/V_4_3_1_6__init_max_login_record_time.sql diff --git a/server/odc-migrate/src/main/resources/migrate/web/V_4_3_1_6__init_max_login_record_time.sql b/server/odc-migrate/src/main/resources/migrate/web/V_4_3_1_6__init_max_login_record_time.sql new file mode 100644 index 0000000000..3d060495cc --- /dev/null +++ b/server/odc-migrate/src/main/resources/migrate/web/V_4_3_1_6__init_max_login_record_time.sql @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2024 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('odc.security.max-login-record-time-minutes', '0', 'Interval of login records.') ON DUPLICATE KEY UPDATE `id`=`id`; From c586b8974956782f076b27133d3a0719af0276ff Mon Sep 17 00:00:00 2001 From: guowl3 <70270140+guowl3@users.noreply.github.com> Date: Thu, 25 Jul 2024 15:37:17 +0800 Subject: [PATCH 47/64] validate trigger interval when create scheduler. (#3012) --- server/odc-server/src/main/resources/data.sql | 1 + .../CreateFlowInstanceProcessAspect.java | 41 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/server/odc-server/src/main/resources/data.sql b/server/odc-server/src/main/resources/data.sql index 8c76775ba1..39fc925b0c 100644 --- a/server/odc-server/src/main/resources/data.sql +++ b/server/odc-server/src/main/resources/data.sql @@ -716,6 +716,7 @@ INSERT INTO config_system_configuration ( `key`, `value`, `description` ) VALUES INSERT INTO config_system_configuration ( `key`, `value`, `description` ) VALUES( 'odc.data-security.masking.enabled', 'true', '是否开启数据脱敏，默认为开启' ) ON DUPLICATE KEY UPDATE `id` = `id`; INSERT INTO config_system_configuration ( `key`, `value`, `description` ) VALUES( 'odc.task.partition-plan.schedule-cron', '0 0 * * * ?', '默认调度周期：每天 0 点' ) ON DUPLICATE KEY UPDATE `id` = `id`; +INSERT INTO config_system_configuration ( `key`, `value`, `description` ) VALUES( 'odc.task.trigger.minimum-interval', '600', '计划任务最小触发间隔，默认值：600 ，单位：秒' ) ON DUPLICATE KEY UPDATE `id` = `id`; -- -- v4.2.1 diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/processor/CreateFlowInstanceProcessAspect.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/processor/CreateFlowInstanceProcessAspect.java index 3c5ee9e0b2..ec9abbe225 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/processor/CreateFlowInstanceProcessAspect.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/processor/CreateFlowInstanceProcessAspect.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.service.flow.processor; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -26,6 +27,7 @@ import org.aspectj.lang.annotation.Pointcut; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import com.oceanbase.odc.core.shared.PreConditions; @@ -41,10 +43,14 @@ import com.oceanbase.odc.service.flow.task.model.DBStructureComparisonParameter; import com.oceanbase.odc.service.flow.util.DescriptionGenerator; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; +import com.oceanbase.odc.service.partitionplan.model.PartitionPlanConfig; +import com.oceanbase.odc.service.quartz.util.QuartzCronExpressionUtils; import com.oceanbase.odc.service.schedule.ScheduleService; import com.oceanbase.odc.service.schedule.flowtask.AlterScheduleParameters; import com.oceanbase.odc.service.schedule.flowtask.OperationType; import com.oceanbase.odc.service.schedule.model.JobType; +import com.oceanbase.odc.service.schedule.model.TriggerConfig; +import com.oceanbase.odc.service.schedule.model.TriggerStrategy; import lombok.extern.slf4j.Slf4j; @@ -69,6 +75,8 @@ public class CreateFlowInstanceProcessAspect implements InitializingBean { private List preprocessors; @Autowired private ProjectService projectService; + @Value("${odc.task.trigger.minimum-interval:600}") + private Long triggerMinimumIntervalSeconds; private final Map scheduleTaskPreprocessors = new HashMap<>(); @@ -81,6 +89,7 @@ public void processBeforeCreateFlowInstance() {} @Before("processBeforeCreateFlowInstance()") public void preprocess(JoinPoint point) throws Throwable { CreateFlowInstanceReq req = (CreateFlowInstanceReq) point.getArgs()[0]; + validateTriggerConfig(req); if (req.getTaskType() == TaskType.STRUCTURE_COMPARISON) { DBStructureComparisonParameter parameters = (DBStructureComparisonParameter) req.getParameters(); req.setDatabaseId(parameters.getSourceDatabaseId()); @@ -163,4 +172,36 @@ private void adaptCreateFlowInstanceReq(CreateFlowInstanceReq req) { config.setDatabaseId(req.getDatabaseId()); } } + + private void validateTriggerConfig(CreateFlowInstanceReq req) { + if (req.getParameters() instanceof PartitionPlanConfig) { + PartitionPlanConfig parameters = (PartitionPlanConfig) req.getParameters(); + validateTriggerConfig(parameters.getCreationTrigger()); + if (parameters.getDroppingTrigger() != null) { + validateTriggerConfig(parameters.getDroppingTrigger()); + } + return; + } + if (req.getParameters() instanceof AlterScheduleParameters) { + AlterScheduleParameters parameters = (AlterScheduleParameters) req.getParameters(); + validateTriggerConfig(parameters.getTriggerConfig()); + } + } + + private void validateTriggerConfig(TriggerConfig triggerConfig) { + PreConditions.notNull(triggerConfig, "triggerConfig"); + if (triggerConfig.getTriggerStrategy() == TriggerStrategy.CRON) { + List nextFiveFireTimes = + QuartzCronExpressionUtils.getNextFiveFireTimes(triggerConfig.getCronExpression(), 2); + if (nextFiveFireTimes.size() != 2) { + throw new IllegalArgumentException("Invalid cron expression"); + } + long intervalMills = nextFiveFireTimes.get(1).getTime() - nextFiveFireTimes.get(0).getTime(); + if (intervalMills / 1000 < triggerMinimumIntervalSeconds) { + throw new IllegalArgumentException( + String.format("The minimum interval is 10 minutes,cron=%s", triggerConfig.getCronExpression())); + } + } + } + } From 743c034e8331032598841065563f6c24e4f51382 Mon Sep 17 00:00:00 2001 From: guowl3 <70270140+guowl3@users.noreply.github.com> Date: Thu, 25 Jul 2024 15:42:05 +0800 Subject: [PATCH 48/64] fix(dlm): don't compare the table structure if syncTableStructure is off (#3014) --- .../flowtask/DataArchivePreprocessor.java | 18 ++++++++++++++---- .../service/schedule/job/AbstractDlmJob.java | 3 ++- .../task/executor/task/DataArchiveTask.java | 2 +- .../listener/DefaultJobTerminateListener.java | 2 +- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DataArchivePreprocessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DataArchivePreprocessor.java index 81194f622b..87f57e942c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DataArchivePreprocessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DataArchivePreprocessor.java @@ -86,10 +86,9 @@ public void process(CreateFlowInstanceReq req) { // permission to access it. Database sourceDb = databaseService.detail(dataArchiveParameters.getSourceDatabaseId()); Database targetDb = databaseService.detail(dataArchiveParameters.getTargetDataBaseId()); - if (!dataArchiveParameters.getSyncTableStructure().isEmpty() - && sourceDb.getDataSource().getDialectType() != targetDb.getDataSource().getDialectType()) { - throw new UnsupportedException( - "Different types of databases do not support structural synchronization."); + if (!dataArchiveParameters.getSyncTableStructure().isEmpty()) { + supportSyncTableStructure(sourceDb.getDataSource().getDialectType(), targetDb.getDataSource() + .getDialectType()); } dataArchiveParameters.setSourceDatabaseName(sourceDb.getName()); dataArchiveParameters.setTargetDatabaseName(targetDb.getName()); @@ -188,4 +187,15 @@ private void initDefaultConfig(DataArchiveParameters parameters) { parameters.setQueryTimeout(dlmConfiguration.getTaskConnectionQueryTimeout()); parameters.setShardingStrategy(dlmConfiguration.getShardingStrategy()); } + + private void supportSyncTableStructure(DialectType srcDbType, DialectType tgtDbType) { + if (srcDbType != tgtDbType) { + throw new UnsupportedException( + "Different types of databases do not support table structure synchronization."); + } + if (!srcDbType.isMysql()) { + throw new UnsupportedException( + String.format("The database does not support table structure synchronization,type=%s", srcDbType)); + } + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java index f42012bf88..a279b5521c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java @@ -114,7 +114,8 @@ public void executeTask(Long taskId, List dlmTableUnits, Long time dlmTableUnit.getTableName()); continue; } - if (dlmTableUnit.getType() == JobType.MIGRATE) { + if (dlmTableUnit.getType() == JobType.MIGRATE + && !dlmTableUnit.getParameters().getSyncDBObjectType().isEmpty()) { try { DLMTableStructureSynchronizer.sync(dlmTableUnit.getSourceDatasourceInfo(), dlmTableUnit.getTargetDatasourceInfo(), dlmTableUnit.getTableName(), diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DataArchiveTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DataArchiveTask.java index b342ba195c..9f01c00d27 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DataArchiveTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DataArchiveTask.java @@ -88,7 +88,7 @@ protected boolean doStart(JobContext context) throws Exception { log.info("The table had been completed,tableName={}", dlmTableUnit.getTableName()); continue; } - if (parameters.getJobType() == JobType.MIGRATE) { + if (parameters.getJobType() == JobType.MIGRATE && !parameters.getSyncTableStructure().isEmpty()) { try { DLMTableStructureSynchronizer.sync( DataSourceInfoMapper.toConnectionConfig(parameters.getSourceDs()), diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobTerminateListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobTerminateListener.java index 135349e83f..a81c76f06f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobTerminateListener.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobTerminateListener.java @@ -77,7 +77,7 @@ public void onEvent(JobTerminateEvent event) { scheduleService.refreshScheduleStatus(Long.parseLong(o.getJobName())); // Trigger the data-delete job if necessary after the data-archive task is completed. if (parameters.getJobType() == com.oceanbase.tools.migrator.common.enums.JobType.MIGRATE - && parameters.isDeleteAfterMigration()) { + && parameters.isDeleteAfterMigration() && taskStatus == TaskStatus.DONE) { scheduleService.dataArchiveDelete(Long.parseLong(o.getJobName()), o.getId()); log.info("Trigger delete job succeed."); } From 29d49fa8f927118ce6cdd33af6abde81fec5ed2b Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Thu, 25 Jul 2024 16:00:35 +0800 Subject: [PATCH 49/64] feat(kill session): adapt global client session and using block in oracle model in kill session (#2978) * adapt global client session and oracle model pl * modify according to pr commit * modify code according to pr * modify code format * modify code format * modify code format * add judgment of directing observer * modify code format * Check whether the directly connected sql matches the oracle and mysql modes * Modify the code according to pr * modify code format * add comment for main method * modify according to pr * Optimize readability --- .../v2/ConnectSessionController.java | 2 +- .../db/session/DefaultDBSessionManage.java | 235 +++++++++++++++++- 2 files changed, 223 insertions(+), 14 deletions(-) diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ConnectSessionController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ConnectSessionController.java index b3407c7146..daac207cf9 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ConnectSessionController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ConnectSessionController.java @@ -209,7 +209,7 @@ public SuccessResponse killQuery(@PathVariable String sessionId) { * kill session * * @param req - * @return + * @return kill result */ @ApiOperation(value = "kill session", notes = "终止会话接口") @RequestMapping(value = "/sessions/killSession", method = RequestMethod.POST) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java index 9a06d5dacb..f10569cfee 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java @@ -15,8 +15,11 @@ */ package com.oceanbase.odc.service.db.session; +import static com.oceanbase.odc.core.shared.constant.DialectType.OB_MYSQL; + import java.sql.Connection; import java.sql.Statement; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; @@ -40,12 +43,14 @@ import com.google.common.collect.Lists; import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.common.util.VersionUtils; import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.datasource.SingleConnectionDataSource; import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.session.ConnectionSessionConstants; import com.oceanbase.odc.core.session.ConnectionSessionUtil; import com.oceanbase.odc.core.shared.Verify; +import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.exception.InternalServerError; import com.oceanbase.odc.core.shared.model.OdcDBSession; import com.oceanbase.odc.core.sql.execute.model.JdbcGeneralResult; @@ -78,6 +83,13 @@ public class DefaultDBSessionManage implements DBSessionManageFacade { + "(?[0-9]{1,5})\\*/.*"; private static final Pattern SERVER_PATTERN = Pattern.compile(SERVER_REGEX); private static final ConnectionMapper CONNECTION_MAPPER = ConnectionMapper.INSTANCE; + private static final String GLOBAL_CLIENT_SESSION_OB_PROXY_VERSION_NUMBER = "4.2.3"; + private static final String GLOBAL_CLIENT_SESSION_OB_VERSION_NUMBER = "4.2.5"; + private static final String ORACLE_MODEL_KILL_SESSION_WITH_BLOCK_OB_VERSION_NUMBER = "4.2.1.0"; + private static final byte GLOBAL_CLIENT_SESSION_PROXY_ID_MIN = 0; + private static final short GLOBAL_CLIENT_SESSION_PROXY_ID_MAX = 8191; + private static final byte GLOBAL_CLIENT_SESSION_ID_VERSION = 2; + @Autowired private ConnectSessionService sessionService; @@ -198,6 +210,16 @@ private List doKillSessionOrQuery( @SkipAuthorize("odc internal usage") public List executeKillSession(ConnectionSession connectionSession, List sqlTuples, String sqlScript) { + List results = executeKillCommands(connectionSession, sqlTuples, sqlScript); + if (connectionSession.getDialectType() == DialectType.OB_MYSQL + || connectionSession.getDialectType() == DialectType.OB_ORACLE) { + return processResults(connectionSession, results); + } + return results; + } + + private List executeKillCommands(ConnectionSession connectionSession, List sqlTuples, + String sqlScript) { List results = connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY) .execute(new OdcStatementCallBack(sqlTuples, connectionSession, true, null, false)); @@ -205,26 +227,213 @@ public List executeKillSession(ConnectionSession connectionSe log.warn("Execution of the kill session command failed with unknown error, sql={}", sqlScript); throw new InternalServerError("Unknown error"); } - return results.stream().map(jdbcGeneralResult -> { - SqlTuple sqlTuple = jdbcGeneralResult.getSqlTuple(); + return results; + } + + /** + * process the execution result after the first kill commands. if the result contains unknown thread + * id exception, try to use other solutions to execute the kill commands. + * + * @param connectionSession + * @param results + * @return + */ + private List processResults(ConnectionSession connectionSession, + List results) { + Boolean isDirectedOBServer = isObServerDirected(connectionSession); + String obProxyVersion = getObProxyVersion(connectionSession, isDirectedOBServer); + String obVersion = ConnectionSessionUtil.getVersion(connectionSession); + boolean isEnabledGlobalClientSession = + isGlobalClientSessionEnabled(connectionSession, obProxyVersion, obVersion); + boolean isSupportedOracleModeKillSession = isOracleModeKillSessionSupported(obVersion, connectionSession); + List finalResults = new ArrayList<>(); + for (JdbcGeneralResult jdbcGeneralResult : results) { try { jdbcGeneralResult.getQueryResult(); } catch (Exception e) { - if (StringUtils.contains(e.getMessage(), "Unknown thread id")) { - try { - log.info("Kill query/session Unknown thread id error, try direct connect observer"); - directLinkServerAndExecute(sqlTuple.getExecutedSql(), - connectionSession); - return JdbcGeneralResult.successResult(sqlTuple); - } catch (Exception e1) { - log.warn("Failed to direct connect observer {}", e1.getMessage()); - } + if (isUnknownThreadIdError(e)) { + jdbcGeneralResult = handleUnknownThreadIdError(connectionSession, + jdbcGeneralResult, isDirectedOBServer, + isEnabledGlobalClientSession, isSupportedOracleModeKillSession); } else { - log.warn("Failed to execute sql in kill session scenario, sqlTuple={}", sqlTuple, e); + log.warn("Failed to execute sql in kill session scenario, sqlTuple={}", + jdbcGeneralResult.getSqlTuple(), e); } } + finalResults.add(jdbcGeneralResult); + } + return finalResults; + } + + /** + * Check whether client directed oceanbase database server. If an exception occurs or the version + * does not support, return null. + * + * @param connectionSession + * @return + */ + private Boolean isObServerDirected(ConnectionSession connectionSession) { + String sql = (connectionSession.getDialectType() == OB_MYSQL) + ? "select PROXY_SESSID from oceanbase.gv$ob_processlist where ID =(select connection_id());" + : "select PROXY_SESSID from gv$ob_processlist where ID =(select sys_context('userenv','sid') from dual);"; + try { + List proxySessids = connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY) + .query(sql, (rs, rowNum) -> rs.getString("PROXY_SESSID")); + if (proxySessids != null && proxySessids.size() == 1) { + return proxySessids.get(0) == null; + } + } catch (Exception e) { + log.warn("Failed to obtain the PROXY_SESSID: {}", e.getMessage()); + } + // Return null if the version is not supported or an exception occurs + return null; + } + + /** + * Get the OBProxy version number. If an exception occurs or the version does not support, return + * null. + * + * @param connectionSession + * @param isDirectedOBServer + * @return + */ + private String getObProxyVersion(ConnectionSession connectionSession, Boolean isDirectedOBServer) { + if (Boolean.TRUE.equals(isDirectedOBServer)) { + return null; + } + try { + return connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY) + .queryForObject("select proxy_version()", String.class); + } catch (Exception e) { + log.warn("Failed to obtain the OBProxy version number: {}", e.getMessage()); + return null; + } + } + + private boolean isGlobalClientSessionEnabled(ConnectionSession connectionSession, String obProxyVersion, + String obVersion) { + // verification version requirement + if (StringUtils.isBlank(obProxyVersion) + || StringUtils.isBlank(obVersion) + || VersionUtils.isLessThan(obProxyVersion, GLOBAL_CLIENT_SESSION_OB_PROXY_VERSION_NUMBER) + || VersionUtils.isLessThan(obVersion, GLOBAL_CLIENT_SESSION_OB_VERSION_NUMBER)) { + return false; + } + try { + Integer proxyId = getOBProxyConfig(connectionSession, "proxy_id"); + Integer clientSessionIdVersion = getOBProxyConfig(connectionSession, "client_session_id_version"); + + return proxyId != null + && proxyId >= GLOBAL_CLIENT_SESSION_PROXY_ID_MIN + && proxyId <= GLOBAL_CLIENT_SESSION_PROXY_ID_MAX + && clientSessionIdVersion != null + && clientSessionIdVersion == GLOBAL_CLIENT_SESSION_ID_VERSION; + } catch (Exception e) { + log.warn("Failed to determine if global client session is enabled: {}", e.getMessage()); + return false; + } + } + + /** + * Gets the value of OBProxy's configuration variable If an exception occurs or the version does not + * support, return null. + * + * @param connectionSession + * @param configName + * @return + */ + private Integer getOBProxyConfig(ConnectionSession connectionSession, String configName) { + try { + return connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY) + .query("show proxyconfig like '" + configName + "';", + rs -> rs.next() ? rs.getInt("value") : null); + } catch (Exception e) { + log.warn("Failed to obtain the value of OBProxy's configuration variable: {}", e.getMessage()); + return null; + } + } + + private boolean isUnknownThreadIdError(Exception e) { + return StringUtils.containsIgnoreCase(e.getMessage(), "Unknown thread id"); + } + + private JdbcGeneralResult handleUnknownThreadIdError(ConnectionSession connectionSession, + JdbcGeneralResult jdbcGeneralResult, Boolean isDirectedOBServer, + boolean isEnabledGlobalClientSession, boolean isSupportedOracleModeKillSession) { + if (Boolean.TRUE.equals(isDirectedOBServer)) { + log.info("The current connection mode is directing observer, return error result directly"); + return jdbcGeneralResult; + } + if (isEnabledGlobalClientSession) { + log.info("The OBProxy has enabled the global client session, return error result directly"); + return jdbcGeneralResult; + } + if (isSupportedOracleModeKillSession) { + return tryKillSessionByAnonymousBlock(connectionSession, jdbcGeneralResult, + jdbcGeneralResult.getSqlTuple()); + } + return tryKillSessionViaDirectConnectObServer(connectionSession, jdbcGeneralResult, + jdbcGeneralResult.getSqlTuple()); + } + + private boolean isOracleModeKillSessionSupported(String obVersion, ConnectionSession connectionSession) { + return StringUtils.isNotBlank(obVersion) && + VersionUtils.isGreaterThanOrEqualsTo(obVersion, ORACLE_MODEL_KILL_SESSION_WITH_BLOCK_OB_VERSION_NUMBER) + && + connectionSession.getDialectType() == DialectType.OB_ORACLE; + } + + /** + * Try to kill session by using anonymous code blocks. If successful, return a successful + * jdbcGeneralResult, otherwise return the original jdbcGeneralResult. + * + * @param connectionSession + * @param jdbcGeneralResult + * @param sqlTuple + * @return + */ + private JdbcGeneralResult tryKillSessionByAnonymousBlock(ConnectionSession connectionSession, + JdbcGeneralResult jdbcGeneralResult, SqlTuple sqlTuple) { + log.info("Kill query/session Unknown thread id error, try anonymous code blocks"); + String executedSql = sqlTuple.getExecutedSql(); + if (executedSql != null && executedSql.endsWith(";")) { + executedSql = executedSql.substring(0, executedSql.length() - 1); + } + String anonymousCodeBlock = "BEGIN\n" + + "EXECUTE IMMEDIATE '" + + executedSql + + "';\n" + + "END;"; + try { + connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY) + .execute(anonymousCodeBlock); + return JdbcGeneralResult.successResult(sqlTuple); + + } catch (Exception e) { + log.warn("Failed to kill session by using anonymous code blocks: {}", e.getMessage()); return jdbcGeneralResult; - }).collect(Collectors.toList()); + } + } + + /** + * Try to kill session by direct connect observer. If successful, return a successful + * jdbcGeneralResult, otherwise return the original jdbcGeneralResult. + * + * @param connectionSession + * @param jdbcGeneralResult + * @param sqlTuple + * @return + */ + private JdbcGeneralResult tryKillSessionViaDirectConnectObServer(ConnectionSession connectionSession, + JdbcGeneralResult jdbcGeneralResult, SqlTuple sqlTuple) { + try { + log.info("Kill query/session Unknown thread id error, try direct connect observer"); + directLinkServerAndExecute(sqlTuple.getExecutedSql(), connectionSession); + return JdbcGeneralResult.successResult(sqlTuple); + } catch (Exception e) { + log.warn("Failed to direct connect observer {}", e.getMessage()); + return jdbcGeneralResult; + } } private void directLinkServerAndExecute(String sql, ConnectionSession session) From 16a2b84a738ddf36ddf052c2637ab5b9861d0fa9 Mon Sep 17 00:00:00 2001 From: LuckyLeo Date: Mon, 29 Jul 2024 12:12:45 +0800 Subject: [PATCH 50/64] builds: update 4.3.1 submodule (#3030) --- client | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client b/client index 1c34b729cf..0b81dd6e73 160000 --- a/client +++ b/client @@ -1 +1 @@ -Subproject commit 1c34b729cf6e84b2accf1ee6a4e0af9a8fabe426 +Subproject commit 0b81dd6e73f475787e2b33460e21435f91fd740e From cec69545416ef2495ceac3dea945f1843fb8a254 Mon Sep 17 00:00:00 2001 From: pynzzZ <39962741+MarkPotato777@users.noreply.github.com> Date: Mon, 29 Jul 2024 16:03:45 +0800 Subject: [PATCH 51/64] fix(database): creating databases under the data source failed (#3037) --- .../odc/service/connection/database/DatabaseService.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java index dd4e693135..63a87a1600 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java @@ -92,6 +92,7 @@ import com.oceanbase.odc.service.connection.database.model.CreateDatabaseReq; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.connection.database.model.DatabaseSyncStatus; +import com.oceanbase.odc.service.connection.database.model.DatabaseType; import com.oceanbase.odc.service.connection.database.model.DatabaseUser; import com.oceanbase.odc.service.connection.database.model.DeleteDatabasesReq; import com.oceanbase.odc.service.connection.database.model.ModifyDatabaseOwnerReq; @@ -352,6 +353,8 @@ public Database create(@NonNull CreateDatabaseReq req) { database.setOrganizationId(authenticationFacade.currentOrganizationId()); database.setLastSyncTime(new Date(System.currentTimeMillis())); database.setObjectSyncStatus(DBObjectSyncStatus.INITIALIZED); + database.setDialectType(connection.getDialectType()); + database.setType(DatabaseType.PHYSICAL); DatabaseEntity saved = databaseRepository.saveAndFlush(database); List userResourceRoles = buildUserResourceRoles(Collections.singleton(saved.getId()), req.getOwnerIds()); From 12b524cd0e1b503292921c7d4a1cadc43af3b5b5 Mon Sep 17 00:00:00 2001 From: pynzzZ <39962741+MarkPotato777@users.noreply.github.com> Date: Mon, 29 Jul 2024 16:05:07 +0800 Subject: [PATCH 52/64] fix(table-permission): not select the specific database/table by default when create the permission application ticket (#3035) --- .../database/model/UnauthorizedDBResource.java | 4 +++- .../service/permission/DBResourcePermissionHelper.java | 10 ++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/UnauthorizedDBResource.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/UnauthorizedDBResource.java index fa5d07f513..65e163ce23 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/UnauthorizedDBResource.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/UnauthorizedDBResource.java @@ -34,9 +34,10 @@ public class UnauthorizedDBResource extends DBResource { private Boolean applicable; private Set unauthorizedPermissionTypes; + private Long projectId; public static UnauthorizedDBResource from(DBResource dbResource, Set types, - boolean applicable) { + boolean applicable, Long projectId) { UnauthorizedDBResource obj = new UnauthorizedDBResource(); obj.setType(dbResource.getType()); obj.setDialectType(dbResource.getDialectType()); @@ -48,6 +49,7 @@ public static UnauthorizedDBResource from(DBResource dbResource, Set filterUnauthorizedDBResources( } if (resource.getType() == ResourceType.ODC_DATABASE) { if (resource.getDatabaseId() == null) { - unauthorizedDBResources.add(UnauthorizedDBResource.from(resource, needs, false)); + unauthorizedDBResources.add(UnauthorizedDBResource.from(resource, needs, false, null)); continue; } DatabaseEntity database = dbId2Entity.get(resource.getDatabaseId()); @@ -346,11 +346,12 @@ public List filterUnauthorizedDBResources( needs.stream().filter(p -> !authorized.contains(p)).collect(Collectors.toSet()); if (CollectionUtils.isNotEmpty(unauthorized)) { unauthorizedDBResources.add(UnauthorizedDBResource.from(resource, unauthorized, - database.getProjectId() != null && involvedProjectIds.contains(database.getProjectId()))); + database.getProjectId() != null && involvedProjectIds.contains(database.getProjectId()), + database.getProjectId())); } } else if (resource.getType() == ResourceType.ODC_TABLE) { if (resource.getDatabaseId() == null) { - unauthorizedDBResources.add(UnauthorizedDBResource.from(resource, needs, false)); + unauthorizedDBResources.add(UnauthorizedDBResource.from(resource, needs, false, null)); continue; } Set authorized = new HashSet<>(); @@ -376,7 +377,8 @@ public List filterUnauthorizedDBResources( needs.stream().filter(p -> !authorized.contains(p)).collect(Collectors.toSet()); if (CollectionUtils.isNotEmpty(unauthorized)) { unauthorizedDBResources.add(UnauthorizedDBResource.from(resource, unauthorized, - database.getProjectId() != null && involvedProjectIds.contains(database.getProjectId()))); + database.getProjectId() != null && involvedProjectIds.contains(database.getProjectId()), + database.getProjectId())); } } else { throw new IllegalStateException("Unsupported resource type: " + resource.getType()); From 41ede5a9304aa9600acf94c39c3239f0924bd2d0 Mon Sep 17 00:00:00 2001 From: guowl3 <70270140+guowl3@users.noreply.github.com> Date: Mon, 29 Jul 2024 16:48:37 +0800 Subject: [PATCH 53/64] fix(dlm): several bug related to editing data cleaning (#3033) --- .../CreateFlowInstanceProcessAspect.java | 5 ++- .../flowtask/DataDeletePreprocessor.java | 34 ++++++++++++++----- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/processor/CreateFlowInstanceProcessAspect.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/processor/CreateFlowInstanceProcessAspect.java index ec9abbe225..1ac7deb550 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/processor/CreateFlowInstanceProcessAspect.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/processor/CreateFlowInstanceProcessAspect.java @@ -184,7 +184,10 @@ private void validateTriggerConfig(CreateFlowInstanceReq req) { } if (req.getParameters() instanceof AlterScheduleParameters) { AlterScheduleParameters parameters = (AlterScheduleParameters) req.getParameters(); - validateTriggerConfig(parameters.getTriggerConfig()); + if (parameters.getOperationType() == OperationType.CREATE + || parameters.getOperationType() == OperationType.UPDATE) { + validateTriggerConfig(parameters.getTriggerConfig()); + } } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DataDeletePreprocessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DataDeletePreprocessor.java index d48cf1a00c..26f6aebf91 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DataDeletePreprocessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DataDeletePreprocessor.java @@ -33,6 +33,7 @@ import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.schedule.ScheduleService; import com.oceanbase.odc.service.schedule.model.JobType; +import com.oceanbase.odc.service.schedule.model.ScheduleStatus; import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; import lombok.extern.slf4j.Slf4j; @@ -64,7 +65,8 @@ public class DataDeletePreprocessor extends AbstractDlmJobPreprocessor { @Override public void process(CreateFlowInstanceReq req) { AlterScheduleParameters parameters = (AlterScheduleParameters) req.getParameters(); - if (parameters.getOperationType() == OperationType.CREATE) { + if (parameters.getOperationType() == OperationType.CREATE + || parameters.getOperationType() == OperationType.UPDATE) { DataDeleteParameters dataDeleteParameters = (DataDeleteParameters) parameters.getScheduleTaskParameters(); initDefaultConfig(dataDeleteParameters); @@ -94,13 +96,29 @@ public void process(CreateFlowInstanceReq req) { } log.info("QUICK-DELETE job preprocessing has been completed."); // pre create - ScheduleEntity scheduleEntity = buildScheduleEntity(req); - scheduleEntity.setCreatorId(authenticationFacade.currentUser().id()); - scheduleEntity.setModifierId(scheduleEntity.getCreatorId()); - scheduleEntity.setOrganizationId(authenticationFacade.currentOrganizationId()); - scheduleEntity = scheduleService.create(scheduleEntity); - parameters.setTaskId(scheduleEntity.getId()); - initLimiterConfig(scheduleEntity.getId(), dataDeleteParameters.getRateLimit(), limiterService); + if (parameters.getOperationType() == OperationType.CREATE) { + ScheduleEntity scheduleEntity = buildScheduleEntity(req); + scheduleEntity.setCreatorId(authenticationFacade.currentUser().id()); + scheduleEntity.setModifierId(scheduleEntity.getCreatorId()); + scheduleEntity.setOrganizationId(authenticationFacade.currentOrganizationId()); + scheduleEntity = scheduleService.create(scheduleEntity); + parameters.setTaskId(scheduleEntity.getId()); + initLimiterConfig(scheduleEntity.getId(), dataDeleteParameters.getRateLimit(), limiterService); + } + + if (parameters.getOperationType() == OperationType.UPDATE) { + parameters.setDescription(req.getDescription()); + ScheduleEntity scheduleEntity = scheduleService.nullSafeGetById(parameters.getTaskId()); + if (scheduleEntity.getStatus() != ScheduleStatus.PAUSE + || scheduleService.hasExecutingScheduleTask(scheduleEntity.getId())) { + throw new IllegalStateException( + String.format( + "The task can only be edited when it is paused and there are no active subtasks executing,status=%s", + scheduleEntity.getStatus())); + } + // update job limit config + limiterService.updateByOrderId(scheduleEntity.getId(), dataDeleteParameters.getRateLimit()); + } } req.setParentFlowInstanceId(parameters.getTaskId()); } From 3b2fbccb7293b364e74e992170ce72208a30d9f1 Mon Sep 17 00:00:00 2001 From: guowl3 <70270140+guowl3@users.noreply.github.com> Date: Mon, 29 Jul 2024 17:09:38 +0800 Subject: [PATCH 54/64] fix(partition-plan): can not recognize the partition key's data type on mysql mode (#3039) * bugfix * response comments. * response comments. --- .../partitionplan/OBMySQLAutoPartitionExtensionPoint.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/plugins/task-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/task/obmysql/partitionplan/OBMySQLAutoPartitionExtensionPoint.java b/server/plugins/task-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/task/obmysql/partitionplan/OBMySQLAutoPartitionExtensionPoint.java index 0e771b33fc..ed77cf2401 100644 --- a/server/plugins/task-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/task/obmysql/partitionplan/OBMySQLAutoPartitionExtensionPoint.java +++ b/server/plugins/task-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/task/obmysql/partitionplan/OBMySQLAutoPartitionExtensionPoint.java @@ -128,7 +128,7 @@ public boolean supports(@NonNull DBTablePartition partition) { @Override public String unquoteIdentifier(@NonNull String identifier) { - return StringUtils.unquoteMySqlIdentifier(identifier); + return StringUtils.unquoteMySqlIdentifier(identifier.toLowerCase()); } @Override From f60ef63b15a7e31820756713a58391c872409221 Mon Sep 17 00:00:00 2001 From: pynzzZ <39962741+MarkPotato777@users.noreply.github.com> Date: Mon, 29 Jul 2024 21:09:14 +0800 Subject: [PATCH 55/64] fix(table-permission): could create tickets when users have no permission to the database (#3046) --- .../odc/service/permission/DBResourcePermissionHelper.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java index 5fd02abca2..2329828fd8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java @@ -425,9 +425,10 @@ private Map> getInnerDBPermissionTypes(Collect })); Map> typesFromTable = userTablePermissionRepository .findNotExpiredByUserIdAndDatabaseIdIn(authenticationFacade.currentUserId(), databaseIds) - .stream() - .collect(Collectors.groupingBy(UserTablePermissionEntity::getDatabaseId, - Collectors.mapping(e -> DatabasePermissionType.from(e.getAction()), Collectors.toSet()))); + .stream().collect(Collectors.toMap( + UserTablePermissionEntity::getDatabaseId, + e -> Collections.singleton(DatabasePermissionType.ACCESS), + (e1, e2) -> e1)); typesFromTable.forEach((k, v) -> typesFromDatabase.merge(k, v, (v1, v2) -> { v1.addAll(v2); return v1; From fb73278901536a216fdccc8a396e3dfe1c07b9de Mon Sep 17 00:00:00 2001 From: pynzzZ <39962741+MarkPotato777@users.noreply.github.com> Date: Mon, 29 Jul 2024 21:10:29 +0800 Subject: [PATCH 56/64] fix(table-permission): table permission apply tickets warning log not found (#3049) --- server/odc-server/src/main/resources/log4j2.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/odc-server/src/main/resources/log4j2.xml b/server/odc-server/src/main/resources/log4j2.xml index be56baccec..8e692d90cc 100644 --- a/server/odc-server/src/main/resources/log4j2.xml +++ b/server/odc-server/src/main/resources/log4j2.xml @@ -385,8 +385,8 @@ + fileName="${LOG_DIRECTORY}/apply-table/${ctx:taskWorkSpace}/${ctx:taskId}/apply-table-task.warn" + filePattern="${LOG_DIRECTORY}/apply-table/${ctx:taskWorkSpace}/${ctx:taskId}/${date:yyyy-MM}/apply-table-task-%d{yyyy-MM-dd}-%i.warn.gz"> %d{yyyy-MM-dd HH:mm:ss} %p %c{1.} - %m%n From 82cdc2d56a38f51f4edbdbb6ebf653eebc3c3d8d Mon Sep 17 00:00:00 2001 From: guowl3 <70270140+guowl3@users.noreply.github.com> Date: Tue, 30 Jul 2024 10:31:10 +0800 Subject: [PATCH 57/64] fix(dlm): create the target table if the sync table structure is off in MySQL mode (#3050) --- .../odc/service/dlm/DLMTableStructureSynchronizer.java | 5 +++++ .../oceanbase/odc/service/schedule/job/AbstractDlmJob.java | 3 +-- .../odc/service/task/executor/task/DataArchiveTask.java | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMTableStructureSynchronizer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMTableStructureSynchronizer.java index 84b9945bf9..411908d7ea 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMTableStructureSynchronizer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMTableStructureSynchronizer.java @@ -60,6 +60,11 @@ public static void sync(DataSourceInfo sourceInfo, DataSourceInfo targetInfo, St public static void sync(ConnectionConfig srcConfig, ConnectionConfig tgtConfig, String srcTableName, String tgtTableName, Set targetType) throws Exception { + if (!srcConfig.getDialectType().isMysql() || srcConfig.getDialectType() != tgtConfig.getDialectType()) { + log.warn("Table structure is unsupported,sourceDbType={},targetDbType={}", srcConfig.getDialectType(), + tgtConfig.getDialectType()); + return; + } DataSource sourceDs = new DruidDataSourceFactory(srcConfig).getDataSource(); DataSource targetDs = new DruidDataSourceFactory(tgtConfig).getDataSource(); try { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java index a279b5521c..f42012bf88 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java @@ -114,8 +114,7 @@ public void executeTask(Long taskId, List dlmTableUnits, Long time dlmTableUnit.getTableName()); continue; } - if (dlmTableUnit.getType() == JobType.MIGRATE - && !dlmTableUnit.getParameters().getSyncDBObjectType().isEmpty()) { + if (dlmTableUnit.getType() == JobType.MIGRATE) { try { DLMTableStructureSynchronizer.sync(dlmTableUnit.getSourceDatasourceInfo(), dlmTableUnit.getTargetDatasourceInfo(), dlmTableUnit.getTableName(), diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DataArchiveTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DataArchiveTask.java index 9f01c00d27..b342ba195c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DataArchiveTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DataArchiveTask.java @@ -88,7 +88,7 @@ protected boolean doStart(JobContext context) throws Exception { log.info("The table had been completed,tableName={}", dlmTableUnit.getTableName()); continue; } - if (parameters.getJobType() == JobType.MIGRATE && !parameters.getSyncTableStructure().isEmpty()) { + if (parameters.getJobType() == JobType.MIGRATE) { try { DLMTableStructureSynchronizer.sync( DataSourceInfoMapper.toConnectionConfig(parameters.getSourceDs()), From 64434075011282d593cc0bf504d8aab671a0a494 Mon Sep 17 00:00:00 2001 From: LuckyLeo Date: Wed, 31 Jul 2024 10:25:11 +0800 Subject: [PATCH 58/64] docs: update 4.3.1 changelog (#3038) * update changelog * update changelog * update changelog * update changelog * response to CR comments --- CHANGELOG-zh-CN.md | 75 ++++++++++++++++++++++++++++++++++++++++ CHANGELOG.md | 78 +++++++++++++++++++++++++++++++++++++++++ CHANGELOG.rst | 86 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 239 insertions(+) diff --git a/CHANGELOG-zh-CN.md b/CHANGELOG-zh-CN.md index a372c7847c..56b1937217 100644 --- a/CHANGELOG-zh-CN.md +++ b/CHANGELOG-zh-CN.md @@ -1,5 +1,80 @@ # OceanBase Developer Center (ODC) CHANGELOG +## 4.3.1 (2024-07-) + +### 功能变化 + +变更风险管控 + +- 新增表级别权限管控，允许项目成员对不同表拥有不同操作权限，包括对表的查询、变更和导出操作，进一步加强管控协同能力 + +会话管理 + +- 关闭会话/查询支持非直连 OBServer 的更多场景 + - 当连接为 OceanBase 4.2.3 和 obproxy 4.2.5 或更高的版本时，使用 Client session 能力进行会话管理 + - 在 OceanBase 4.2.1 oracle 模式下，使用匿名块进行会话管理 + +AP 开发 + +- 新增实时执行剖析，可视化、交互式呈现 sql_plan_monitor + - 此功能需要数据源版本 OceanBase 4.2.4+ + - 不仅支持对已完成的 SQL 执行进行分析，也支持对执行中的 SQL 进行实时分析 + - 提供执行计划的图形视图、表格视图和文本视图，直观展示算子之间的连接关系和步骤顺序 + - 全局视图提供 Top5 耗时算子的排序和各项执行耗时阶段的全局汇总，快速定位性能瓶颈 + - 算子节点包含算子的执行状态和详情信息，详细信息包括 CPU、内存、磁盘、吐行和节点属性 + - 对于并行执行节点，支持按照 DB 耗时、IO 内存和吐行行数排序，快速定位数据倾斜，不仅支持分析单机执行计划也支持分析分布式执行计划 + - 全新设计的一体化实时诊断页面，可以结合执行计划、全链路诊断，在一个页面完成执行剖析 + +SQL 开发 + +- SQL 执行时支持查看执行进度，包括：总执行条数、当前执行条数及当前正在执行 SQL 的 trace id；支持实时查看已完成的执行结果 +- 支持以图形格式查看 OceanBase 的逻辑 SQL 执行计划 + +数据源 + +- 全功能适配 OB 4.2.4、OB 4.3.1、OB 4.3.2 + + +### 易用性改进 + +- 数据清理任务支持编辑任务配置 +- 数据源模块，批量导入支持 mysql、oracle 和 doris 数据源 + +### 缺陷修复 + +数据生命周期管理 +- 未启用表结构同步时也进行了表结构比较 [#3014](https://github.com/oceanbase/odc/pull/3014) + +变更风险管控 +- 自动授权规则对 LoginSuccess 事件不生效 [#3003](https://github.com/oceanbase/odc/pull/3003) + +导入导出 +- 桌面版模式下，重新安装 ODC 场景，导入任务可能会受到历史执行任务导入文件影响 [#3006](https://github.com/oceanbase/odc/pull/3006) + +SQL 检查 + +- 启用 SQL 窗口规范时，SQL 窗口的提交、回滚按钮可能会失效 [#2985](https://github.com/oceanbase/odc/pull/2985) + +SQL 开发 +- PL 调试期间可能会报 NPE 异常 [#2930](https://github.com/oceanbase/odc/pull/2930) +- oracle 数据源修改会话变量的 sql 错误 [#2872](https://github.com/oceanbase/odc/pull/2872) + +模拟数据 +- 模拟数据任务无法终止 [#2850](https://github.com/oceanbase/odc/pull/2850) + +全局对象检索 +- 对象同步无法停止 [#2928](https://github.com/obase/odc/pull/2928) + +工单 +- 检查结果文件不存在本机时，无法获取 SQL 检查的结果 [#2943](https://github.com/oceanbase/odc/pull/2943) + +审计 +- `content_ip_address`列值的实际长度超过了该列的长度限制 [#2863](https://github.com/oceanbase/odc/pull/2863) + +其他 +- 多节点部署使用进程模式调度任务时，所有任务会被调度到同一个节点 [#2408](https://github.com/oceanbase/odc/pull/2408) + + ## 4.3.0_bp1（2024-06-24） ### 易用性改进 diff --git a/CHANGELOG.md b/CHANGELOG.md index b1cea66f55..59399efc80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,81 @@ +## 4.3.1 (2024-07-) + +Change Risk Control + +- Added table level permission control. Allow project members to have different permissions for different tables. Including query, change, and export operations on tables. + +Session Management + +- Killing query/session supports more scenarios for non directly connected OBServer + - When killing session on OceanBase 4.2.3 and obproxy 4.2.5, use global client session to solve unknown thread id exception. + - Using anonymous block to kill session in OceanBase 4.2.1 oracle mode. + +AP Development + +- Added query profile feature, which supports querying SQL execution details during or completed execution for OceanBase 4.2.4 or higher + - Provides graphical, tabular, and textual views of OceanBase's actual execution plan, visually displaying the connection relationships and step sequences between operators + - The query profile supports displaying the execution status of operators, providing a global summary of various execution time stages and detailed information at the operator level. It also supports sorting of Top5 time-consuming operators, making it easy to quickly identify and locate performance bottlenecks + - Implemented the ability to collect and analyze SQL execution data in real-time. Support viewing of I/O data and output row count for each thread during standalone/distributed execution + +- Added real-time query profile, which supports visual and interactive presentation of sql_plan_monitor + - This feature requires the data source version OceanBase 4.2.4+ + - Not only does it support analysis of completed SQL executions, but also supports real-time analysis of executing SQL + - Provide graphical, tabular, and textual views for executing plans, visually displaying the connection relationships and step sequences between operators + - The global view provides sorting of Top5 time-consuming operators and a summary of each execution time-consuming stage, helping identifying performance bottlenecks + - The operator node contains the execution status and detailed information, including CPU, memory, disk, output rows, and node properties + - For parallel execution nodes, it supports sorting by DB time, IO memory, and number of rows to quickly locate data skew. It not only supports analyzing standalone execution plans but also supports analyzing distributed execution plans + - The newly designed integrated real-time diagnostic page can combine execution plans and full link trace to complete execution analysis in one page + +SQL Development + +- SQL execution supports viewing execution progress, including: total count of executions, current number, and trace ID of the SQL currently being executed; Support real-time viewing of completed execution results +- Support viewing the logical execution plan of OceanBase in graphical format + +Data Source + +- Adapted to OceanBase 4.2.4/4.3.1/4.3.2 + +### Usability Improvements + +- SQL execution supports obtaining execution progress. And the completed execution results could be viewed in real time. +- Data cleaning tasks support editing task configuration. +- Support import mysql, oracle and doris datasource. + +### Bug Fixes + +DLM +- Even if syncTableStructure config is off, the table structure would still be compared [#3014](https://github.com/oceanbase/odc/pull/3014) + +Change Risk Control +- Automatic authorization rules do not take effect for LoginSuccess events [#3003](https://github.com/oceanbase/odc/pull/3003) + +Data Transfer +- If ODC was reinstalled, dirty data files may be imported [#3006](https://github.com/oceanbase/odc/pull/3006) + +SQL Check + +- When the SQL specification is enabled, the submit and rollback function of SQL window may be invalid [#2985](https://github.com/oceanbase/odc/pull/2985) + +SQL Development +- NPE would be reported during the pl debugging [#2930](https://github.com/oceanbase/odc/pull/2930) +- Modifying session sql in oracle is incorrect [#2872](https://github.com/oceanbase/odc/pull/2872) + +Mock Data +- Unable to cancel the mock data task [#2850](https://github.com/oceanbase/odc/pull/2850) + +Global Object Search +- Unable to stop data object synchronization [#2928](https://github.com/obase/odc/pull/2928) + +Ticket +- Failed to get sql check result when the check result file is not on this machine [#2943](https://github.com/oceanbase/odc/pull/2943) + + +Audit +- The actual length of the value in the 'content_ip_address' column exceeds the length limit of the column [#2863](https://github.com/oceanbase/odc/pull/2863) + +Others +- Daemon job in cluster model would be fired at one time [#2408](https://github.com/oceanbase/odc/pull/2408) + ## 4.3.0_bp1 (2024-06-24) ### Usability Improvements diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8ea453ffdb..728b075fec 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,92 @@ (unreleased) ------------ +New +~~~ +- Feat(kill session): adapt global client session and using block in + oracle model in kill session (#2978) [zijia.cj] +- Feat(server): allows some beans to be loaded only in server mode + (#2757) [Ang] +- Feat(sql-execute): supports service layer for query profile (#2423) + [LuckyLeo] +- Feat(CI): support run build release by ob farm (#2738) [niyuhang] +- Feat(osc): add rate limiter for osc (#2402) [krihy] +- Feat(logical-database): logical database metadata management (#2358) + [pynzzZ] +- Feat(config):add creator_id to config entity (#2485) [Ang] +- Feat(table-permission): supports table level authority control (#2324) + [XiaoYang, isadba] + +Changes +~~~~~~~ +- Refactor(statefulRoute): generalization plDebugsession to UUID stateId + (#2960) [Ang] +- Refactor: change the CODEOWNERS (#2931) [IL MARE] +- Chore: update client version (#2821) [Xiao Kang] +- Refactor(flow): add organizationId to flow instance detail (#2841) + [Ang] +- Refactor(config): Add more fields to configEntity #2493. [Ang] + +Fix +~~~ +- Fix(dlm): don't compare the table structure if syncTableStructure is + off (#3014) [guowl3] +- Fix(login): set max_login_record_time_minutes default value to 0 in + web mode (#3003) [Ang] +- Fix(data-transfer): clean work directory before import (#3006) + [LuckyLeo] +- Fix(alarm): task alarm add exception message (#3004) [Ang] +- Fix(query-profile): modified the version supporting query profile + (#3002) [LuckyLeo] +- Fix(diagnose): failed to view query profile for distributed OB (#2945) + [LuckyLeo] +- Fix(db-browser): failed to recognize the commit and rollback statement + (#2985) [IL MARE] +- Fix(global-search): unable to stop data object synchronization (#2928) + [IL MARE] +- Fix(pre-check): failed to get sql check result when the check result + file is not on this machine (#2943) [IL MARE] +- Fix(security): update oauth2 client version (#2981) [Ang] +- Fix(stateful): remove wrong condition (#2975) [Ang] +- Fix(osc): the online schema change blocked when rate limiter modified + before swap table action (#2908) [LioRoger] +- Fix(web): modify tomcat keepAliveTimeout to 70 seconds (#2964) [Ang] +- Fix(statefulRoute): failed to list built-in snippets (#2935) [Ang] +- Fix(permission): fail to submit ticket if lack of database permission + (#2946) [LuckyLeo] +- Fix(statefulRoute): fix list column can't reach (#2953) [Ang] +- Fix(import): add template api and supports mysql, oracle and doris + datasource importing (#2936) [IL MARE] +- Fix(pl-debug): avoid npe during the pl debugging (#2930) [IL MARE] +- Fix(mock data): failed to cancel the mock data task (#2850) [zijia.cj] +- Fix(sql): the sql of modifying session parameter in oracle is error + (#2872) [zijia.cj] +- Fix(migrate): fix login process resource load faild (#2883) + [yiminpeng] +- Fix(flow): failed to startup a ticket (#2798) [IL MARE] +- Fix(audit): client ip length more langer then audit column + client_ip_address (#2863) [CHLK] +- Fix(ob-sql-parser): failed to recognize interval expression in ob- + oracle mode (#2873) [IL MARE] +- Fix(data viewing): get result-set timeout (#2848) [zijia.cj] +- Fix(database-permission): mistake caused by code merge (#2786) + [XiaoYang] +- Fix(metadb): change systemConfigDao to systemConfigRepository. (#2467) + [Ang] +- Fix(deserialization): failed to deserialize the page object (#2434) + [Ang] +- Fix(taskframework): daemon job be fired at one time in cluster model + (#2408) [krihy] + +Security +~~~~~~~~ +- Security: modify annotations on some service classes (#2955) + [LuckyLeo] + + +v4.3.0_bp1 (2024-06-24) +----------------------- + Fix ~~~ - Fix(sql-check): remove the word 'id' from the reserved words (#2796) From e2ed75414fac2035858958909aaa9af74d2df385 Mon Sep 17 00:00:00 2001 From: pynzzZ <39962741+MarkPotato777@users.noreply.github.com> Date: Wed, 31 Jul 2024 12:00:38 +0800 Subject: [PATCH 59/64] fix(database-permission): wrongly recognize packages as schemas (#3067) --- .../flow/task/PreCheckRuntimeFlowableTask.java | 10 +++++++++- .../interceptor/DBResourcePermissionInterceptor.java | 11 ++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTask.java index 1dc8b16333..20acb46ece 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTask.java @@ -23,6 +23,7 @@ import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -393,9 +394,16 @@ private void doSqlCheckAndResourcePermissionCheck(TaskEntity preCheckTaskEntity, private List getUnauthorizedDBResources(List sqls, ConnectionConfig config, String defaultSchema, TaskType taskType) { + Set existedDatabaseNames = + databaseService.listDatabasesByConnectionIds(Collections.singleton(connectionConfig.getId())) + .stream().filter(database -> database.getExisted()).map(database -> database.getName()) + .collect(Collectors.toSet()); Map> identity2Types = DBSchemaExtractor.listDBSchemasWithSqlTypes( sqls.stream().map(e -> SqlTuple.newTuple(e.getStr())).collect(Collectors.toList()), - config.getDialectType(), defaultSchema); + config.getDialectType(), defaultSchema).entrySet().stream() + .filter(entry -> Objects.isNull(entry.getKey().getSchema()) + || existedDatabaseNames.contains(entry.getKey().getSchema())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));; Map> resource2PermissionTypes = new HashMap<>(); for (Entry> entry : identity2Types.entrySet()) { DBSchemaIdentity identity = entry.getKey(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/DBResourcePermissionInterceptor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/DBResourcePermissionInterceptor.java index fc65984fd9..c4450b197d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/DBResourcePermissionInterceptor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/DBResourcePermissionInterceptor.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.service.session.interceptor; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -80,10 +81,18 @@ public boolean doPreHandle(@NonNull SqlAsyncExecuteReq request, @NonNull SqlAsyn return true; } ConnectionConfig connectionConfig = (ConnectionConfig) ConnectionSessionUtil.getConnectionConfig(session); + Set existedDatabaseNames = + databaseService.listDatabasesByConnectionIds(Collections.singleton(connectionConfig.getId())) + .stream().filter(database -> database.getExisted()).map(database -> database.getName()) + .collect(Collectors.toSet()); + String currentSchema = ConnectionSessionUtil.getCurrentSchema(session); Map> identity2Types = DBSchemaExtractor.listDBSchemasWithSqlTypes( response.getSqls().stream().map(SqlTuplesWithViolation::getSqlTuple).collect(Collectors.toList()), - session.getDialectType(), currentSchema); + session.getDialectType(), currentSchema).entrySet().stream() + .filter(entry -> Objects.isNull(entry.getKey().getSchema()) + || existedDatabaseNames.contains(entry.getKey().getSchema())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); Map> resource2PermissionTypes = new HashMap<>(); for (Entry> entry : identity2Types.entrySet()) { DBSchemaIdentity identity = entry.getKey(); From d3cad08799c2f307f93ecc0e8399f03be6c200db Mon Sep 17 00:00:00 2001 From: pynzzZ <39962741+MarkPotato777@users.noreply.github.com> Date: Wed, 31 Jul 2024 15:38:02 +0800 Subject: [PATCH 60/64] fix(flow): wrong approval flow for database/table permission apply ticket (#3072) --- .../odc/service/flow/task/PreCheckRuntimeFlowableTask.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTask.java index 20acb46ece..85afd5c154 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTask.java @@ -77,6 +77,7 @@ import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; import com.oceanbase.odc.service.permission.table.TablePermissionService; import com.oceanbase.odc.service.regulation.approval.ApprovalFlowConfigSelector; +import com.oceanbase.odc.service.regulation.risklevel.RiskLevelService; import com.oceanbase.odc.service.regulation.risklevel.model.RiskLevel; import com.oceanbase.odc.service.regulation.risklevel.model.RiskLevelDescriber; import com.oceanbase.odc.service.resultset.ResultSetExportTaskParameter; @@ -126,6 +127,8 @@ public class PreCheckRuntimeFlowableTask extends BaseODCFlowTaskDelegate { private PreCheckTaskProperties preCheckTaskProperties; @Autowired private ObjectStorageFacade storageFacade; + @Autowired + private RiskLevelService riskLevelService; @Autowired private TablePermissionService tablePermissionService; @@ -221,6 +224,9 @@ protected Void start(Long taskId, TaskService taskService, DelegateExecution exe .map(approvalFlowConfigSelector::select) .max(Comparator.comparingInt(RiskLevel::getLevel)) .orElseThrow(() -> new IllegalStateException("Unknown error")); + } else if (taskEntity.getTaskType() == TaskType.APPLY_DATABASE_PERMISSION + || taskEntity.getTaskType() == TaskType.APPLY_TABLE_PERMISSION) { + riskLevel = riskLevelService.findHighestRiskLevel(); } else if (riskLevelDescriber != null) { riskLevel = approvalFlowConfigSelector.select(riskLevelDescriber); } else { From 3bb724210ed9c948aa2a125c81507ae682ab2658 Mon Sep 17 00:00:00 2001 From: pynzzZ <39962741+MarkPotato777@users.noreply.github.com> Date: Wed, 31 Jul 2024 15:52:56 +0800 Subject: [PATCH 61/64] fix(table-permission): creating table needs table change permissions (#3057) * fix table permission * fix export all db objects * response to comments * code format * resolve conflicts --- .../connection/database/model/DBResource.java | 10 ++--- .../odc/service/flow/FlowInstanceService.java | 9 ++++- .../task/PreCheckRuntimeFlowableTask.java | 20 ++-------- .../DBResourcePermissionHelper.java | 39 +++++++++++++++++++ .../DBResourcePermissionInterceptor.java | 18 +-------- 5 files changed, 55 insertions(+), 41 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/DBResource.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/DBResource.java index 3d1c0d9bb5..0bd400f188 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/DBResource.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/model/DBResource.java @@ -37,19 +37,15 @@ public class DBResource { private Long tableId; private String tableName; - public static DBResource from(ConnectionConfig dataSource, String databaseName, String tableName) { + public static DBResource from(ConnectionConfig dataSource, String databaseName, String tableName, + ResourceType type) { DBResource obj = new DBResource(); obj.setDataSourceId(dataSource.getId()); obj.setDataSourceName(dataSource.getName()); obj.setDialectType(dataSource.getDialectType()); obj.setDatabaseName(databaseName); obj.setTableName(tableName); - if (databaseName != null) { - obj.setType(ResourceType.ODC_DATABASE); - } - if (tableName != null) { - obj.setType(ResourceType.ODC_TABLE); - } + obj.setType(type); return obj; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java index e1e2c3f03a..7f95e48f1f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java @@ -783,10 +783,17 @@ private void checkCreateFlowInstancePermission(CreateFlowInstanceReq req) { ConnectionConfig config = connectionService.getBasicWithoutPermissionCheck(req.getConnectionId()); parameters.getExportDbObjects().forEach(item -> { if (item.getDbObjectType() == ObjectType.TABLE) { - resource2Types.put(DBResource.from(config, req.getDatabaseName(), item.getObjectName()), + resource2Types.put( + DBResource.from(config, req.getDatabaseName(), item.getObjectName(), + ResourceType.ODC_TABLE), DatabasePermissionType.from(TaskType.EXPORT)); } }); + } else if (parameters.isExportAllObjects()) { + ConnectionConfig config = connectionService.getBasicWithoutPermissionCheck(req.getConnectionId()); + resource2Types.put( + DBResource.from(config, req.getDatabaseName(), null, ResourceType.ODC_DATABASE), + DatabasePermissionType.from(TaskType.EXPORT)); } List unauthorizedDBResources = this.permissionHelper .filterUnauthorizedDBResources(resource2Types, false); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTask.java index 85afd5c154..8a11c67d4b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTask.java @@ -25,10 +25,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -409,21 +407,9 @@ private List getUnauthorizedDBResources(List Objects.isNull(entry.getKey().getSchema()) || existedDatabaseNames.contains(entry.getKey().getSchema())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));; - Map> resource2PermissionTypes = new HashMap<>(); - for (Entry> entry : identity2Types.entrySet()) { - DBSchemaIdentity identity = entry.getKey(); - Set sqlTypes = entry.getValue(); - if (CollectionUtils.isNotEmpty(sqlTypes)) { - Set permissionTypes = sqlTypes.stream().map(DatabasePermissionType::from) - .filter(Objects::nonNull).collect(Collectors.toSet()); - permissionTypes.addAll(DatabasePermissionType.from(taskType)); - if (CollectionUtils.isNotEmpty(permissionTypes)) { - resource2PermissionTypes.put( - DBResource.from(config, identity.getSchema(), identity.getTable()), permissionTypes); - } - } - } + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + Map> resource2PermissionTypes = + DBResourcePermissionHelper.getDBResource2PermissionTypes(identity2Types, config, taskType); return dbResourcePermissionHelper.filterUnauthorizedDBResources(resource2PermissionTypes, false); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java index 2329828fd8..7b0201b50a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java @@ -22,6 +22,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.Set; import java.util.TreeMap; @@ -38,6 +39,7 @@ import com.oceanbase.odc.core.shared.constant.OrganizationType; import com.oceanbase.odc.core.shared.constant.ResourceRoleName; import com.oceanbase.odc.core.shared.constant.ResourceType; +import com.oceanbase.odc.core.shared.constant.TaskType; import com.oceanbase.odc.core.shared.exception.AccessDeniedException; import com.oceanbase.odc.core.shared.exception.BadRequestException; import com.oceanbase.odc.metadb.connection.ConnectionConfigRepository; @@ -53,10 +55,13 @@ import com.oceanbase.odc.service.collaboration.project.ProjectService; import com.oceanbase.odc.service.connection.database.model.DBResource; import com.oceanbase.odc.service.connection.database.model.UnauthorizedDBResource; +import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.permission.common.PermissionCheckWhitelist; import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; +import com.oceanbase.odc.service.session.util.DBSchemaExtractor.DBSchemaIdentity; import com.oceanbase.tools.dbbrowser.model.DBObjectType; +import com.oceanbase.tools.dbbrowser.parser.constant.SqlType; /** * @author gaoda.xy @@ -399,6 +404,40 @@ public List filterUnauthorizedDBResources( return unauthorizedDBResources; } + public static Map> getDBResource2PermissionTypes( + Map> identity2Types, ConnectionConfig connectionConfig, TaskType taskType) { + Map> resource2PermissionTypes = new HashMap<>(); + for (Entry> entry : identity2Types.entrySet()) { + DBSchemaIdentity identity = entry.getKey(); + Set sqlTypes = entry.getValue(); + Set permissionTypes = new HashSet<>(); + if (CollectionUtils.isNotEmpty(sqlTypes)) { + if (sqlTypes.contains(SqlType.CREATE)) { + permissionTypes.add(DatabasePermissionType.from(SqlType.CREATE)); + resource2PermissionTypes.put( + DBResource.from(connectionConfig, identity.getSchema(), identity.getTable(), + ResourceType.ODC_DATABASE), + permissionTypes); + sqlTypes.remove(SqlType.CREATE); + } + permissionTypes = sqlTypes.stream().map(DatabasePermissionType::from) + .filter(Objects::nonNull).collect(Collectors.toSet()); + if (Objects.nonNull(taskType)) { + permissionTypes.addAll(DatabasePermissionType.from(taskType)); + } + if (CollectionUtils.isNotEmpty(permissionTypes)) { + resource2PermissionTypes.computeIfAbsent( + DBResource.from(connectionConfig, identity.getSchema(), identity.getTable(), + Objects.isNull(identity.getTable()) ? ResourceType.ODC_DATABASE + : ResourceType.ODC_TABLE), + k -> new HashSet<>()) + .addAll(permissionTypes); + } + } + } + return resource2PermissionTypes; + } + private Set getPermittedProjectIds() { // OWNER, DBA or DEVELOPER can access all databases inner the project Map> projectIds2Roles = projectService.getProjectId2ResourceRoleNames(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/DBResourcePermissionInterceptor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/DBResourcePermissionInterceptor.java index c4450b197d..6fff793d5a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/DBResourcePermissionInterceptor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/DBResourcePermissionInterceptor.java @@ -16,10 +16,8 @@ package com.oceanbase.odc.service.session.interceptor; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -93,20 +91,8 @@ public boolean doPreHandle(@NonNull SqlAsyncExecuteReq request, @NonNull SqlAsyn .filter(entry -> Objects.isNull(entry.getKey().getSchema()) || existedDatabaseNames.contains(entry.getKey().getSchema())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - Map> resource2PermissionTypes = new HashMap<>(); - for (Entry> entry : identity2Types.entrySet()) { - DBSchemaIdentity identity = entry.getKey(); - Set sqlTypes = entry.getValue(); - if (CollectionUtils.isNotEmpty(sqlTypes)) { - Set permissionTypes = sqlTypes.stream().map(DatabasePermissionType::from) - .filter(Objects::nonNull).collect(Collectors.toSet()); - if (CollectionUtils.isNotEmpty(permissionTypes)) { - resource2PermissionTypes.put( - DBResource.from(connectionConfig, identity.getSchema(), identity.getTable()), - permissionTypes); - } - } - } + Map> resource2PermissionTypes = + DBResourcePermissionHelper.getDBResource2PermissionTypes(identity2Types, connectionConfig, null); List unauthorizedDBResource = dbResourcePermissionHelper .filterUnauthorizedDBResources(resource2PermissionTypes, false); if (CollectionUtils.isNotEmpty(unauthorizedDBResource)) { From b99e66145b8662741c3fa7e09260646c4deedae8 Mon Sep 17 00:00:00 2001 From: IL MARE Date: Wed, 31 Jul 2024 16:03:12 +0800 Subject: [PATCH 62/64] fix(parser): failed to recognize the schema or package name from an anonymous block (#3069) * fix anonymous block parse error * fix anonymous block parse error * response to pr comments --- client | 2 +- .../session/util/DBSchemaExtractor.java | 13 +++++++++++++ .../session/DBSchemaExtractorTest.java | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/client b/client index 0b81dd6e73..1c34b729cf 160000 --- a/client +++ b/client @@ -1 +1 @@ -Subproject commit 0b81dd6e73f475787e2b33460e21435f91fd740e +Subproject commit 1c34b729cf6e84b2accf1ee6a4e0af9a8fabe426 diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/DBSchemaExtractor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/DBSchemaExtractor.java index 82508fbd74..4a3bc54d49 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/DBSchemaExtractor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/DBSchemaExtractor.java @@ -61,6 +61,7 @@ import com.oceanbase.tools.sqlparser.oboracle.OBParser.Var_nameContext; import com.oceanbase.tools.sqlparser.oboracle.PLParser.IdentifierContext; import com.oceanbase.tools.sqlparser.oboracle.PLParser.Pl_schema_nameContext; +import com.oceanbase.tools.sqlparser.oboracle.PLParser.Sql_stmtContext; import com.oceanbase.tools.sqlparser.statement.Expression; import com.oceanbase.tools.sqlparser.statement.common.RelationFactor; import com.oceanbase.tools.sqlparser.statement.expression.FunctionCall; @@ -420,6 +421,18 @@ public RelationFactor visitPl_schema_name(Pl_schema_nameContext ctx) { return null; } + @Override + public RelationFactor visitSql_stmt(Sql_stmtContext ctx) { + try { + OBOracleRelationFactorVisitor visitor = new OBOracleRelationFactorVisitor(); + visitor.visit(ctx); + identities.addAll(visitor.getIdentities()); + } catch (Exception e) { + // eat the exception + } + return null; + } + } diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/session/DBSchemaExtractorTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/session/DBSchemaExtractorTest.java index ca2b1a31c9..9314425a87 100644 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/session/DBSchemaExtractorTest.java +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/session/DBSchemaExtractorTest.java @@ -17,6 +17,8 @@ import static org.junit.Assert.*; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -60,6 +62,23 @@ public void test() { } } + @Test + public void listDBSchemasWithSqlTypes_anonymousBlock_listSucceed() { + String pl = "DECLARE\n" + + " i VARCHAR2(300);\n" + + "BEGIN\n" + + " select ps_auto_refresh_publish_pkg.getloopup_meaning('YES_NO', 'Y') into i from dual;\n" + + " dbms_output.put_line(i);\n" + + "END;"; + Map> actual = DBSchemaExtractor.listDBSchemasWithSqlTypes( + Collections.singletonList(SqlTuple.newTuple(pl)), DialectType.OB_ORACLE, "aps"); + Map> expect = new HashMap<>(); + DBSchemaIdentity dbSchemaIdentity = new DBSchemaIdentity(); + dbSchemaIdentity.setSchema("PS_AUTO_REFRESH_PUBLISH_PKG"); + expect.put(dbSchemaIdentity, Collections.singleton(SqlType.OTHERS)); + Assert.assertEquals(expect, actual); + } + @Data @NoArgsConstructor @AllArgsConstructor From 1205f2b4b0d9f2046569afcc4076febcc403031f Mon Sep 17 00:00:00 2001 From: IL MARE Date: Wed, 31 Jul 2024 20:08:14 +0800 Subject: [PATCH 63/64] update submodule (#3076) --- client | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client b/client index 1c34b729cf..5db658d7bf 160000 --- a/client +++ b/client @@ -1 +1 @@ -Subproject commit 1c34b729cf6e84b2accf1ee6a4e0af9a8fabe426 +Subproject commit 5db658d7bfba1b1471301be3c1144b0c22a75020 From a66c22c29387dfd332a4ff23db9362b82897013b Mon Sep 17 00:00:00 2001 From: LuckyLeo Date: Thu, 1 Aug 2024 17:56:52 +0800 Subject: [PATCH 64/64] docs: update 4.3.1 changelog (#3081) --- CHANGELOG.md | 92 +++++++++++++++++++++++++++------------------------ CHANGELOG.rst | 26 +++++++++++++-- 2 files changed, 72 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59399efc80..269d36c36d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,80 +1,84 @@ -## 4.3.1 (2024-07-) +## 4.3.1 (2024-07-31) -Change Risk Control +### Feature Updates -- Added table level permission control. Allow project members to have different permissions for different tables. Including query, change, and export operations on tables. +Risk Control Changes -Session Management +- Added table-level permission control, allowing project members to have different operation permissions on different tables, including query, change, and export operations, enhancing collaborative control capabilities. -- Killing query/session supports more scenarios for non directly connected OBServer - - When killing session on OceanBase 4.2.3 and obproxy 4.2.5, use global client session to solve unknown thread id exception. - - Using anonymous block to kill session in OceanBase 4.2.1 oracle mode. +Session Management -AP Development +- Support for closing sessions/queries has been extended to more scenarios where OBServer is not directly connected. + - When connected to OceanBase V4.2.3 and OBProxy V4.2.5 or higher versions, session management is performed using client session capability. + - In OceanBase V4.2.1 Oracle mode, session management is handled using anonymous blocks. -- Added query profile feature, which supports querying SQL execution details during or completed execution for OceanBase 4.2.4 or higher - - Provides graphical, tabular, and textual views of OceanBase's actual execution plan, visually displaying the connection relationships and step sequences between operators - - The query profile supports displaying the execution status of operators, providing a global summary of various execution time stages and detailed information at the operator level. It also supports sorting of Top5 time-consuming operators, making it easy to quickly identify and locate performance bottlenecks - - Implemented the ability to collect and analyze SQL execution data in real-time. Support viewing of I/O data and output row count for each thread during standalone/distributed execution +OLAP Development -- Added real-time query profile, which supports visual and interactive presentation of sql_plan_monitor - - This feature requires the data source version OceanBase 4.2.4+ - - Not only does it support analysis of completed SQL executions, but also supports real-time analysis of executing SQL - - Provide graphical, tabular, and textual views for executing plans, visually displaying the connection relationships and step sequences between operators - - The global view provides sorting of Top5 time-consuming operators and a summary of each execution time-consuming stage, helping identifying performance bottlenecks - - The operator node contains the execution status and detailed information, including CPU, memory, disk, output rows, and node properties - - For parallel execution nodes, it supports sorting by DB time, IO memory, and number of rows to quickly locate data skew. It not only supports analyzing standalone execution plans but also supports analyzing distributed execution plans - - The newly designed integrated real-time diagnostic page can combine execution plans and full link trace to complete execution analysis in one page +- Added real-time execution profiling, providing visual and interactive presentation of `sql_plan_monitor`. + - This feature requires data source version OceanBase V4.2.4 or higher. + - Supports analysis not only of completed SQL executions but also real-time analysis of ongoing SQL executions. + - Provides graphical, tabular, and text views of execution plans, intuitively displaying operator connections and step sequences. + - Global view offers sorting of Top 5 time-consuming operators and overall summary of execution phases, quickly pinpointing performance bottlenecks. + - Operator nodes include execution status and detailed information such as CPU, memory, disk, output rows, and node attributes. + - For parallel execution nodes, supports sorting by DB time, IO memory, and rows processed, quickly identifying data skew. Supports analysis of both standalone and distributed execution plans. + - Newly designed integrated real-time diagnostic page for comprehensive query profiling combining execution plans and end-to-end trace diagnostics. SQL Development -- SQL execution supports viewing execution progress, including: total count of executions, current number, and trace ID of the SQL currently being executed; Support real-time viewing of completed execution results -- Support viewing the logical execution plan of OceanBase in graphical format +- During SQL execution, supports viewing execution progress, including total number of executions, current execution count, and trace ID of currently executing SQL. Real-time viewing of completed execution results is also supported. +- Supports graphical format viewing of OceanBase's logical SQL execution plan. -Data Source +Data Sources -- Adapted to OceanBase 4.2.4/4.3.1/4.3.2 +- Fully compatible with OceanBase V4.2.4, OceanBase V4.3.1, OceanBase V4.3.2. ### Usability Improvements -- SQL execution supports obtaining execution progress. And the completed execution results could be viewed in real time. -- Data cleaning tasks support editing task configuration. -- Support import mysql, oracle and doris datasource. +- Data delete tasks now support editing task configurations. +- Data source module supports batch import from MySQL, Oracle, and Doris data sources. ### Bug Fixes -DLM -- Even if syncTableStructure config is off, the table structure would still be compared [#3014](https://github.com/oceanbase/odc/pull/3014) +Data Lifecycle Management + +- Table structure comparison performed even when structure synchronization is not enabled. [#3014](https://github.com/oceanbase/odc/pull/3014) Change Risk Control -- Automatic authorization rules do not take effect for LoginSuccess events [#3003](https://github.com/oceanbase/odc/pull/3003) -Data Transfer -- If ODC was reinstalled, dirty data files may be imported [#3006](https://github.com/oceanbase/odc/pull/3006) +- Automatic authorization rules did not take effect for LoginSuccess events. [#3003](https://github.com/oceanbase/odc/pull/3003) + +Import/Export + +- In the desktop mode, reinstalling ODC may lead to the unintended import of historical files generated during previous import tasks when initiating new tasks. [#3006](https://github.com/oceanbase/odc/pull/3006) SQL Check -- When the SQL specification is enabled, the submit and rollback function of SQL window may be invalid [#2985](https://github.com/oceanbase/odc/pull/2985) +- When SQL window rules are enabled, commit and rollback buttons in SQL window may become ineffective. [#2985](https://github.com/oceanbase/odc/pull/2985) SQL Development -- NPE would be reported during the pl debugging [#2930](https://github.com/oceanbase/odc/pull/2930) -- Modifying session sql in oracle is incorrect [#2872](https://github.com/oceanbase/odc/pull/2872) + +- NPE may occur during PL debugging. [#2930](https://github.com/oceanbase/odc/pull/2930) +- SQL error when modifying session variables for Oracle data sources. [#2872](https://github.com/oceanbase/odc/pull/2872) Mock Data -- Unable to cancel the mock data task [#2850](https://github.com/oceanbase/odc/pull/2850) -Global Object Search -- Unable to stop data object synchronization [#2928](https://github.com/obase/odc/pull/2928) +- Unable to terminate mock data tasks. [#2850](https://github.com/oceanbase/odc/pull/2850) -Ticket -- Failed to get sql check result when the check result file is not on this machine [#2943](https://github.com/oceanbase/odc/pull/2943) +Global Object Retrieval +- Object synchronization cannot be stopped. [#2928](https://github.com/obase/odc/pull/2928) + +Tickets -Audit -- The actual length of the value in the 'content_ip_address' column exceeds the length limit of the column [#2863](https://github.com/oceanbase/odc/pull/2863) +- Unable to retrieve SQL check results when check result file does not exist locally. [#2943](https://github.com/oceanbase/odc/pull/2943) + +Auditing + +- Actual length of `content_ip_address` column values exceeds the column length limit. [#2863](https://github.com/oceanbase/odc/pull/2863) + +Other -Others -- Daemon job in cluster model would be fired at one time [#2408](https://github.com/oceanbase/odc/pull/2408) +- When deploying across multiple nodes using process mode to schedule tasks, all tasks may be scheduled to the same node. [#2408](https://github.com/oceanbase/odc/pull/2408) ## 4.3.0_bp1 (2024-06-24) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 728b075fec..3dd4c563b6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,5 @@ -(unreleased) ------------- +v4.3.1 (2024-07-31) +------------------- New ~~~ @@ -29,6 +29,28 @@ Changes Fix ~~~ +- Fix(parser): failed to recognize the schema or package name from an + anonymous block (#3069) [IL MARE] +- Fix(table-permission): creating table needs table change permissions + (#3057) [pynzzZ] +- Fix(flow): wrong approval flow for database/table permission apply + ticket (#3072) [pynzzZ] +- Fix(database-permission): wrongly recognize packages as schemas + (#3067) [pynzzZ] +- Fix(dlm): create the target table if the sync table structure is off + in MySQL mode (#3050) [guowl3] +- Fix(table-permission): table permission apply tickets warning log not + found (#3049) [pynzzZ] +- Fix(table-permission): could create tickets when users have no + permission to the database (#3046) [pynzzZ] +- Fix(partition-plan): can not recognize the partition key's data type + on mysql mode (#3039) [guowl3] +- Fix(dlm): several bug related to editing data cleaning (#3033) + [guowl3] +- Fix(table-permission): not select the specific database/table by + default when create the permission application ticket (#3035) [pynzzZ] +- Fix(database): creating databases under the data source failed (#3037) + [pynzzZ] - Fix(dlm): don't compare the table structure if syncTableStructure is off (#3014) [guowl3] - Fix(login): set max_login_record_time_minutes default value to 0 in