diff --git a/pkg/transfer/config/metadata_options.go b/pkg/transfer/config/metadata_options.go index 524e4346b..07b357017 100644 --- a/pkg/transfer/config/metadata_options.go +++ b/pkg/transfer/config/metadata_options.go @@ -152,6 +152,8 @@ const ( // ResultTableOptLogSeparatorRegexp : 日志正则提取清洗专用,提取字段 ResultTableOptLogSeparatorRegexp = "separator_regexp" + ResultTableOptLogSeparatorConfigs = "separator_configs" + // 事件类 // 结果是否可以使用新的自定义维度 ResultTableOptEventAllowNewDimension = "allow_new_dimension" diff --git a/pkg/transfer/template/etl/plugin.go b/pkg/transfer/template/etl/plugin.go index ddf0bc43c..da457d42d 100644 --- a/pkg/transfer/template/etl/plugin.go +++ b/pkg/transfer/template/etl/plugin.go @@ -69,8 +69,8 @@ func SchemaByResultTablePlugin(table *config.MetaResultTableConfig) etl.Containe } // GetSeparatorFieldByOption -func GetSeparatorFieldByOption(table *config.MetaResultTableConfig) (etl.Field, error) { - helper := utils.NewMapHelper(table.Option) +func GetSeparatorFieldByOption(option map[string]interface{}, table *config.MetaResultTableConfig) (etl.Field, error) { + helper := utils.NewMapHelper(option) action, ok := helper.GetString(config.ResultTableOptSeparatorAction) if !ok { return nil, nil @@ -108,16 +108,52 @@ func GetSeparatorFieldByOption(table *config.MetaResultTableConfig) (etl.Field, } } -// PrepareByResultTablePlugin: 根据字段提取方法[json|regexp|delimiter],解析上报的日志数据内容 +func GetSeparatorFieldByMultiOption(table *config.MetaResultTableConfig) ([]etl.Field, error) { + var fields []etl.Field + field, err := GetSeparatorFieldByOption(table.Option, table) + if err != nil { + return nil, err + } + fields = append(fields, field) + + obj, ok := table.Option[config.ResultTableOptLogSeparatorConfigs] + if !ok { + return fields, nil + } + + option, ok := obj.([]interface{}) + if !ok { + return fields, nil + } + + for _, item := range option { + opt, ok := item.(map[string]interface{}) + if !ok { + continue + } + field, err := GetSeparatorFieldByOption(opt, table) + if err != nil { + continue + } + fields = append(fields, field) + } + return fields, nil +} + +// PrepareByResultTablePlugin 根据字段提取方法[json|regexp|delimiter],解析上报的日志数据内容 func PrepareByResultTablePlugin(table *config.MetaResultTableConfig) etl.ContainerSchemaBuilderPlugin { return func(builder *etl.ContainerSchemaBuilder) error { fields := make([]etl.Field, 0) - field, err := GetSeparatorFieldByOption(table) + items, err := GetSeparatorFieldByMultiOption(table) if err != nil { return err } - if field != nil { - fields = append(fields, field) + + for i := 0; i < len(items); i++ { + field := items[i] + if field != nil { + fields = append(fields, field) + } } if len(fields) > 0 {