diff --git a/package.json b/package.json index d8a97951b897d..0a7c0d6936d0a 100644 --- a/package.json +++ b/package.json @@ -1292,6 +1292,7 @@ "xstate": "^4.38.2", "xstate5": "npm:xstate@^5.18.1", "xterm": "^5.1.0", + "yaml": "^2.5.1", "yauzl": "^2.10.0", "yazl": "^2.5.1", "zod": "^3.22.3" diff --git a/src/dev/build/tasks/clean_tasks.ts b/src/dev/build/tasks/clean_tasks.ts index ad8eeaadaad60..19af3954fde45 100644 --- a/src/dev/build/tasks/clean_tasks.ts +++ b/src/dev/build/tasks/clean_tasks.ts @@ -56,6 +56,7 @@ export const CleanExtraFilesFromModules: Task = { // docs '**/doc', + '!**/yaml/dist/**/doc', // yaml package store code under doc https://github.com/eemeli/yaml/issues/384 '**/docs', '**/README', '**/CONTRIBUTING.md', diff --git a/src/dev/yarn_deduplicate/index.ts b/src/dev/yarn_deduplicate/index.ts index 3f942252e39ab..f95ee583fba01 100644 --- a/src/dev/yarn_deduplicate/index.ts +++ b/src/dev/yarn_deduplicate/index.ts @@ -17,7 +17,7 @@ const yarnLock = readFileSync(yarnLockFile, 'utf-8'); const output = fixDuplicates(yarnLock, { useMostCommon: false, excludeScopes: ['@types'], - excludePackages: ['axe-core', '@babel/types', 'csstype'], + excludePackages: ['axe-core', '@babel/types', 'csstype', 'yaml'], }); writeFileSync(yarnLockFile, output); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/logs_2_3_0.ts b/x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/logs_2_3_0.ts new file mode 100644 index 0000000000000..abbf60400271e --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/logs_2_3_0.ts @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const LOGS_2_3_0_PACKAGE_INFO = { + name: 'log', + version: '2.3.0', + title: 'Custom Logs', + owner: { github: 'elastic/elastic-agent-data-plane' }, + type: 'input', + categories: ['custom', 'custom_logs'], + conditions: { 'kibana.version': '^8.8.0' }, + icons: [{ src: '/img/icon.svg', type: 'image/svg+xml' }], + policy_templates: [ + { + name: 'logs', + title: 'Custom log file', + description: 'Collect your custom log files.', + multiple: true, + input: 'logfile', + type: 'logs', + template_path: 'input.yml.hbs', + vars: [ + { + name: 'paths', + required: true, + title: 'Log file path', + description: 'Path to log files to be collected', + type: 'text', + multi: true, + }, + { + name: 'exclude_files', + required: false, + show_user: false, + title: 'Exclude files', + description: 'Patterns to be ignored', + type: 'text', + multi: true, + }, + { + name: 'ignore_older', + type: 'text', + title: 'Ignore events older than', + default: '72h', + required: false, + show_user: false, + description: + 'If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".', + }, + { + name: 'data_stream.dataset', + required: true, + title: 'Dataset name', + description: + "Set the name for your dataset. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html).\n", + type: 'text', + }, + { + name: 'tags', + type: 'text', + title: 'Tags', + description: 'Tags to include in the published event', + multi: true, + show_user: false, + }, + { + name: 'processors', + type: 'yaml', + title: 'Processors', + multi: false, + required: false, + show_user: false, + description: + 'Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.', + }, + { + name: 'custom', + title: 'Custom configurations', + description: + 'Here YAML configuration options can be used to be added to your configuration. Be careful using this as it might break your configuration file.\n', + type: 'yaml', + default: '', + }, + ], + }, + ], + elasticsearch: {}, + description: 'Collect custom logs with Elastic Agent.', + format_version: '2.6.0', + readme: '/package/log/2.3.0/docs/README.md', + release: 'ga', + latestVersion: '2.3.2', + assets: {}, + licensePath: '/package/log/2.3.0/LICENSE.txt', + keepPoliciesUpToDate: false, + status: 'not_installed', +}; + +export const LOGS_2_3_0_ASSETS_MAP = new Map([ + [ + 'log-2.3.0/agent/input/input.yml.hbs', + Buffer.from(`paths: +{{#each paths}} + - {{this}} +{{/each}} + +{{#if exclude_files}} +exclude_files: +{{#each exclude_files}} + - {{this}} +{{/each}} +{{/if}} +{{#if ignore_older}} +ignore_older: {{ignore_older}} +{{/if}} +data_stream: + dataset: {{data_stream.dataset}} +{{#if processors.length}} +processors: +{{processors}} +{{/if}} +{{#if tags.length}} +tags: +{{#each tags as |tag i|}} +- {{tag}} +{{/each}} +{{/if}} + +{{custom}} +`), + ], +]); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/redis_1_18_0_package_info.json b/x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/redis_1_18_0_package_info.json new file mode 100644 index 0000000000000..57c9b0c68fac9 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/redis_1_18_0_package_info.json @@ -0,0 +1,245 @@ +{ + "name": "redis", + "title": "Redis", + "version": "1.18.0", + "release": "ga", + "description": "Collect logs and metrics from Redis servers with Elastic Agent.", + "type": "integration", + "download": "/epr/redis/redis-1.18.0.zip", + "path": "/package/redis/1.18.0", + "icons": [ + { + "src": "/img/logo_redis.svg", + "path": "/package/redis/1.18.0/img/logo_redis.svg", + "title": "logo redis", + "size": "32x32", + "type": "image/svg+xml" + } + ], + "conditions": { + "kibana": { + "version": "^8.13.0" + }, + "elastic": { + "subscription": "basic" + } + }, + "owner": { + "type": "elastic", + "github": "elastic/obs-infraobs-integrations" + }, + "categories": ["datastore", "observability"], + "signature_path": "/epr/redis/redis-1.18.0.zip.sig", + "format_version": "3.0.2", + "readme": "/package/redis/1.18.0/docs/README.md", + "license": "basic", + "screenshots": [ + { + "src": "/img/kibana-redis.png", + "path": "/package/redis/1.18.0/img/kibana-redis.png", + "title": "kibana redis", + "size": "1124x1079", + "type": "image/png" + }, + { + "src": "/img/metricbeat_redis_key_dashboard.png", + "path": "/package/redis/1.18.0/img/metricbeat_redis_key_dashboard.png", + "title": "metricbeat redis key dashboard", + "size": "1855x949", + "type": "image/png" + }, + { + "src": "/img/metricbeat_redis_overview_dashboard.png", + "path": "/package/redis/1.18.0/img/metricbeat_redis_overview_dashboard.png", + "title": "metricbeat redis overview dashboard", + "size": "1855x949", + "type": "image/png" + } + ], + "assets": [ + "/package/redis/1.18.0/LICENSE.txt", + "/package/redis/1.18.0/changelog.yml", + "/package/redis/1.18.0/manifest.yml", + "/package/redis/1.18.0/docs/README.md", + "/package/redis/1.18.0/img/kibana-redis.png", + "/package/redis/1.18.0/img/logo_redis.svg", + "/package/redis/1.18.0/img/metricbeat_redis_key_dashboard.png", + "/package/redis/1.18.0/img/metricbeat_redis_overview_dashboard.png", + "/package/redis/1.18.0/data_stream/info/manifest.yml", + "/package/redis/1.18.0/data_stream/info/sample_event.json", + "/package/redis/1.18.0/data_stream/key/manifest.yml", + "/package/redis/1.18.0/data_stream/key/sample_event.json", + "/package/redis/1.18.0/data_stream/keyspace/manifest.yml", + "/package/redis/1.18.0/data_stream/keyspace/sample_event.json", + "/package/redis/1.18.0/data_stream/log/manifest.yml", + "/package/redis/1.18.0/data_stream/slowlog/manifest.yml", + "/package/redis/1.18.0/kibana/dashboard/redis-28969190-0511-11e9-9c60-d582a238e2c5.json", + "/package/redis/1.18.0/kibana/dashboard/redis-7fea2930-478e-11e7-b1f0-cb29bac6bf8b.json", + "/package/redis/1.18.0/kibana/dashboard/redis-AV4YjZ5pux-M-tCAunxK.json", + "/package/redis/1.18.0/data_stream/info/fields/agent.yml", + "/package/redis/1.18.0/data_stream/info/fields/base-fields.yml", + "/package/redis/1.18.0/data_stream/info/fields/ecs.yml", + "/package/redis/1.18.0/data_stream/info/fields/fields.yml", + "/package/redis/1.18.0/data_stream/key/fields/agent.yml", + "/package/redis/1.18.0/data_stream/key/fields/base-fields.yml", + "/package/redis/1.18.0/data_stream/key/fields/ecs.yml", + "/package/redis/1.18.0/data_stream/key/fields/fields.yml", + "/package/redis/1.18.0/data_stream/keyspace/fields/agent.yml", + "/package/redis/1.18.0/data_stream/keyspace/fields/base-fields.yml", + "/package/redis/1.18.0/data_stream/keyspace/fields/ecs.yml", + "/package/redis/1.18.0/data_stream/keyspace/fields/fields.yml", + "/package/redis/1.18.0/data_stream/log/fields/agent.yml", + "/package/redis/1.18.0/data_stream/log/fields/base-fields.yml", + "/package/redis/1.18.0/data_stream/log/fields/fields.yml", + "/package/redis/1.18.0/data_stream/slowlog/fields/agent.yml", + "/package/redis/1.18.0/data_stream/slowlog/fields/base-fields.yml", + "/package/redis/1.18.0/data_stream/slowlog/fields/fields.yml", + "/package/redis/1.18.0/data_stream/info/agent/stream/stream.yml.hbs", + "/package/redis/1.18.0/data_stream/key/agent/stream/stream.yml.hbs", + "/package/redis/1.18.0/data_stream/keyspace/agent/stream/stream.yml.hbs", + "/package/redis/1.18.0/data_stream/log/agent/stream/stream.yml.hbs", + "/package/redis/1.18.0/data_stream/log/elasticsearch/ingest_pipeline/default.yml", + "/package/redis/1.18.0/data_stream/slowlog/agent/stream/stream.yml.hbs", + "/package/redis/1.18.0/data_stream/slowlog/elasticsearch/ingest_pipeline/default.json" + ], + "policy_templates": [ + { + "name": "redis", + "title": "Redis logs and metrics", + "description": "Collect logs and metrics from Redis instances", + "inputs": [ + { + "type": "logfile", + "title": "Collect Redis application logs", + "description": "Collecting application logs from Redis instances" + }, + { + "type": "redis", + "title": "Collect Redis slow logs", + "description": "Collecting slow logs from Redis instances" + }, + { + "type": "redis/metrics", + "vars": [ + { + "name": "hosts", + "type": "text", + "title": "Hosts", + "multi": true, + "required": true, + "show_user": true, + "default": ["127.0.0.1:6379"] + }, + { + "name": "idle_timeout", + "type": "text", + "title": "Idle Timeout", + "multi": false, + "required": false, + "show_user": false, + "default": "20s" + }, + { + "name": "maxconn", + "type": "integer", + "title": "Maxconn", + "multi": false, + "required": false, + "show_user": false, + "default": 10 + }, + { + "name": "network", + "type": "text", + "title": "Network", + "multi": false, + "required": false, + "show_user": false, + "default": "tcp" + }, + { + "name": "username", + "type": "text", + "title": "Username", + "multi": false, + "required": false, + "show_user": false, + "default": "" + }, + { + "name": "password", + "type": "password", + "title": "Password", + "multi": false, + "required": false, + "show_user": false, + "default": "" + }, + { + "name": "ssl", + "type": "yaml", + "title": "SSL Configuration", + "description": "i.e. certificate_authorities, supported_protocols, verification_mode etc.", + "multi": false, + "required": false, + "show_user": false, + "default": "# ssl.certificate_authorities: |\n# -----BEGIN CERTIFICATE-----\n# MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT\n# AlVTMQwwCgYDVQQKEwNJQk0xFjAUBgNVBAsTDURlZmF1bHROb2RlMDExFjAUBgNV\n# BAsTDURlZmF1bHRDZWxsMDExGTAXBgNVBAsTEFJvb3QgQ2VydGlmaWNhdGUxEjAQ\n# BgNVBAMTCWxvY2FsaG9zdDAeFw0yMTEyMTQyMjA3MTZaFw0yMjEyMTQyMjA3MTZa\n# MF8xCzAJBgNVBAYTAlVTMQwwCgYDVQQKEwNJQk0xFjAUBgNVBAsTDURlZmF1bHRO\n# b2RlMDExFjAUBgNVBAsTDURlZmF1bHRDZWxsMDExEjAQBgNVBAMTCWxvY2FsaG9z\n# dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMv5HCsJZIpI5zCy+jXV\n# z6lmzNc9UcVSEEHn86h6zT6pxuY90TYeAhlZ9hZ+SCKn4OQ4GoDRZhLPTkYDt+wW\n# CV3NTIy9uCGUSJ6xjCKoxClJmgSQdg5m4HzwfY4ofoEZ5iZQ0Zmt62jGRWc0zuxj\n# hegnM+eO2reBJYu6Ypa9RPJdYJsmn1RNnC74IDY8Y95qn+WZj//UALCpYfX41hko\n# i7TWD9GKQO8SBmAxhjCDifOxVBokoxYrNdzESl0LXvnzEadeZTd9BfUtTaBHhx6t\n# njqqCPrbTY+3jAbZFd4RiERPnhLVKMytw5ot506BhPrUtpr2lusbN5svNXjuLeea\n# MMUCAwEAAaOBoDCBnTATBgNVHSMEDDAKgAhOatpLwvJFqjAdBgNVHSUEFjAUBggr\n# BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4E+UHJvZmlsZVVVSUQ6QXBwU3J2\n# MDEtQkFTRS05MDkzMzJjMC1iNmFiLTQ2OTMtYWI5NC01Mjc1ZDI1MmFmNDiCCWxv\n# Y2FsaG9zdDARBgNVHQ4ECgQITzqhA5sO8O4wDQYJKoZIhvcNAQELBQADggEBAKR0\n# gY/BM69S6BDyWp5dxcpmZ9FS783FBbdUXjVtTkQno+oYURDrhCdsfTLYtqUlP4J4\n# CHoskP+MwJjRIoKhPVQMv14Q4VC2J9coYXnePhFjE+6MaZbTjq9WaekGrpKkMaQA\n# iQt5b67jo7y63CZKIo9yBvs7sxODQzDn3wZwyux2vPegXSaTHR/rop/s/mPk3YTS\n# hQprs/IVtPoWU4/TsDN3gIlrAYGbcs29CAt5q9MfzkMmKsuDkTZD0ry42VjxjAmk\n# xw23l/k8RoD1wRWaDVbgpjwSzt+kl+vJE/ip2w3h69eEZ9wbo6scRO5lCO2JM4Pr\n# 7RhLQyWn2u00L7/9Omw=\n# -----END CERTIFICATE-----\n" + } + ], + "title": "Collect Redis metrics", + "description": "Collecting info, key and keyspace metrics from Redis instances" + } + ], + "multiple": true + } + ], + "data_streams": [ + { + "type": "metrics", + "dataset": "redis.key", + "title": "Redis key metrics", + "release": "ga", + "streams": [ + { + "input": "redis/metrics", + "vars": [ + { + "name": "key.patterns", + "type": "yaml", + "title": "Key Patterns", + "multi": false, + "required": true, + "show_user": true, + "default": "- limit: 20\n pattern: '*'\n" + }, + { + "name": "period", + "type": "text", + "title": "Period", + "multi": false, + "required": true, + "show_user": true, + "default": "10s" + }, + { + "name": "processors", + "type": "yaml", + "title": "Processors", + "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/fleet/current/elastic-agent-processor-configuration.html) for details. \n", + "multi": false, + "required": false, + "show_user": false + } + ], + "template_path": "stream.yml.hbs", + "title": "Redis key metrics", + "description": "Collect Redis key metrics", + "enabled": true + } + ], + "package": "redis", + "elasticsearch": {}, + "path": "key" + } + ] +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/redis_1_18_0_streams_template.ts b/x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/redis_1_18_0_streams_template.ts new file mode 100644 index 0000000000000..5ff46f358bbe7 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/__fixtures__/redis_1_18_0_streams_template.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const REDIS_ASSETS_MAP = new Map([ + [ + 'redis-1.18.0/data_stream/slowlog/agent/stream/stream.yml.hbs', + Buffer.from(`hosts: +{{#each hosts as |host i|}} + - {{host}} +{{/each}} +password: {{password}} +{{#if processors}} +processors: +{{processors}} +{{/if}} +`), + ], + [ + 'redis-1.18.0/data_stream/log/agent/stream/stream.yml.hbs', + Buffer.from(`paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +exclude_files: [".gz$"] +exclude_lines: ["^\\s+[\\-\`('.|_]"] # drop asciiart lines\n +{{#if processors}} +processors: +{{processors}} +{{/if}} +`), + ], + [ + 'redis-1.18.0/data_stream/key/agent/stream/stream.yml.hbs', + Buffer.from(`metricsets: ["key"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +{{#if idle_timeout}} +idle_timeout: {{idle_timeout}} +{{/if}} +{{#if key.patterns}} +key.patterns: {{key.patterns}} +{{/if}} +{{#if maxconn}} +maxconn: {{maxconn}} +{{/if}} +{{#if network}} +network: {{network}} +{{/if}} +{{#if username}} +username: {{username}} +{{/if}} +{{#if password}} +password: {{password}} +{{/if}} +{{#if ssl}} +{{ssl}} +{{/if}} +period: {{period}} +{{#if processors}} +processors: +{{processors}} +{{/if}} +`), + ], +]); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/__snapshots__/get_templates_inputs.test.ts.snap b/x-pack/plugins/fleet/server/services/epm/packages/__snapshots__/get_templates_inputs.test.ts.snap new file mode 100644 index 0000000000000..b3a428c0e5a55 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/__snapshots__/get_templates_inputs.test.ts.snap @@ -0,0 +1,56 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Fleet - getTemplateInputs should work for input package 1`] = ` +"inputs: + # Custom log file: Collect your custom log files. + - id: logs-logfile + type: logfile + streams: + # Custom log file: Custom log file + - id: logfile-log.logs + data_stream: + dataset: + # Dataset name: Set the name for your dataset. Changing the dataset will send the data to a different index. You can't use \`-\` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html). + + paths: + - # Log file path: Path to log files to be collected + exclude_files: + - # Exclude files: Patterns to be ignored + ignore_older: 72h + tags: + - # Tags: Tags to include in the published event +" +`; + +exports[`Fleet - getTemplateInputs should work for integration package 1`] = ` +"inputs: + # Collect Redis application logs: Collecting application logs from Redis instances + - id: redis-logfile + type: logfile + # Collect Redis slow logs: Collecting slow logs from Redis instances + - id: redis-redis + type: redis + # Collect Redis metrics: Collecting info, key and keyspace metrics from Redis instances + - id: redis-redis/metrics + type: redis/metrics + streams: + # Redis key metrics: Collect Redis key metrics + - id: redis/metrics-redis.key + data_stream: + dataset: redis.key + type: metrics + metricsets: + - key + hosts: + - 127.0.0.1:6379 + idle_timeout: 20s + key.patterns: + - limit: 20 + pattern: '*' + maxconn: 10 + network: tcp + username: # Username + password: # Password + period: 10s +" +`; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get_template_inputs.ts b/x-pack/plugins/fleet/server/services/epm/packages/get_template_inputs.ts index 640fc3877eabf..8c63f4b093dd0 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get_template_inputs.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get_template_inputs.ts @@ -8,8 +8,14 @@ import type { SavedObjectsClientContract } from '@kbn/core/server'; import { merge } from 'lodash'; import { dump } from 'js-yaml'; +import yamlDoc from 'yaml'; -import { packageToPackagePolicy } from '../../../../common/services/package_to_package_policy'; +import { getNormalizedInputs, isIntegrationPolicyTemplate } from '../../../../common/services'; + +import { + getStreamsForInputType, + packageToPackagePolicy, +} from '../../../../common/services/package_to_package_policy'; import { getInputsWithStreamIds, _compilePackagePolicyInputs } from '../../package_policy'; import { appContextService } from '../../app_context'; import type { @@ -17,6 +23,10 @@ import type { NewPackagePolicy, PackagePolicyInput, TemplateAgentPolicyInput, + RegistryVarsEntry, + RegistryStream, + PackagePolicyConfigRecordEntry, + RegistryInput, } from '../../../../common/types'; import { _sortYamlKeys } from '../../../../common/services/full_agent_policy_to_yaml'; @@ -27,6 +37,18 @@ import { getPackageAssetsMap } from './get'; type Format = 'yml' | 'json'; +type PackageWithInputAndStreamIndexed = Record< + string, + RegistryInput & { + streams: Record< + string, + RegistryStream & { + data_stream: { type: string; dataset: string }; + } + >; + } +>; + // Function based off storedPackagePolicyToAgentInputs, it only creates the `streams` section instead of the FullAgentPolicyInput export const templatePackagePolicyToFullInputStreams = ( packagePolicyInputs: PackagePolicyInput[] @@ -38,7 +60,7 @@ export const templatePackagePolicyToFullInputStreams = ( packagePolicyInputs.forEach((input) => { const fullInputStream = { // @ts-ignore-next-line the following id is actually one level above the one in fullInputStream, but the linter thinks it gets overwritten - id: input.policy_template ? `${input.type}-${input.policy_template}` : `${input.type}`, + id: input.policy_template ? `${input.policy_template}-${input.type}` : `${input.type}`, type: input.type, ...getFullInputStreams(input, true), }; @@ -81,22 +103,53 @@ export async function getTemplateInputs( prerelease?: boolean, ignoreUnverified?: boolean ) { - const packageInfoMap = new Map(); - let packageInfo: PackageInfo; - - if (packageInfoMap.has(pkgName)) { - packageInfo = packageInfoMap.get(pkgName)!; - } else { - packageInfo = await getPackageInfo({ - savedObjectsClient: soClient, - pkgName, - pkgVersion, - prerelease, - ignoreUnverified, - }); - } + const packageInfo = await getPackageInfo({ + savedObjectsClient: soClient, + pkgName, + pkgVersion, + prerelease, + ignoreUnverified, + }); + const emptyPackagePolicy = packageToPackagePolicy(packageInfo, ''); + const inputsWithStreamIds = getInputsWithStreamIds(emptyPackagePolicy, undefined, true); + + const indexedInputsAndStreams = buildIndexedPackage(packageInfo); + + if (format === 'yml') { + // Add a placeholder to all variables without default value + for (const inputWithStreamIds of inputsWithStreamIds) { + const inputId = inputWithStreamIds.policy_template + ? `${inputWithStreamIds.policy_template}-${inputWithStreamIds.type}` + : inputWithStreamIds.type; + + const packageInput = indexedInputsAndStreams[inputId]; + if (!packageInput) { + continue; + } + + for (const [inputVarKey, inputVarValue] of Object.entries(inputWithStreamIds.vars ?? {})) { + const varDef = packageInput.vars?.find((_varDef) => _varDef.name === inputVarKey); + if (varDef) { + addPlaceholderIfNeeded(varDef, inputVarValue); + } + } + for (const stream of inputWithStreamIds.streams) { + const packageStream = packageInput.streams[stream.id]; + if (!packageStream) { + continue; + } + for (const [streamVarKey, streamVarValue] of Object.entries(stream.vars ?? {})) { + const varDef = packageStream.vars?.find((_varDef) => _varDef.name === streamVarKey); + if (varDef) { + addPlaceholderIfNeeded(varDef, streamVarValue); + } + } + } + } + } + const assetsMap = await getPackageAssetsMap({ logger: appContextService.getLogger(), packageInfo, @@ -128,7 +181,146 @@ export async function getTemplateInputs( sortKeys: _sortYamlKeys, } ); - return yaml; + return addCommentsToYaml(yaml, buildIndexedPackage(packageInfo)); } + return { inputs: [] }; } + +function getPlaceholder(varDef: RegistryVarsEntry) { + return `<${varDef.name.toUpperCase()}>`; +} + +function addPlaceholderIfNeeded( + varDef: RegistryVarsEntry, + varValue: PackagePolicyConfigRecordEntry +) { + const placeHolder = `<${varDef.name.toUpperCase()}>`; + if (varDef && !varValue.value && varDef.type !== 'yaml') { + varValue.value = placeHolder; + } else if (varDef && varValue.value && varValue.value.length === 0 && varDef.type === 'text') { + varValue.value = [placeHolder]; + } +} + +function buildIndexedPackage(packageInfo: PackageInfo): PackageWithInputAndStreamIndexed { + return ( + packageInfo.policy_templates?.reduce( + (inputsAcc, policyTemplate) => { + const inputs = getNormalizedInputs(policyTemplate); + + inputs.forEach((packageInput) => { + const inputId = `${policyTemplate.name}-${packageInput.type}`; + + const streams = getStreamsForInputType( + packageInput.type, + packageInfo, + isIntegrationPolicyTemplate(policyTemplate) && policyTemplate.data_streams + ? policyTemplate.data_streams + : [] + ).reduce< + Record< + string, + RegistryStream & { + data_stream: { type: string; dataset: string }; + } + > + >((acc, stream) => { + const streamId = `${packageInput.type}-${stream.data_stream.dataset}`; + acc[streamId] = { + ...stream, + }; + return acc; + }, {}); + + inputsAcc[inputId] = { + ...packageInput, + streams, + }; + }); + return inputsAcc; + }, + {} + ) ?? {} + ); +} + +function addCommentsToYaml( + yaml: string, + packageIndexInputAndStreams: PackageWithInputAndStreamIndexed +) { + const doc = yamlDoc.parseDocument(yaml); + // Add input and streams comments + const yamlInputs = doc.get('inputs'); + if (yamlDoc.isCollection(yamlInputs)) { + yamlInputs.items.forEach((inputItem) => { + if (!yamlDoc.isMap(inputItem)) { + return; + } + const inputIdNode = inputItem.get('id', true); + if (!yamlDoc.isScalar(inputIdNode)) { + return; + } + const inputId = inputIdNode.value as string; + const pkgInput = packageIndexInputAndStreams[inputId]; + if (pkgInput) { + inputItem.commentBefore = ` ${pkgInput.title}${ + pkgInput.description ? `: ${pkgInput.description}` : '' + }`; + + yamlDoc.visit(inputItem, { + Scalar(key, node) { + if (node.value) { + const val = node.value.toString(); + for (const varDef of pkgInput.vars ?? []) { + const placeholder = getPlaceholder(varDef); + if (val.includes(placeholder)) { + node.comment = ` ${varDef.title}${ + varDef.description ? `: ${varDef.description}` : '' + }`; + } + } + } + }, + }); + + const yamlStreams = inputItem.get('streams'); + if (!yamlDoc.isCollection(yamlStreams)) { + return; + } + yamlStreams.items.forEach((streamItem) => { + if (!yamlDoc.isMap(streamItem)) { + return; + } + const streamIdNode = streamItem.get('id', true); + if (yamlDoc.isScalar(streamIdNode)) { + const streamId = streamIdNode.value as string; + const pkgStream = pkgInput.streams[streamId]; + if (pkgStream) { + streamItem.commentBefore = ` ${pkgStream.title}${ + pkgStream.description ? `: ${pkgStream.description}` : '' + }`; + yamlDoc.visit(streamItem, { + Scalar(key, node) { + if (node.value) { + const val = node.value.toString(); + for (const varDef of pkgStream.vars ?? []) { + const placeholder = getPlaceholder(varDef); + if (val.includes(placeholder)) { + node.comment = ` ${varDef.title}${ + varDef.description ? `: ${varDef.description}` : '' + }`; + } + } + } + }, + }); + } + } + }); + } + }); + } + + return doc.toString(); +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get_templates_inputs.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/get_templates_inputs.test.ts index ce80532b3b623..087002f212852 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get_templates_inputs.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get_templates_inputs.test.ts @@ -5,9 +5,19 @@ * 2.0. */ +import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks'; + +import { createAppContextStartContractMock } from '../../../mocks'; import type { PackagePolicyInput } from '../../../../common/types'; +import { appContextService } from '../..'; + +import { getTemplateInputs, templatePackagePolicyToFullInputStreams } from './get_template_inputs'; +import REDIS_1_18_0_PACKAGE_INFO from './__fixtures__/redis_1_18_0_package_info.json'; +import { getPackageAssetsMap, getPackageInfo } from './get'; +import { REDIS_ASSETS_MAP } from './__fixtures__/redis_1_18_0_streams_template'; +import { LOGS_2_3_0_ASSETS_MAP, LOGS_2_3_0_PACKAGE_INFO } from './__fixtures__/logs_2_3_0'; -import { templatePackagePolicyToFullInputStreams } from './get_template_inputs'; +jest.mock('./get'); const packageInfoCache = new Map(); packageInfoCache.set('mock_package-0.0.0', { @@ -29,6 +39,9 @@ packageInfoCache.set('limited_package-0.0.0', { ], }); +packageInfoCache.set('redis-1.18.0', REDIS_1_18_0_PACKAGE_INFO); +packageInfoCache.set('log-2.3.0', LOGS_2_3_0_PACKAGE_INFO); + describe('Fleet - templatePackagePolicyToFullInputStreams', () => { const mockInput: PackagePolicyInput = { type: 'test-logs', @@ -189,7 +202,7 @@ describe('Fleet - templatePackagePolicyToFullInputStreams', () => { it('returns agent inputs without streams', async () => { expect(await templatePackagePolicyToFullInputStreams([mockInput2])).toEqual([ { - id: 'test-metrics-some-template', + id: 'some-template-test-metrics', type: 'test-metrics', streams: [ { @@ -305,3 +318,43 @@ describe('Fleet - templatePackagePolicyToFullInputStreams', () => { ]); }); }); + +describe('Fleet - getTemplateInputs', () => { + beforeEach(() => { + appContextService.start(createAppContextStartContractMock()); + jest.mocked(getPackageAssetsMap).mockImplementation(async ({ packageInfo }) => { + if (packageInfo.name === 'redis' && packageInfo.version === '1.18.0') { + return REDIS_ASSETS_MAP; + } + + if (packageInfo.name === 'log') { + return LOGS_2_3_0_ASSETS_MAP; + } + + return new Map(); + }); + jest.mocked(getPackageInfo).mockImplementation(async ({ pkgName, pkgVersion }) => { + const pkgInfo = packageInfoCache.get(`${pkgName}-${pkgVersion}`); + if (!pkgInfo) { + throw new Error('package not mocked'); + } + + return pkgInfo; + }); + }); + it('should work for integration package', async () => { + const soMock = savedObjectsClientMock.create(); + soMock.get.mockResolvedValue({ attributes: {} } as any); + const template = await getTemplateInputs(soMock, 'redis', '1.18.0', 'yml'); + + expect(template).toMatchSnapshot(); + }); + + it('should work for input package', async () => { + const soMock = savedObjectsClientMock.create(); + soMock.get.mockResolvedValue({ attributes: {} } as any); + const template = await getTemplateInputs(soMock, 'log', '2.3.0', 'yml'); + + expect(template).toMatchSnapshot(); + }); +}); diff --git a/x-pack/test/fleet_api_integration/apis/epm/get_templates_inputs.ts b/x-pack/test/fleet_api_integration/apis/epm/get_templates_inputs.ts index a1eac19eed8b7..cca480c45f56d 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/get_templates_inputs.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/get_templates_inputs.ts @@ -51,9 +51,11 @@ export default function (providerContext: FtrProviderContext) { await uninstallPackage(testPkgName, testPkgVersion); }); const expectedYml = `inputs: - - id: logfile-apache + # Collect logs from Apache instances: Collecting Apache access and error logs + - id: apache-logfile type: logfile streams: + # Apache access logs: Collect Apache access logs - id: logfile-apache.access data_stream: dataset: apache.access @@ -69,6 +71,7 @@ export default function (providerContext: FtrProviderContext) { target: '' fields: ecs.version: 1.5.0 + # Apache error logs: Collect Apache error logs - id: logfile-apache.error data_stream: dataset: apache.error @@ -84,9 +87,11 @@ export default function (providerContext: FtrProviderContext) { target: '' fields: ecs.version: 1.5.0 - - id: apache/metrics-apache + # Collect metrics from Apache instances: Collecting Apache status metrics + - id: apache-apache/metrics type: apache/metrics streams: + # Apache status metrics: Collect Apache status metrics - id: apache/metrics-apache.status data_stream: dataset: apache.status @@ -100,7 +105,7 @@ export default function (providerContext: FtrProviderContext) { `; const expectedJson = [ { - id: 'logfile-apache', + id: 'apache-logfile', type: 'logfile', streams: [ { @@ -151,7 +156,7 @@ export default function (providerContext: FtrProviderContext) { ], }, { - id: 'apache/metrics-apache', + id: 'apache-apache/metrics', type: 'apache/metrics', streams: [ { diff --git a/yarn.lock b/yarn.lock index 11778ed7abcc9..ec4c8f0e0837f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -32974,6 +32974,11 @@ yaml@^2.0.0, yaml@^2.2.1, yaml@^2.2.2: resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.3.4.tgz#53fc1d514be80aabf386dc6001eb29bf3b7523b2" integrity sha512-8aAvwVUSHpfEqTQ4w/KMlf3HcRdt50E5ODIQJBw1fQ5RL34xabzxtUlzTXVqc4rkZsPbvrXKWnABCD7kWSmocA== +yaml@^2.5.1: + version "2.5.1" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.5.1.tgz#c9772aacf62cb7494a95b0c4f1fb065b563db130" + integrity sha512-bLQOjaX/ADgQ20isPJRvF0iRUHIxVhYvr53Of7wGcWlO2jvtUlH5m87DsmulFVxRpNLOnI4tB6p/oh8D7kpn9Q== + yargs-parser@20.2.4, yargs-parser@^20.2.2, yargs-parser@^20.2.3: version "20.2.4" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.4.tgz#b42890f14566796f85ae8e3a25290d205f154a54"