diff --git a/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_mv-1.0.0.sql new file mode 100644 index 0000000000..8361e0c5dd --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_mv-1.0.0.sql @@ -0,0 +1,34 @@ +CREATE MATERIALIZED VIEW {table_name}_mview AS +SELECT + owner_id AS `aws.s3.bucket_owner`, + bucket_name AS `aws.s3.bucket`, + CONCAT(request_time, ' ', request_time_zone) AS `aws.s3.request_time`, + remote_ip AS `aws.s3.remote_ip`, + requester AS `aws.s3.requester`, + request_id AS `aws.s3.request_id`, + operation AS `aws.s3.operation`, + key AS `aws.s3.key`, + request_uri AS `aws.s3.request_uri`, + http_status AS `aws.s3.http_status`, + error_code AS `aws.s3.error_code`, + bytes_sent AS `aws.s3.bytes_sent`, + object_size AS `aws.s3.object_size`, + CAST(total_time AS INTEGER) AS `aws.s3.total_time`, + CAST(turn_around_time AS INTEGER) AS `aws.s3.turn_around_time`, + referrer AS `aws.s3.referrer`, + user_agent AS `aws.s3.user_agent`, + version_id AS `aws.s3.version_id`, + host_id AS `aws.s3.host_id`, + signature_version AS `aws.s3.signature_version`, + cipher_suite AS `aws.s3.cipher_suite`, + authentication_type AS `aws.s3.authentication_type`, + host_header AS `aws.s3.host_header`, + tls_version AS `aws.s3.tls_version` +FROM + {table_name} +WITH ( + auto_refresh = 'true', + checkpoint_location = '{s3_bucket_location}', + watermark_delay = '1 Minute', + extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}' +); \ No newline at end of file diff --git a/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_table-1.0.0.sql new file mode 100644 index 0000000000..9261e65e4e --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_table-1.0.0.sql @@ -0,0 +1,32 @@ +CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( + owner_id STRING, + bucket_name STRING, + request_time STRING, + request_time_zone STRING, + remote_ip STRING, + requester STRING, + request_id STRING, + operation STRING, + key STRING, + request_uri STRING, + http_status STRING, + error_code STRING, + bytes_sent BIGINT, + object_size BIGINT, + total_time STRING, + turn_around_time STRING, + referrer STRING, + user_agent STRING, + version_id STRING, + host_id STRING, + signature_version STRING, + cipher_suite STRING, + authentication_type STRING, + host_header STRING, + tls_version STRING +) +USING csv +OPTIONS ( + sep=' ' +) +LOCATION '{s3_bucket_location}'; \ No newline at end of file diff --git a/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json index ecc190e1d3..1a552edc17 100644 --- a/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json @@ -5,9 +5,23 @@ "description": "Analyze Amazon S3 server logs.", "license": "Apache-2.0", "type": "logs_s3", - "labels": ["Observability", "Logs", "AWS", "Cloud"], + "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_s3/info", + "workflows": [ + { + "name": "queries", + "label": "Queries (recommended)", + "description": "Tables and pre-written queries for quickly getting insights on your data.", + "enabled_by_default": true + }, + { + "name": "dashboards", + "label": "Dashboards & Visualizations", + "description": "Dashboards and indices that enable you to easily visualize important metrics.", + "enabled_by_default": false + } + ], "statics": { "logo": { "annotation": "S3 Logo", @@ -39,7 +53,21 @@ "name": "aws_s3", "version": "1.0.0", "extension": "ndjson", - "type": "savedObjectBundle" + "type": "savedObjectBundle", + "workflows": ["dashboards"] + }, + { + "name": "create_table", + "version": "1.0.0", + "extension": "sql", + "type": "query" + }, + { + "name": "create_mv", + "version": "1.0.0", + "extension": "sql", + "type": "query", + "workflows": ["dashboards"] } ], "sampleData": {