diff --git a/vector-ingestion/example.toml b/vector-ingestion/example.toml new file mode 100644 index 0000000..73a66bd --- /dev/null +++ b/vector-ingestion/example.toml @@ -0,0 +1,44 @@ +[sources.host_metrics] +type = "host_metrics" +collectors = ["cpu", "load", "memory"] +# namespace = "metrics" +scrape_interval_secs = 5 + +[sources.apache_logs] +type = "demo_logs" +count = 100 +format = "apache_common" +interval = 1 +lines = ["line1"] + +[sinks.metrics] +type = "greptimedb_metrics" +inputs = ["host_metrics"] +endpoint = "${GT_HOST}:${GT_GRPC_PORT:-5001}" +dbname = "${GT_DB_NAME}" +username = "${GT_USERNAME}" +password = "${GT_PASSWORD}" +grpc_compression = "gzip" +###tls = {} + +[sinks.logs] +type = "greptimedb_logs" +inputs = ["apache_logs"] +compression = "gzip" +dbname = "${GT_DB_NAME}" +endpoint = "${GT_SCHEMA-:https}://${GT_HOST}:${GT_HTTP_PORT:-80}" +username = "${GT_USERNAME}" +password = "${GT_PASSWORD}" +pipeline_name = "greptime_identity" +table = "demo_logs" + +[sinks.structured_logs] +type = "greptimedb_logs" +inputs = ["apache_logs"] +compression = "gzip" +dbname = "${GT_DB_NAME}" +endpoint = "${GT_SCHEMA-:https}://${GT_HOST}:${GT_HTTP_PORT:-80}" +username = "${GT_USERNAME}" +password = "${GT_PASSWORD}" +pipeline_name = "apache_common_pipeline" +table = "demo_structured_logs" \ No newline at end of file diff --git a/vector-ingestion/pipeline.yaml b/vector-ingestion/pipeline.yaml new file mode 100644 index 0000000..1dc921d --- /dev/null +++ b/vector-ingestion/pipeline.yaml @@ -0,0 +1,40 @@ +#145.251.205.39 - devankoshal [29/Oct/2024:17:51:29 +0800] "DELETE /booper/bopper/mooper/mopper HTTP/1.1" 304 43420 +processors: + - dissect: + fields: + - message + patterns: + - '%{ip_address} - %{user} [%{request_time}] "%{http_method} %{request_path} HTTP/%{http_version}" %{status_code} %{response_size}' + ignore_missing: true + - date: + fields: + - request_time + formats: + # 29/Oct/2024:17:51:29 +0800 + - "%d/%b/%Y:%H:%M:%S %z" + +transform: + - fields: + - ip_address + - http_method + - http_version + type: string + index: tag + - fields: + - status_code + type: int32 + index: tag + - fields: + - request_path + type: string + index: fulltext + - fields: + - user + type: string + - fields: + - response_size + type: int32 + - fields: + - request_time + type: time + index: timestamp \ No newline at end of file diff --git a/vector-ingestion/start.env b/vector-ingestion/start.env new file mode 100644 index 0000000..ba702ef --- /dev/null +++ b/vector-ingestion/start.env @@ -0,0 +1,10 @@ +GT_SCHEMA=http +GT_HOST=localhost +GT_DB_NAME=public +GT_USERNAME= +GT_PASSWORD= +GT_GTPC_TLS=false +GT_GRPC_PORT=4001 +GT_HTTP_PORT=4000 + + diff --git a/vector-ingestion/start.sh b/vector-ingestion/start.sh new file mode 100644 index 0000000..48e688e --- /dev/null +++ b/vector-ingestion/start.sh @@ -0,0 +1,70 @@ +[ ! -f start.env ] || export $(grep -v '^#' start.env | xargs) + +GT_SCHEMA=${GT_SCHEMA-:https} + +if [ -z "$GT_SCHEMA" ]; then + echo "GT_SCHEMA is not set" + exit 1 +fi + +if [ -z "$GT_HOST" ]; then + echo "GT_HOST is not set" + exit 1 +fi + +if [ -z "$GT_DB_NAME" ]; then + echo "GT_DB_NAME is not set" + exit 1 +fi + +function create_pipeline { +## if GT_USERNAME and GT_PASSWORD are not set, then disable basic auth + +if [ -z "$GT_USERNAME" ] || [ -z "$GT_PASSWORD" ]; +then + echo "GT_USERNAME or GT_PASSWORD is not set, disabling basic auth" + curl -X "POST" "$GT_SCHEMA://$GT_HOST:$GT_HTTP_PORT/v1/events/pipelines/apache_common_pipeline?db=$GT_DB_NAME" \ + -F "file=@pipeline.yaml" +else + echo "GT_USERNAME and GT_PASSWORD are set, enabling basic" + curl -X "POST" "$GT_SCHEMA://$GT_HOST:$GT_HTTP_PORT/v1/events/pipelines/apache_common_pipeline?db=$GT_DB_NAME" \ + -u "$GT_USERNAME:$GT_PASSWORD" \ + -F "file=@pipeline.yaml" +fi +} + +function run_demo { + +## if GT_GRPC_TLS is true set example.toml to use tls +if [ "$GT_GRPC_TLS" = "true" ]; +then + echo "GT_GRPC_TLS is true, setting example.toml grpc to use tls" + sed -i 's/#tls = {}/tls = {}/g' example.toml +else + echo "GT_GRPC_TLS is not set or false, setting example.toml grpc to not use tls" + sed -i 's/tls = {}/#tls = {}/g' example.toml +fi + +podman run \ + --env-file start.env \ + --rm \ + -v $PWD/example.toml:/etc/vector/vector.toml:ro \ + --name vector \ + --network host \ + timberio/vector:0.42.0-debian --config-toml /etc/vector/vector.toml + +} + +## two subcommands are supported, create_pipeline and run_demo +case "$1" in + create_pipeline) + create_pipeline + ;; + run_demo) + run_demo + ;; + *) + echo "Usage: $0 {create_pipeline|run_demo}" + exit 1 + ;; +esac \ No newline at end of file