update default config values to better align with production workloads (

#4340)

* update default config values to better align with production workloads

* Update CHANGELOG.md and config docs

CHANGELOG.md

@@ -1,5 +1,6 @@
 ## main / unreleased
 * [FEATURE] tempo-cli: support dropping multiple traces in a single operation [#4266](https://github.com/grafana/tempo/pull/4266) (@ndk)
+* [CHANGE] update default config values to better align with production workloads [#4340](https://github.com/grafana/tempo/pull/4340) (@electron0zero)
 * [CHANGE] fix deprecation warning by switching to DoBatchWithOptions [#4343](https://github.com/grafana/tempo/pull/4343) (@dastrobu)
 * [CHANGE] **BREAKING CHANGE** The Tempo serverless is now deprecated and will be removed in an upcoming release [#4017](https://github.com/grafana/tempo/pull/4017/) @electron0zero
 * [CHANGE] **BREAKING CHANGE** Change the AWS Lambda serverless build tooling output from "main" to "bootstrap". Refer to https://aws.amazon.com/blogs/compute/migrating-aws-lambda-functions-from-the-go1-x-runtime-to-the-custom-runtime-on-amazon-linux-2/ for migration steps [#3852](https://github.com/grafana/tempo/pull/3852) (@zatlodan)

docs/sources/tempo/configuration/_index.md

@@ -639,7 +639,7 @@ query_frontend:
         [throughput_bytes_slo: <float> | default = 0 ]
 
         # The number of shards to break ingester queries into.
-        [ingester_shards]: <int> | default = 1]
+        [ingester_shards]: <int> | default = 3]
 
         # SLO configuration for Metadata (tags and tag values) endpoints.
         metadata_slo:
@@ -1571,7 +1571,7 @@ overrides:
       # tags with high cardinality or large values such as HTTP URLs or SQL queries.
       # This override limit is used by the ingester and the querier.
       # A value of 0 disables the limit.
-      [max_bytes_per_tag_values_query: <int> | default = 5000000 (5MB) ]
+      [max_bytes_per_tag_values_query: <int> | default = 1000000 (1MB) ]
 
       # Maximum number of blocks to be inspected for a tag values query. Tag-values
       # query is used mainly to populate the autocomplete dropdown. This limit

docs/sources/tempo/configuration/manifest.md

@@ -18,7 +18,7 @@ go run ./cmd/tempo --storage.trace.backend=local --storage.trace.local.path=/var
 ## Complete configuration
 
 {{< admonition type="note" >}}
-This manifest was generated on 2024-10-21.
+This manifest was generated on 2024-11-18.
 {{% /admonition %}}
 
 ```yaml
@@ -316,7 +316,7 @@ query_frontend:
         max_duration: 168h0m0s
         query_backend_after: 15m0s
         query_ingesters_until: 30m0s
-        ingester_shards: 1
+        ingester_shards: 3
     trace_by_id:
         query_shards: 50
     metrics:
@@ -328,6 +328,11 @@ query_frontend:
         max_exemplars: 100
     multi_tenant_queries_enabled: true
     response_consumers: 10
+    weights:
+        request_with_weights: true
+        retry_with_weights: true
+        max_traceql_conditions: 4
+        max_regex_conditions: 1
 compactor:
     ring:
         kvstore:
@@ -713,7 +718,7 @@ overrides:
             burst_size_bytes: 20000000
             max_traces_per_user: 10000
         read:
-            max_bytes_per_tag_values_query: 5000000
+            max_bytes_per_tag_values_query: 1000000
         metrics_generator:
             generate_native_histograms: classic
             ingestion_time_range_slack: 0s

modules/frontend/config.go

@@ -78,7 +78,7 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(string, *flag.FlagSet) {
 			MaxDuration:           168 * time.Hour, // 1 week
 			ConcurrentRequests:    defaultConcurrentRequests,
 			TargetBytesPerRequest: defaultTargetBytesPerRequest,
-			IngesterShards:        1,
+			IngesterShards:        3,
 		},
 		SLO: slo,
 	}

modules/overrides/config.go

@@ -271,7 +271,7 @@ func (c *Config) RegisterFlagsAndApplyDefaults(f *flag.FlagSet) {
 	f.IntVar(&c.Defaults.Global.MaxBytesPerTrace, "ingester.max-bytes-per-trace", 50e5, "Maximum size of a trace in bytes.  0 to disable.")
 
 	// Querier limits
-	f.IntVar(&c.Defaults.Read.MaxBytesPerTagValuesQuery, "querier.max-bytes-per-tag-values-query", 50e5, "Maximum size of response for a tag-values query. Used mainly to limit large the number of values associated with a particular tag")
+	f.IntVar(&c.Defaults.Read.MaxBytesPerTagValuesQuery, "querier.max-bytes-per-tag-values-query", 10e5, "Maximum size of response for a tag-values query. Used mainly to limit large the number of values associated with a particular tag")
 	f.IntVar(&c.Defaults.Read.MaxBlocksPerTagValuesQuery, "querier.max-blocks-per-tag-values-query", 0, "Maximum number of blocks to query for a tag-values query. 0 to disable.")
 
 	f.StringVar(&c.PerTenantOverrideConfig, "config.per-user-override-config", "", "File name of per-user Overrides.")