From ec987ecc4a73d7205db544d98898c025579c4911 Mon Sep 17 00:00:00 2001 From: Chris Hain Date: Tue, 15 Oct 2024 13:53:44 -0700 Subject: [PATCH 01/10] Add batch max size, rename processor, and re-order after interval proc --- config/ast_defaults.yaml | 2 +- services/otel_collector/defaults/bigip-scraper-config.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/config/ast_defaults.yaml b/config/ast_defaults.yaml index c4df056..bc380eb 100644 --- a/config/ast_defaults.yaml +++ b/config/ast_defaults.yaml @@ -53,5 +53,5 @@ pipelines: # detail. metrics/f5-datafabric: #receivers list are generated via the config helper script - processors: [batch, interval/f5-datafabric, attributes/f5-datafabric] + processors: [interval/f5-datafabric, attributes/f5-datafabric, batch/f5-datafabric] exporters: [otlp/f5-datafabric, debug/bigip] \ No newline at end of file diff --git a/services/otel_collector/defaults/bigip-scraper-config.yaml b/services/otel_collector/defaults/bigip-scraper-config.yaml index 14e9f22..ba67316 100644 --- a/services/otel_collector/defaults/bigip-scraper-config.yaml +++ b/services/otel_collector/defaults/bigip-scraper-config.yaml @@ -1,7 +1,8 @@ receivers: ${file:/etc/otel-collector-config/receivers.yaml} processors: - batch: + batch/f5-datafabric: + send_batch_max_size: 8192 # Only export data to f5 (if enabled) every 300s interval/f5-datafabric: interval: 300s From 4c92f30011cb86502f899e736c9c2ad7e79fc636 Mon Sep 17 00:00:00 2001 From: Chris Hain Date: Tue, 15 Oct 2024 15:11:04 -0700 Subject: [PATCH 02/10] Remove default-config.yaml, switch otel scrape job to prometheus, update docs --- config/ast_defaults.yaml | 2 +- docker-compose.yaml | 3 +- pages/config_management.md | 9 ++--- pages/config_migration.md | 2 +- pages/metric_obfuscation.md | 2 +- .../defaults/bigip-scraper-config.yaml | 1 + .../defaults/default-config.yaml | 37 ------------------- services/otel_collector/pipelines.yaml | 2 +- services/prometheus/prometheus.yml | 6 ++- 9 files changed, 15 insertions(+), 49 deletions(-) delete mode 100644 services/otel_collector/defaults/default-config.yaml diff --git a/config/ast_defaults.yaml b/config/ast_defaults.yaml index bc380eb..8ec3c6c 100644 --- a/config/ast_defaults.yaml +++ b/config/ast_defaults.yaml @@ -44,7 +44,7 @@ pipelines: # detail. metrics/local: #receivers list are generated via the config helper script - processors: [batch] + processors: [batch/local] exporters: [otlphttp/metrics-local, debug/bigip] # These pipeline configs are written to the OTEL config after having the configured receivers diff --git a/docker-compose.yaml b/docker-compose.yaml index 497b887..7c121c7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -31,8 +31,7 @@ services: restart: unless-stopped volumes: - ./services/otel_collector:/etc/otel-collector-config - command: - - "--config=/etc/otel-collector-config/defaults/default-config.yaml" + command: - "--config=/etc/otel-collector-config/defaults/bigip-scraper-config.yaml" env_file: - ".env" diff --git a/pages/config_management.md b/pages/config_management.md index b619c6a..e1d1e92 100644 --- a/pages/config_management.md +++ b/pages/config_management.md @@ -69,7 +69,7 @@ pipelines: # detail. metrics/local: #receivers list are generated via the config helper script - processors: [batch] + processors: [batch/local] exporters: [otlphttp/metrics-local, debug/bigip] # These pipeline configs are written to the OTEL config after having the configured receivers @@ -78,7 +78,7 @@ pipelines: # detail. metrics/f5-datafabric: #receivers list are generated via the config helper script - processors: [batch, interval/f5-datafabric, attributes/f5-datafabric] + processors: [interval/f5-datafabric, attributes/f5-datafabric, batch/f5-datafabric] exporters: [otlp/f5-datafabric, debug/bigip] ``` @@ -118,7 +118,7 @@ metrics/bigip: - otlphttp/metrics-bigip - debug/bigip processors: - - batch + - batch/local receivers: # This was inserted here because the pipeline / default_pipeline for this device was # set to "metrics/bigip" @@ -177,7 +177,7 @@ receivers and services.pipelines section of the receivers: ${file:/etc/otel-collector-config/receivers.yaml} processors: - batch: + batch/local: exporters: otlphttp/metrics-bigip: @@ -209,7 +209,6 @@ own config files: - ./services/otel_collector:/etc/otel-collector-config # Update these as needed command: - - "--config=/etc/otel-collector-config/defaults/default-config.yaml" - "--config=/etc/otel-collector-config/defaults/bigip-scraper-config.yaml" ... ``` \ No newline at end of file diff --git a/pages/config_migration.md b/pages/config_migration.md index bba9995..2d8bc2b 100644 --- a/pages/config_migration.md +++ b/pages/config_migration.md @@ -81,7 +81,7 @@ pipeline_default: metrics/bigip pipelines: metrics/bigip: #receivers list are generated via the config helper script - processors: [batch] + processors: [batch/local] exporters: [otlphttp/metrics-bigip, debug/bigip] ``` diff --git a/pages/metric_obfuscation.md b/pages/metric_obfuscation.md index 62a27d6..a449d4f 100644 --- a/pages/metric_obfuscation.md +++ b/pages/metric_obfuscation.md @@ -41,7 +41,7 @@ pipelines: metrics/f5-datafabric: # receivers list are generated via the config helper script # Adding attributes/mask-some-attributes to the list of enabled processors. - processors: [batch, interval/f5-datafabric, attributes/mask-some-attributes, attributes/f5-datafabric] + processors: [interval/f5-datafabric, attributes/mask-some-attributes, attributes/f5-datafabric, batch/f5-datafabric] exporters: [otlp/f5-datafabric, debug/bigip] ... ``` \ No newline at end of file diff --git a/services/otel_collector/defaults/bigip-scraper-config.yaml b/services/otel_collector/defaults/bigip-scraper-config.yaml index ba67316..4933fc7 100644 --- a/services/otel_collector/defaults/bigip-scraper-config.yaml +++ b/services/otel_collector/defaults/bigip-scraper-config.yaml @@ -1,6 +1,7 @@ receivers: ${file:/etc/otel-collector-config/receivers.yaml} processors: + batch/local: batch/f5-datafabric: send_batch_max_size: 8192 # Only export data to f5 (if enabled) every 300s diff --git a/services/otel_collector/defaults/default-config.yaml b/services/otel_collector/defaults/default-config.yaml deleted file mode 100644 index 61923f1..0000000 --- a/services/otel_collector/defaults/default-config.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# This file contains default / non-BigIP specific settings that allow -# The otel collector to monitor itself. Most people won't need to configure -# anything in this file. -receivers: - prometheus/collector: - config: - scrape_configs: - - job_name: 'opentelemetry-collector' - static_configs: - - targets: ['localhost:8888'] - -processors: - batch: - interval/xcdf-interval: - interval: 300s - attributes/xcdf-datatype: - actions: - - key: dataType - action: upsert - value: bigip-ast-metric - -exporters: - otlphttp/prometheus-default: - endpoint: http://prometheus:9090/api/v1/otlp - # Support sending downsampled metrics to F5 Data Fabric if enabled. - - debug/default: - verbosity: basic - sampling_initial: 5 - sampling_thereafter: 200 - -service: - pipelines: - metrics/prometheus: - receivers: [prometheus/collector] - processors: [batch] - exporters: [otlphttp/prometheus-default, debug/default] \ No newline at end of file diff --git a/services/otel_collector/pipelines.yaml b/services/otel_collector/pipelines.yaml index 36a4bf5..7d7c849 100644 --- a/services/otel_collector/pipelines.yaml +++ b/services/otel_collector/pipelines.yaml @@ -3,6 +3,6 @@ metrics/local: - otlphttp/metrics-local - debug/bigip processors: - - batch + - batch/local receivers: - bigip/1 diff --git a/services/prometheus/prometheus.yml b/services/prometheus/prometheus.yml index d8d219e..a22aa81 100644 --- a/services/prometheus/prometheus.yml +++ b/services/prometheus/prometheus.yml @@ -5,4 +5,8 @@ scrape_configs: - job_name: 'prometheus' scrape_interval: 1m static_configs: - - targets: ['localhost:9090'] \ No newline at end of file + - targets: ['localhost:9090'] + - job_name: 'otel-collector' + scrape_interval: 30s + static_configs: + - targets: ['otel-collector:8888'] \ No newline at end of file From 48a4002d8f629b4179a005ca2ab4ad9a86e20054 Mon Sep 17 00:00:00 2001 From: Chris Hain Date: Wed, 16 Oct 2024 09:29:41 -0700 Subject: [PATCH 03/10] Set default http client timeout to 10s and illustrate how to overide --- config/ast_defaults.yaml | 4 ++++ config/bigip_receivers.yaml | 1 + 2 files changed, 5 insertions(+) diff --git a/config/ast_defaults.yaml b/config/ast_defaults.yaml index 8ec3c6c..a266779 100644 --- a/config/ast_defaults.yaml +++ b/config/ast_defaults.yaml @@ -8,6 +8,10 @@ bigip_receiver_defaults: # The password (not recommended) or a reference to an env variable (recommended, shown) # Below tells the collector to look for an environment variable named BIGIP_PASSWORD_1 password: "${env:BIGIP_PASSWORD_1}" + # The timeout field can be used to adjust the amount of time the collector will wait for a response + # to BigIP iControl Rest requests. Larger boxes with more complex config may require setting this value + # higher. Set for individual devices in bigip_receivers.yaml + timeout: 10s # The data_types that should be enabled or disabled. DNS and GTM users can enable those modules # by setting the below to true. These will apply to all devices and may be better specified on the # per-reciever settings file below. diff --git a/config/bigip_receivers.yaml b/config/bigip_receivers.yaml index f2beb84..9d79c7e 100644 --- a/config/bigip_receivers.yaml +++ b/config/bigip_receivers.yaml @@ -13,6 +13,7 @@ bigip/1: # username: SOME_OVERRIDE_ACCOUNT_NAME # password: "${SOME_OTHER_ENV_VAR_WITH_ANOTHER_PASSWORD}" # collection_interval: 30s + # timeout: 20s # data_types: # f5.dns: # enabled: false From d7c018445d2a310ba533acc6462c8385beb9bb46 Mon Sep 17 00:00:00 2001 From: Chris Hain Date: Thu, 17 Oct 2024 10:07:34 -0700 Subject: [PATCH 04/10] docsite theme, restructure, add content --- README.md | 28 ++- _config.yml | 1 - pages/.gitignore | 16 +- pages/Gemfile | 13 +- pages/Gemfile.lock | 118 ++++++++++ pages/_config.yml | 45 +++- pages/components/components.md | 26 +++ pages/{ => components/grafana}/dashboard.md | 55 ++--- pages/components/grafana/grafana.md | 60 +++++ .../otel_collector/otel_collector.md | 47 ++++ .../otel_collector}/receiver_metrics.md | 13 +- .../otel_collector}/receiver_readme.md | 12 + pages/components/prometheus/prometheus.md | 43 ++++ pages/config/config.md | 28 +++ .../config/config_helper/config_datafabric.md | 48 ++++ pages/config/config_helper/config_defaults.md | 136 +++++++++++ pages/config/config_helper/config_dns_gtm.md | 52 +++++ pages/config/config_helper/config_helper.md | 132 +++++++++++ .../config/config_helper/config_receivers.md | 49 ++++ pages/config/config_helper/config_tls.md | 35 +++ .../config_helper}/metric_obfuscation.md | 12 + pages/{ => config}/config_migration.md | 13 +- pages/config/manual_config/default_otel.md | 59 +++++ pages/config/manual_config/manual_config.md | 12 + .../manual_config/receivers_pipelines.md | 99 ++++++++ pages/config_management.md | 214 ------------------ pages/datafabric/configuration.md | 8 + pages/datafabric/datafabric.md | 19 ++ pages/{quickstart.md => getting_started.md} | 27 ++- pages/index.md | 38 +--- pages/troubleshooting/troubleshooting.md | 141 ++++++++++++ 31 files changed, 1293 insertions(+), 306 deletions(-) delete mode 100644 _config.yml create mode 100644 pages/Gemfile.lock create mode 100644 pages/components/components.md rename pages/{ => components/grafana}/dashboard.md (68%) create mode 100644 pages/components/grafana/grafana.md create mode 100644 pages/components/otel_collector/otel_collector.md rename pages/{ => components/otel_collector}/receiver_metrics.md (99%) rename pages/{ => components/otel_collector}/receiver_readme.md (95%) create mode 100644 pages/components/prometheus/prometheus.md create mode 100644 pages/config/config.md create mode 100644 pages/config/config_helper/config_datafabric.md create mode 100644 pages/config/config_helper/config_defaults.md create mode 100644 pages/config/config_helper/config_dns_gtm.md create mode 100644 pages/config/config_helper/config_helper.md create mode 100644 pages/config/config_helper/config_receivers.md create mode 100644 pages/config/config_helper/config_tls.md rename pages/{ => config/config_helper}/metric_obfuscation.md (88%) rename pages/{ => config}/config_migration.md (96%) create mode 100644 pages/config/manual_config/default_otel.md create mode 100644 pages/config/manual_config/manual_config.md create mode 100644 pages/config/manual_config/receivers_pipelines.md delete mode 100644 pages/config_management.md create mode 100644 pages/datafabric/configuration.md create mode 100644 pages/datafabric/datafabric.md rename pages/{quickstart.md => getting_started.md} (88%) create mode 100644 pages/troubleshooting/troubleshooting.md diff --git a/README.md b/README.md index fc7e531..26226a2 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,12 @@ update a legacy configuration, see [pages/config_migration.md](pages/config_migr > > Before you start, make sure to backup the /config/big-ips.json file! + ## Overview +> See the [AST Docsite](https://f5devcentral.github.io/application-study-tool/) for detailed +configuration, troubleshooting info, etc. + The Application Study Tool is intended to provide enhanced insights into (classic) BIG-IP products, leveraging best in class open source telemetry tools. The full installation includes: @@ -27,7 +31,15 @@ the Openetlemetry Collector can be configured to send data to existing productio ### Prerequisites -docker (or compatible) - [Installation Instructions](https://docs.docker.com/engine/install/) +[Git Client](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) + +Docker (or compatible) container environment with compose. + +Installation Instructions: + * [General (docker engine)](https://docs.docker.com/engine/install/) + * [Ubuntu (docker engine)](https://docs.docker.com/engine/install/ubuntu/) + * [RHEL (docker engine)](https://docs.docker.com/engine/install/rhel/) + * [Podman](https://podman.io/docs/installation) ### Installation @@ -206,9 +218,19 @@ as SENSOR_ID and SENSOR_SECRET_TOKEN (see [.env-example](./.env-example) for exa ## Run The Configuration Helper The config helper script can be run natively or via docker to merge the default and device level configs into the final OTEL Collector config from the project root directory as follows: -```shell + +**Run With Docker** +```bash # Run the configuration generator from the project root directory -docker run --rm -it -w /app -v ${PWD}:/app --entrypoint /app/src/bin/init_entrypoint.sh python:3.12.6-slim-bookworm --generate-config +# If `echo $PWD` doesn't give you the current directory on your system, +# replace the '-v ${PWD}' section with '-v /path/to/your/directory' +$ docker run --rm -it -w /app -v ${PWD}:/app --entrypoint /app/src/bin/init_entrypoint.sh python:3.12.6-slim-bookworm --generate-config +``` + +**Run With System Python** +```bash +$ pip install PyYAML==6.0.2 +$ python /app/src/config_helper.py --generate-config ``` This will write 2 new files in the services/otel_collector directory: diff --git a/_config.yml b/_config.yml deleted file mode 100644 index 911bafd..0000000 --- a/_config.yml +++ /dev/null @@ -1 +0,0 @@ -markdown: kramdown \ No newline at end of file diff --git a/pages/.gitignore b/pages/.gitignore index a9cc8ab..350388c 100644 --- a/pages/.gitignore +++ b/pages/.gitignore @@ -1,6 +1,12 @@ -_site -.sass-cache -.jekyll-cache +# These are directly copied from Jekyll's first-party docs on `.gitignore` files: +# https://jekyllrb.com/tutorials/using-jekyll-with-bundler/#commit-to-source-control + +# Ignore the default location of the built site, and caches and metadata generated by Jekyll +_site/ +.sass-cache/ +.jekyll-cache/ .jekyll-metadata -vendor -Gemfile.lock \ No newline at end of file + +# Ignore folders generated by Bundler +.bundle/ +vendor/ \ No newline at end of file diff --git a/pages/Gemfile b/pages/Gemfile index 265df2c..ecb5d6f 100644 --- a/pages/Gemfile +++ b/pages/Gemfile @@ -7,17 +7,16 @@ source "https://rubygems.org" # # This will help ensure the proper Jekyll version is running. # Happy Jekylling! -# gem "jekyll", "~> 4.3.4" -gem "github-pages", "~> 232", group: :jekyll_plugins -# This is the default theme for new Jekyll sites. You may change this to anything you like. -gem "minima", "~> 2.5" +gem "jekyll", "~> 4.3.4" +# gem "github-pages", "~> 232", group: :jekyll_plugins +gem "just-the-docs", "0.10.0" # pinned to the current release # If you want to use GitHub Pages, remove the "gem "jekyll"" above and # uncomment the line below. To upgrade, run `bundle update github-pages`. # gem "github-pages", group: :jekyll_plugins # If you have any plugins, put them here! -group :jekyll_plugins do - gem "jekyll-feed", "~> 0.12" -end +# group :jekyll_plugins do +# gem "jekyll-feed", "~> 0.12" +# end # Windows and JRuby does not include zoneinfo files, so bundle the tzinfo-data gem # and associated library. diff --git a/pages/Gemfile.lock b/pages/Gemfile.lock new file mode 100644 index 0000000..ce522d6 --- /dev/null +++ b/pages/Gemfile.lock @@ -0,0 +1,118 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + bigdecimal (3.1.8) + colorator (1.1.0) + concurrent-ruby (1.3.4) + diff-lcs (1.5.1) + em-websocket (0.5.3) + eventmachine (>= 0.12.9) + http_parser.rb (~> 0) + eventmachine (1.2.7) + ffi (1.17.0-arm64-darwin) + ffi (1.17.0-x86_64-linux-gnu) + forwardable-extended (2.6.0) + google-protobuf (4.28.1-arm64-darwin) + bigdecimal + rake (>= 13) + google-protobuf (4.28.1-x86_64-linux) + bigdecimal + rake (>= 13) + http_parser.rb (0.8.0) + i18n (1.14.6) + concurrent-ruby (~> 1.0) + jekyll (4.3.4) + addressable (~> 2.4) + colorator (~> 1.0) + em-websocket (~> 0.5) + i18n (~> 1.0) + jekyll-sass-converter (>= 2.0, < 4.0) + jekyll-watch (~> 2.0) + kramdown (~> 2.3, >= 2.3.1) + kramdown-parser-gfm (~> 1.0) + liquid (~> 4.0) + mercenary (>= 0.3.6, < 0.5) + pathutil (~> 0.9) + rouge (>= 3.0, < 5.0) + safe_yaml (~> 1.0) + terminal-table (>= 1.8, < 4.0) + webrick (~> 1.7) + jekyll-include-cache (0.2.1) + jekyll (>= 3.7, < 5.0) + jekyll-sass-converter (3.0.0) + sass-embedded (~> 1.54) + jekyll-seo-tag (2.8.0) + jekyll (>= 3.8, < 5.0) + jekyll-watch (2.2.1) + listen (~> 3.0) + just-the-docs (0.10.0) + jekyll (>= 3.8.5) + jekyll-include-cache + jekyll-seo-tag (>= 2.0) + rake (>= 12.3.1) + kramdown (2.4.0) + rexml + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) + liquid (4.0.4) + listen (3.9.0) + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) + mercenary (0.4.0) + nokogiri (1.16.7-arm64-darwin) + racc (~> 1.4) + nokogiri (1.16.7-x86_64-linux) + racc (~> 1.4) + pathutil (0.16.2) + forwardable-extended (~> 2.6) + public_suffix (6.0.1) + racc (1.8.1) + rack (2.2.10) + rake (13.2.1) + rb-fsevent (0.11.2) + rb-inotify (0.11.1) + ffi (~> 1.0) + rexml (3.3.7) + rouge (4.3.0) + rspec (3.13.0) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.1) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.1) + safe_yaml (1.0.5) + sass-embedded (1.78.0-arm64-darwin) + google-protobuf (~> 4.27) + sass-embedded (1.78.0-x86_64-linux-gnu) + google-protobuf (~> 4.27) + terminal-table (3.0.2) + unicode-display_width (>= 1.1.1, < 3) + unicode-display_width (2.6.0) + webrick (1.8.1) + +PLATFORMS + arm64-darwin + x86_64-linux-gnu + +DEPENDENCIES + http_parser.rb (~> 0.6.0) + jekyll (~> 4.3.4) + just-the-docs (= 0.10.0) + nokogiri + rack (~> 2.2.4) + rspec + tzinfo (>= 1, < 3) + tzinfo-data + wdm (~> 0.1) + +BUNDLED WITH + 2.5.9 diff --git a/pages/_config.yml b/pages/_config.yml index 6bb978a..21862fb 100644 --- a/pages/_config.yml +++ b/pages/_config.yml @@ -18,13 +18,50 @@ # You can create any custom variable you would like, and they will be accessible # in the templates via {{ site.myvariable }}. -title: Application Study Tool Docs +title: Application Study Tool +description: A tool for monitoring BigIP Devices with Grafana, Prometheus, and OpenTelemetry # Build settings -theme: minima -plugins: - - jekyll-feed +theme: just-the-docs +url: https://f5devcentral.github.io/application-study-tool/ + +aux_links: + AST Repository: https://github.com/f5devcentral/application-study-tool/ + +heading_anchors: true +nav_enabled: true + +nav_sort: case_sensitive + +# External navigation links +nav_external_links: + - title: Application Study Tool on GitHub + url: https://github.com/f5devcentral/application-study-tool/ + +# Back to top link +back_to_top: true +back_to_top_text: "Back to top" + +# Footer last edited timestamp +last_edit_timestamp: true # show or hide edit time - page must have `last_modified_date` defined in the frontmatter +last_edit_time_format: "%b %e %Y at %I:%M %p" # uses ruby's time format: https://ruby-doc.org/stdlib-2.7.0/libdoc/time/rdoc/Time.html + +compress_html: + clippings: all + comments: all + endings: all + startings: [] + blanklines: false + profile: false + +callouts: + warning: + title: Warning + color: red + important: + title: Important + color: blue # Exclude from processing. # The following items will not be processed, by default. diff --git a/pages/components/components.md b/pages/components/components.md new file mode 100644 index 0000000..e8fd11a --- /dev/null +++ b/pages/components/components.md @@ -0,0 +1,26 @@ +--- +layout: page +title: Components +nav_order: 4 +has_children: true +permalink: /components +--- + +# Components + +The Application Study Tool is built using best-of-breed open source components, allowing for +flexibility, community support, and continuous improvement. By leveraging established technologies like +Grafana, Prometheus, and OpenTelemetry, the AST ensures a reliable and robust method for collecting and +visualizing data from BigIP devices. + +## Grafana + +[**Grafana**](https://grafana.com/docs/grafana/latest/) serves as the visualization layer for the Application Study Tool, allowing users to create and manage dashboards that display metrics collected from F5 BigIP devices. With its powerful querying capabilities and customizable visualizations, Grafana enables users to gain insights into application performance and operational metrics effectively. + +## Prometheus + +[**Prometheus**](https://prometheus.io/docs/introduction/overview/) is the time series database utilized within the Application Study Tool to store and manage metric data. It collects data emitted by the OpenTelemetry Collector and provides powerful querying capabilities for analysis. Prometheus is designed for reliability and scalability, making it the backbone of metric storage for many monitoring applications. + +## OpenTelemetry Collector + +The [**OpenTelemetry Collector**](https://opentelemetry.io/docs/collector/) is a custom distribution integrated into the Application Study Tool that collects metrics from F5 BigIP devices. Featuring a **BigIP Receiver** component, it retrieves data from the BigIP iControl REST endpoint and transforms it into OpenTelemetry metrics. This data is then forwarded to Prometheus for storage and visualization, ensuring seamless integration across the monitoring stack. diff --git a/pages/dashboard.md b/pages/components/grafana/dashboard.md similarity index 68% rename from pages/dashboard.md rename to pages/components/grafana/dashboard.md index acd25be..23b6e83 100644 --- a/pages/dashboard.md +++ b/pages/components/grafana/dashboard.md @@ -1,24 +1,17 @@ -# Dashboards +--- +layout: page +title: Dashboard Examples +nav_order: 2 +parent: Grafana +grand_parent: Components +--- +# Dashboard Examples Application Study Tool includes a variety of starter dashboards to help monitor your BigIP devices. These dashboards are built in Grafana and can be customized to suit your unique organizational needs. -## Table of Contents -- [Fleet](#fleet-dashboards) - - [Inventory](#inventory) - - [Device Utilization](#device-utilization) - - [Virtual Servers](#virtual-servers) - - [SSL Certs](#ssl-certs) -- [Device](#device-dashboards) - - [Overview](#overview) - - [Top N](#top-n) - - [Pools](#pools) - - [Virtual Servers](#virtual-servers-1) - - [iRules](#irules) - - [WAF](.dashboard.) -- [Profile](#profile-dashboards) - - [HTTP](#http) -- [Stats](#collector-stats) +1. TOC +{:toc} ## Accessing the dashboards @@ -37,20 +30,20 @@ You can update the Grafana user/pass by changing the values of the `GF_SECURITY_ The fleet dashboards provide top level information of your environment #### Inventory -![](./assets/BigIP-Fleet-Inventory.png) +![](../../assets/BigIP-Fleet-Inventory.png) *The inventory dashboard allows you to view the differences between BigIP devices in your inventory.* #### Device Utilization -![](./assets/BigIP-Fleet-Device-Utilization.png) +![](../../assets/BigIP-Fleet-Device-Utilization.png) *This dashboard provides a quick view on how much usage your box is receiving. To view more information about the device take a look at the [device overview dashboard](#overview).* #### Virtual Servers -![](./assets/BigIP-Fleet-Virtual-Server.png) +![](../../assets/BigIP-Fleet-Virtual-Server.png) *The Virtual Server dashboard provides an overview of all the virtual servers present in your inventory.* #### SSL Certs -![](./assets/BigIP-Fleet-SSL-Certs.png) +![](../../assets/BigIP-Fleet-SSL-Certs.png) *The SSL Certs view allows you to stay up to date on the status of your SSL certficates.* ### Device Dashboards @@ -58,27 +51,27 @@ The fleet dashboards provide top level information of your environment The device dashboards provide more grainular information about how the device is configured and how it is performing #### Overview -![](./assets/BipIP-Device-Overview.png) +![](../../assets/BipIP-Device-Overview.png) *This dashboard provides variety of metrics about the performance of the device.* #### Top N -![](./assets/BigIP-Device-TopN.png) +![](../../assets/BigIP-Device-TopN.png) *The Top N dashboard provides statistical highlights by presenting stats in a Top N, by default N is set to 10 this can be changed at top of the dashboard* #### Pools -![](./assets/BigIP-Device-Pools.png) -*Here you can view stats pretaining to Pools and Pool members on a BigIP.* +![](../../assets/BigIP-Device-Pools.png) +*Here you can view stats pertaining to Pools and Pool members on a BigIP.* #### Virtual Servers -![](./assets/BigIP-Device-Virtual-Server.png) -*Here you can view stats pretaining to all the virtual servers on a BigIP.* +![](../../assets/BigIP-Device-Virtual-Server.png) +*Here you can view stats pertaining to all the virtual servers on a BigIP.* #### iRules -![](./assets/BigIP-Device-iRules.png) +![](../../assets/BigIP-Device-iRules.png) *This dashboard allows you to monitor iRule usage and performance.* #### WAF -![](./assets/BigIP-Device-WAF.png) +![](../../assets/BigIP-Device-WAF.png) *View the status of your WAF policies here.* @@ -89,7 +82,7 @@ The profile dashboards dive into the different profiles configured on the device #### HTTP -![](./assets/BigIP-Device:Profile-HTTP.png) +![](../../assets/BigIP-Device:Profile-HTTP.png) *This dashboard enables the user to monitor their HTTP profiles, view information pretaining to HTTP requests and responses* @@ -97,4 +90,4 @@ The profile dashboards dive into the different profiles configured on the device You can view stats about the Collector from the scrape duration to API requests/responses -![](./assets/Collector-Stats.png) \ No newline at end of file +![](../../assets/Collector-Stats.png) \ No newline at end of file diff --git a/pages/components/grafana/grafana.md b/pages/components/grafana/grafana.md new file mode 100644 index 0000000..f31bcee --- /dev/null +++ b/pages/components/grafana/grafana.md @@ -0,0 +1,60 @@ +--- +layout: page +title: Grafana +nav_order: 1 +parent: Components +has_children: true +permalink: /components/grafana +--- + +## Grafana In AST + +The Application Study Tool leverages Grafana for visualization of all collected metrics stored in +prometheus. + +The instance is pre-configured with dashboards built by the AST team to provide insights into the fleet +of BigIP devices being monitored. Grafana is a flexible visualization tool, and AST users are encouraged +to customize the 'out of the box' dashboards or build new views around the AST data that suit your +specific organization's workflows. + + +### Configure Grafana Settings + +The **Grafana** instance within the **Application Study Tool (AST)** can be easily configured using +environment variables, following the standard [Grafana configuration options](https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#override-configuration-with-environment-variables). This +flexibility allows you to tailor your Grafana setup to meet your specific requirements +(e.g. authentication integration, TLS, and other settings required) + +#### Passing Environment Variables To The Instance + +The included docker-compose file looks for a .env file at the root of the project directory and will +mount that at start time. Variables can be added / modified here, or in the docker-compose file directly +using the standard docker-compose syntax for container environment + +```bash +$ cat .env-example +# Grafana Environment Variables +# These should be updated to more secure values outside of testing environments. +GF_SECURITY_ADMIN_USER=admin +GF_SECURITY_ADMIN_PASSWORD=admin +$ cp .env-example .env +$ vi .env +``` + +### Modify Grafana Views +The AST team will maintain a set of dashboards that cover a wide variety +of use-cases, but for needs that have less broad applicability or to suit particular tastes +(e.g. bar-chart vs pie-chart), users are encouraged to modify or create new views into the AST data. + +By default, all dashboards within an AST instance are open to editing directly via the ui. Follow the +[instructions](https://grafana.com/docs/grafana/latest/dashboards/) from grafana for building or editing +dashboards, visualizations, etc. + +### Save Grafana Views Outside Docker Volume +Saving a dashboard view in the Grafana UI will write it to the docker volume for persistence across +container restarts. In order to persist dashboards outside the docker environment, you can export them as +JSON following these +[instructions](https://grafana.com/docs/grafana/latest/dashboards/share-dashboards-panels/#export-a-dashboard-as-json). + +You can save dashboard files under /services/grafana/provisioning/dashboards from the project root to +automatically load them when the container is started. \ No newline at end of file diff --git a/pages/components/otel_collector/otel_collector.md b/pages/components/otel_collector/otel_collector.md new file mode 100644 index 0000000..f3c894a --- /dev/null +++ b/pages/components/otel_collector/otel_collector.md @@ -0,0 +1,47 @@ +--- +layout: page +title: OTEL Collector +nav_order: 5 +parent: Components +has_children: true +permalink: /components/otel_collector +--- + +The **OpenTelemetry Collector** is a crucial component of the **Application Study Tool (AST)**, +providing a custom distribution designed specifically for monitoring F5 BigIP devices. This collector +features a specialized **BigIP Receiver** component, which +retrieves metric data from the BigIP iControl REST endpoint. + +While working towards open sourcing the receiver component, the AST project will maintain a build of the +collector with the requisite receivers, processors, and exporters needed to collect and send data where +required. If you have a particular exporter that you'd like included in this list, please open a support +issue. + +## Custom Collector Components + +### Exporters +* [OTLPExporter](https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/otlpexporter) +* [OTLPHTTPExporter](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/otlphttpexporter/README.md) +* [DebugExporter](https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/debugexporter) +* [PrometheusRemoteWriteExporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/prometheusremotewriteexporter) +* [LokiExporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/lokiexporter/README.md) +* [ClickHouseExporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/clickhouseexporter) + +### Receivers +* [OTLPReceiver](https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver/otlpreceiver) +* [PrometheusReceiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/prometheusreceiver) +* [SyslogReceiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/syslogreceiver) +* [AST BigIPReceiver](https://github.com/f5devcentral/application-study-tool/) + +### Processors +* [BatchProcessor](https://github.com/open-telemetry/opentelemetry-collector/blob/main/processor/batchprocessor/README.md) +* [TransformProcessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/transformprocessor) +* [ResourceProcessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourceprocessor) +* [MetricsTransformProcessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/metricstransformprocessor) +* [AttributesProcessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/attributesprocessor) +* [FilterProcessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/filterprocessor) +* [IntervalProcessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/intervalprocessor) + +### Extensions +* [BasicAuthExtension](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/basicauthextension) +* [BearerTokenAuthExtension](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/bearertokenauthextension) \ No newline at end of file diff --git a/pages/receiver_metrics.md b/pages/components/otel_collector/receiver_metrics.md similarity index 99% rename from pages/receiver_metrics.md rename to pages/components/otel_collector/receiver_metrics.md index 56e26e6..640a56a 100644 --- a/pages/receiver_metrics.md +++ b/pages/components/otel_collector/receiver_metrics.md @@ -1,6 +1,12 @@ -[comment]: <> (Code generated by mdatagen. DO NOT EDIT.) +--- +layout: page +title: BigIP Receiver Metrics +parent: OTEL Collector +grand_parent: Components +nav_order: 2 +--- +# BigIP Receiver (AST) - Metrics -# bigip ## Default Metrics @@ -12,6 +18,9 @@ metrics: enabled: false ``` +1. TOC +{:toc} + ### endpoint.scrape.duration The time spent performing a HTTP request for a particular endpoint diff --git a/pages/receiver_readme.md b/pages/components/otel_collector/receiver_readme.md similarity index 95% rename from pages/receiver_readme.md rename to pages/components/otel_collector/receiver_readme.md index e346bf3..8f18106 100644 --- a/pages/receiver_readme.md +++ b/pages/components/otel_collector/receiver_readme.md @@ -1,7 +1,19 @@ +--- +layout: page +title: BigIP Receiver Readme +parent: OTEL Collector +grand_parent: Components +nav_order: 1 +--- + # F5 Big-IP Receiver This receiver fetches stats from a F5 Big-IP node using F5's [iControl REST API](https://clouddocs.f5.com/api/icontrol-rest). + +1. TOC +{:toc} + ## Prerequisites This receiver supports Big-IP versions `11.6.5+` diff --git a/pages/components/prometheus/prometheus.md b/pages/components/prometheus/prometheus.md new file mode 100644 index 0000000..ba1034b --- /dev/null +++ b/pages/components/prometheus/prometheus.md @@ -0,0 +1,43 @@ +--- +layout: page +title: Prometheus +nav_order: 1 +parent: Components +has_children: true +permalink: /components/prometheus +--- + +# Prometheus Integration in Application Study Tool + +The **Application Study Tool (AST)** leverages **Prometheus**, a powerful time series database, to collect and visualize metric data from F5 BigIP devices through an OpenTelemetry collector. + +## Configuration + +To customize your Prometheus setup, you can modify the Prometheus configuration file located at: + +``` +/services/prometheus/prometheus.yaml +``` + +For detailed guidance on the configuration options available, refer to the [Prometheus Configuration Documentation](https://prometheus.io/docs/prometheus/latest/configuration/configuration/). + +## Accessing Prometheus + +You can access the Prometheus service directly on port **9090** of the host where the Application Study Tool is running. + +## Querying Metrics + +For querying the metrics collected by Prometheus, either directly or by editing Grafana dashboar panels, +see the [Prometheus Query Syntax Documentation](https://prometheus.io/docs/prometheus/latest/querying/basics/). + +## Storage Sizing + +When scoping out the AST server where your Prometheus instance will run, it's important to consider its +storage and memory requirements, as it is the largest consumer of disk and memory resources in the AST +project. + +A single modestly sized server can accommodate many BigIP Devices, but exact specs will depend heavily on +how the devices are configured, storage retention settings, etc. + +For guidance on instance storage sizing, please review the [Prometheus Storage Sizing Documentation](https://prometheus.io/docs/prometheus/1.8/storage/). + diff --git a/pages/config/config.md b/pages/config/config.md new file mode 100644 index 0000000..81075fd --- /dev/null +++ b/pages/config/config.md @@ -0,0 +1,28 @@ +--- +layout: page +title: Configuration +nav_order: 3 +has_children: true +permalink: /config +--- + + +# AST Configuration Management Overview +This configuration section applies to the AST Opentelemetry Collector. For Grafana, Prometheus or other +components, see the corresponding documentation in [> Components]({{site.baseurl}}/components/) + + +## Config Management Options +In the post v0.6.0 management scheme, users can choose from one of the below options to manage +the AST Otel Collector configs: + +1. Using a python script to generate full Otel Collector Config files from a small set of default +settings and per-device overrides (recommended for most users, and includes migration path from old big-ips.json configs). See [Configuration > Configuration Helper (Recommended)]({{site.baseurl}}/config/config_helper/) for details. + +2. Manual maintenance of the Otel Collector Config Receivers and Pipelines files (recommended for +advanced use cases or people with their own automation pipelines). See [Configuration > Manual Configuration > Receiver/Pipeline Config]({{site.baseurl}}/config/manual_config/receivers_pipelines.html) for details. + +3. Manual maintenance of the Otel Collector Config files in +[/services/otel_collector/defaults](https://github.com/f5devcentral/application-study-tool/blob/main/services/otel_collector/defaults). See [Configuration > Manual Configuration > Std Otel Collector Config]({{site.baseurl}}/config/manual_config/default_otel.html/) for details. + + diff --git a/pages/config/config_helper/config_datafabric.md b/pages/config/config_helper/config_datafabric.md new file mode 100644 index 0000000..9b70631 --- /dev/null +++ b/pages/config/config_helper/config_datafabric.md @@ -0,0 +1,48 @@ +--- +layout: page +title: Configure Export To F5 Datafabric +parent: Configuration Helper (Recommended) +grandparent: Configuration +nav_order: 3 +--- + +## Configure Periodic Metric Data Export To F5 + +1. TOC +{:toc} + +The application study tool can be configured to periodically (every 5 minutes) export a snapshot of your +BigIP metrics to F5. Contact your F5 Sales Representative for a "Sensor ID" (a unique string used to +associate your metrics with your Organization) and a "Sensor Secret Token" (used to authenticate to the +F5 Datafabric as an authorized data sender for your Org). + +{: .important } +The config helper script must run after any changes to the default or device specific configs, +and the otel container restarted before changes will take effect. + +This functionality is enabled as follows: + +### Enable the flag in /config/ast_defaults.yaml file as follows: + +```yaml +# Set this true to enable periodic metric export to F5 DataFabric. +# Requires adding your sensor ID and secret token to the container environment (see .env-example). +# Contact your F5 sales rep to obtain the ID / secret token. +f5_data_export: true +``` + +### Add the Sensor ID and Secret Token to the .env file +Contact your F5 Sales Representative for a "Sensor ID" (a unique string used to +associate your metrics with your Organization) and a "Sensor Secret Token" (used to authenticate to the +F5 Datafabric as an authorized data sender for your Org). + +```bash +$ cp .env-example .env +$ cat .env +# Optional Parameters Required for metrics export to F5 DataFabric +SENSOR_SECRET_TOKEN="YOUR_TOKEN" +SENSOR_ID="YOUR_ID" +$ vi .env +``` + +Finally, [Run The Configuration Helper]({{site.baseurl}}/config/config_helper/#run-the-configuration-helper) and restart the project containers. \ No newline at end of file diff --git a/pages/config/config_helper/config_defaults.md b/pages/config/config_helper/config_defaults.md new file mode 100644 index 0000000..eca436c --- /dev/null +++ b/pages/config/config_helper/config_defaults.md @@ -0,0 +1,136 @@ +--- +layout: page +title: Default Configuration +parent: Configuration Helper (Recommended) +grandparent: Configuration +nav_order: 1 +--- + +## Configure Default Project Settings + +1. TOC +{:toc} + +The file [/config/ast_defaults.yaml](https://github.com/f5devcentral/application-study-tool/blob/main/config/ast_defaults.yaml) +contains global configuration settings that can be used to reduce the amount of boilerplate configuration +required for each BigIP device to be monitored. + +Settings for each device are configured as described in [Configuration > Configuration Helper (Recommended) > Device Configuration]({{site.baseurl}}/config/config_helper/config_receivers.html) + +{: .important } +The config helper script must run after any changes to the default or device specific configs, +and the Otel container restarted before changes will take effect. + +The config file appears as follows. Syntax for the bigip_receiver_defaults settings follow the spec laid +out in the [BigIP Receiver Readme]({{site.baseurl}}/components/otel_collector/receiver_readme.html) file. + +Details for each section are provided below. + +```yaml +bigip_receiver_defaults: + collection_interval: 60s + username: admin + password: "${env:BIGIP_PASSWORD_1}" + data_types: + f5.dns: + enabled: false + f5.gtm: + enabled: false + tls: + insecure_skip_verify: false + ca_file: "" + +f5_data_export: false + +# Most people should not need to modify settings below this line +pipeline_default: metrics/local +f5_pipeline_default: metrics/f5-datafabric + +pipelines: + + metrics/local: + #receivers list are generated via the config helper script + processors: [batch/local] + exporters: [otlphttp/metrics-local, debug/bigip] + + metrics/f5-datafabric: + #receivers list are generated via the config helper script + processors: [interval/f5-datafabric, attributes/f5-datafabric, batch/f5-datafabric] + exporters: [otlp/f5-datafabric, debug/bigip] +``` + + +### Bigip Receiver Defaults +These configs are applied to each entry in the bigip_receivers file where they don't contain an +equivalent / overriding entry. + +```yaml +bigip_receiver_defaults: + # The time to wait between metric collection runs + collection_interval: 60s + # The username to login to the device with + username: admin + # The password (not recommended) or a reference to an env variable (recommended) + # Below tells the collector to look for an environment variable named BIGIP_PASSWORD_1 + password: "${env:BIGIP_PASSWORD_1}" + # The data_types that should be enabled or disabled. DNS and GTM users can enable those modules + # by setting the below to true. These will apply to all devices and may be better specified on the + # per-reciever settings file below. + data_types: + f5.dns: + enabled: false + f5.gtm: + enabled: false + # The TLS settings to use. Either a CA file must be specified or insecure_skip_verify + # set to true (not recommended) + tls: + insecure_skip_verify: false + ca_file: "" +``` + +### F5 Data Export Flag +Set to true to enable periodic metric export to F5 DataFabric. +Requires adding your Sensor ID and secret token to the container environment (see .env-example). +Contact your F5 sales rep to obtain the ID / secret token. + +```yaml +f5_data_export: false +``` + +### Pipeline Default Settings +These settings shouldn't need to be changed for most users, but they control the pipeline assignment +for each configured BigIP Receiver. The name of the pipeline_default and/or f5_pipeline_default must +match a configured pipeline from the pipelines settings section below. + +The f5_pipeline_default is only applied when the above `f5_data_export` flag is set to true + +```yaml +# The default local pipeline to use if one isn't specified in the per-device configs. +pipeline_default: metrics/local +# The default pipeline to use if metric export to F5 is enabled +# (if f5_data_export.sensor_id field above is set) +f5_pipeline_default: metrics/f5-datafabric +``` + +### Pipelines Configs +These pipeline configs are written to the OTEL config after having the configured receivers +added to the dictionary in accordance with the "pipeline_default" field above and "pipeline" +field on the per-receiver config file. Otel Collector documentation explains the syntax in more +detail. + +You can include additional processors or adjust order (but most people shouldn't do this) for any valid +processor type as configured in the base default config file in /services/otel_collector/defaults/ +directory. + +```yaml +pipelines: + metrics/local: + #receivers list are generated via the config helper script + processors: [batch/local] + exporters: [otlphttp/metrics-local, debug/bigip] + + metrics/f5-datafabric: + #receivers list are generated via the config helper script + processors: [interval/f5-datafabric, attributes/f5-datafabric, batch/f5-datafabric] + exporters: [otlp/f5-datafabric, debug/bigip] +``` \ No newline at end of file diff --git a/pages/config/config_helper/config_dns_gtm.md b/pages/config/config_helper/config_dns_gtm.md new file mode 100644 index 0000000..2ed4f11 --- /dev/null +++ b/pages/config/config_helper/config_dns_gtm.md @@ -0,0 +1,52 @@ +--- +layout: page +title: Configure DNS & GTM (Optional) +parent: Configuration Helper (Recommended) +grandparent: Configuration +nav_order: 6 +--- + +## DNS and GTM Metrics + +By default, settings for the collection of DNS and GTM metrics are disabled. They add a relatively large number of +additional API calls, and since many devices may not have these features enabled, it was determined to +make them 'opt in'. + + +They can be enabled on a device by device basis (in `/config/bigip_receivers.yaml`), +or globally (in `/config/ast_defaults.yaml`) as follows: + +{: .important } +The config helper script must run after any changes to the default or device specific configs, +and the otel container restarted before changes will take effect. + +### Enable For Specific devices +Edit the `/config/bigip_receivers.yaml` and add the following data_types configs for the desired devices: + +```yaml +bigip/1: + endpoint: https://10.0.0.1 + # ... + # Enable DNS and/or GTM collection for this device: + data_types: + f5.dns: + enabled: true + f5.gtm: + enabled: true + # ... +``` + +### Enable For All devices +Edit the `/config/ast_defaults.yaml` and add the following data_types configs for the desired devices: + +```yaml +bigip_receiver_defaults: + # ... + # Enable DNS and/or GTM collection for all devices: + data_types: + f5.dns: + enabled: true + f5.gtm: + enabled: true + # ... +``` \ No newline at end of file diff --git a/pages/config/config_helper/config_helper.md b/pages/config/config_helper/config_helper.md new file mode 100644 index 0000000..b428528 --- /dev/null +++ b/pages/config/config_helper/config_helper.md @@ -0,0 +1,132 @@ +--- +layout: page +title: Configuration Helper (Recommended) +parent: Configuration +nav_order: 3 +has_children: true +permalink: /config/config_helper +--- + +# Managing AST Collector Configs With Config Helper + +This configuration section applies to the AST Opentelemetry Collector. For Grafana, Prometheus or other +components, see the corresponding documentation in [> Components]({{site.baseurl}}/components/) + +## Background +AST includes a helper python script that can be used to streamline the management of the Opentelemetry +collector. + +At a high level, it merges a set of default configuration settings with device specific +override settings, and outputs Opentelemetry Collector compatible configuration files that are mounted +by the container at runtime. + +{: .important } +The config helper script must run after any changes to the default or device specific configs, +and the otel container restarted before changes will take effect. + +## Before You Start +The default settings file reduces the amount of boilerplate that must be configured for each BigIP device +to be monitored. + +* [Configuration > Configuration Helper (Recommended) > Default Configuration]({{site.baseurl}}/config/config_helper/config_defaults.html) + +Settings without a logical default (e.g. url of the bigIP device) or wishing to override the default can +be set as described in: + +* [Configuration > Configuration Helper (Recommended) > Device Configuration]({{site.baseurl}}/config/config_helper/config_receivers.html) + +## Run The Configuration Helper +The config helper script can be run via docker (or natively if the system has python installed) +from the project root directory as follows: + +#### Run With Docker +```bash +# Run the configuration generator from the project root directory +# If `echo $PWD` doesn't give you the current directory on your system, +# replace the '-v ${PWD}' section with '-v /path/to/your/directory' +$ docker run --rm -it -w /app -v ${PWD}:/app --entrypoint /app/src/bin/init_entrypoint.sh python:3.12.6-slim-bookworm --generate-config +``` + +#### Run With System Python +```bash +$ pip install PyYAML==6.0.2 +$ python /app/src/config_helper.py --generate-config +``` + + +## Verify Output +Both of the above commands write 2 new files in the /services/otel_collector directory, which are +the actual files used by the OTEL Collector to configure itself at runtime. + +* `receivers.yaml` - The final list of scraper configs and their settings. +* `pipelines.yaml` - The final pipeline configs that map receivers to output destinations +(prometheus). + + +#### Receivers File +The `/services/otel_collector/receivers.yaml` file contains the OTEL Collector receivers configuration +[/services/otel_collector/receivers.yaml](https://github.com/f5devcentral/application-study-tool/blob/main/services/otel_collector/receivers.yaml) +which is the merged contents of the default config settings and the per-device settings: +```yaml +bigip/1: + collection_interval: 30s + data_types: + f5.dns: + enabled: false + f5.gtm: + enabled: false + endpoint: https://10.0.0.1 + password: ${SOME_OTHER_ENV_VAR_WITH_ANOTHER_PASSWORD} + tls: + ca_file: "" + insecure_skip_verify: false + username: SOME_OVERRIDE_ACCOUNT_NAME +``` + +#### Pipelines File +The `/services/otel_collector/pipelines.yaml` file contains the OTEL Collector pipelines configuration +[/services/otel_collector/pipelines.yaml](https://github.com/f5devcentral/application-study-tool/blob/main/services/otel_collector/pipelines.yaml) +which is the merged contents of the default config pipelines section plus the list of receivers +(bigip scrape jobs): +```yaml +metrics/bigip: + exporters: + - otlphttp/metrics-bigip + - debug/bigip + processors: + - batch/local + receivers: + # This was inserted here because the `pipeline` for this device + # or global `default_pipeline` was set to "metrics/bigip" + - bigip/1 +``` + +2. The second contains the OTEL Collector receivers configuration +[/services/otel_collector/receivers.yaml](https://github.com/f5devcentral/application-study-tool/blob/main/services/otel_collector/receivers.yaml) +which is the merged contents of the default config settings and the per-device settings: + +```yaml +bigip/1: + collection_interval: 30s + data_types: + f5.dns: + enabled: false + f5.gtm: + enabled: false + endpoint: https://10.0.0.1 + password: ${SOME_OTHER_ENV_VAR_WITH_ANOTHER_PASSWORD} + username: SOME_OVERRIDE_ACCOUNT_NAME + tls: + ca_file: "" + insecure_skip_verify: false +``` + +## Restart The AST Containers +Whenever the AST Configuration Files are updated, the containers need to be restarted for the updates +to take effect. This can be accomplished in a few ways, but the simplest is typically: + +(From the project root directory) +```bash +$ docker-compose down +$ docker-compose up -d +``` \ No newline at end of file diff --git a/pages/config/config_helper/config_receivers.md b/pages/config/config_helper/config_receivers.md new file mode 100644 index 0000000..a00fbe7 --- /dev/null +++ b/pages/config/config_helper/config_receivers.md @@ -0,0 +1,49 @@ +--- +layout: page +title: Device Configuration +parent: Configuration Helper (Recommended) +grandparent: Configuration +nav_order: 2 +--- + +## Configure Per-Device (BigIP) Settings + +The [/config/bigip_receivers.yaml](https://github.com/f5devcentral/application-study-tool/blob/main/config/bigip_receivers.yaml) +file contains settings that are applied to each "BigIP Receiver" (the part that collects data from a +device) within the OTEL Collector config. + +{: .important } +The config helper script must run after any changes to the default or device specific configs, +and the otel container restarted before changes will take effect. + +The settings in this file are merged with the defaults outlined +in [Configuration > Configuration Helper (Recommended) > Default Configuration]({{site.baseurl}}/config/config_helper/config_defaults.html), with the more specific settings in this file taking precedence. + +The following constraints apply: +* There must be a top level entry for each BigIP Device you wish to monitor. +* The top level key must start with `bigip/`, and be unique (recommend hostname or just incrementing int) +* Each entry must have at minimum the `endpoint` entry containing the url of the device to scrape. + +```yaml +# This file contains the list of BigIP receivers (scrape jobs). +# Each item must have a unique key (e.g. bigip/1, bigip/2, etc). +# Values not explicitly configured here inherit values in ast_defaults.yaml. +bigip/1: + # Endpoint must be specified for each device because there's no rational default. + endpoint: https://10.0.0.1 + ## Pipeline is used to tell the config_helper script which pipeline to attach it to. + ## Most users shouldn't configure this (and it will inherit from the value in ast_defaults.yaml) + # pipeline: metrics/bigip + ## Anything below here could be uncommented to override the default value + # collection_interval: 30s + username: SOME_OVERRIDE_ACCOUNT_NAME + password: "${SOME_OTHER_ENV_VAR_WITH_ANOTHER_PASSWORD}" + # data_types: + # f5.dns: + # enabled: false + # f5.gtm: + # enabled: false + # tls: + # insecure_skip_verify: false + # ca_file: +``` diff --git a/pages/config/config_helper/config_tls.md b/pages/config/config_helper/config_tls.md new file mode 100644 index 0000000..54edf59 --- /dev/null +++ b/pages/config/config_helper/config_tls.md @@ -0,0 +1,35 @@ +--- +layout: page +title: Configure TLS +parent: Configuration Helper (Recommended) +grandparent: Configuration +nav_order: 5 +--- + +### Configure CA File +AST expects a valid TLS cert bundle unless `tls.insecure_skip_verify` is +set to true for each device. In order to mount and use your CA file, you must +configure the docker-compose.yaml file in the root directory, and set the `ca_file` parameter to +the resulting path. Example: + +docker-compose.yaml: +```yaml + otel-collector: + volumes: + - ./services/otel_collector:/etc/otel-collector-config + - ./config/ca_bundle.pem:/etc/ssl/ca_bundle.pem +``` + +config/ast_defaults.yaml (or the tls section of each device in config/bigip_receivers.yaml): +```yaml +bigip_receiver_defaults: + tls: + insecure_skip_verify: false + ca_file: "/etc/ssl/ca_bundle.pem" +``` + +The configuration parameter `tls.insecure_skip_verify` defaults to false. Installers +who would like to opt-in to run in an insecure TLS mode must set +`tls.insecure_skip_verify: true` and understand +that the connection between the OTEL collector and the BIG-IP does not have secure +TLS termination. diff --git a/pages/metric_obfuscation.md b/pages/config/config_helper/metric_obfuscation.md similarity index 88% rename from pages/metric_obfuscation.md rename to pages/config/config_helper/metric_obfuscation.md index a449d4f..567c459 100644 --- a/pages/metric_obfuscation.md +++ b/pages/config/config_helper/metric_obfuscation.md @@ -1,8 +1,20 @@ +--- +layout: page +title: Metric Obfuscation +parent: Configuration Helper (Recommended) +grandparent: Configuration +nav_order: 4 +--- # Metric Obfuscation Metric data can be obfuscated before sending to storage systems (local Prometheus, F5 Datafabric, or your Org metrics store) using the built-in functionality in the Opentelemetry Collector processors. + +{: .important } +The config helper script must run after any changes to the default or device specific configs, +and the Otel container restarted before changes will take effect. + The Application Study Tool Opentelemetry Collector includes the following data processors which can be used to manipulate data before it's exported: diff --git a/pages/config_migration.md b/pages/config/config_migration.md similarity index 96% rename from pages/config_migration.md rename to pages/config/config_migration.md index 2d8bc2b..f979d4a 100644 --- a/pages/config_migration.md +++ b/pages/config/config_migration.md @@ -1,8 +1,17 @@ +--- +layout: page +title: Config Migration (pre v0.6.0) +parent: Configuration +nav_order: 1 +--- # AST Config Migration for Pre v0.6.0 Deployments +1. TOC +{:toc} + ## Background -The configuration mangagement for AST Otel Collector is being updated to allow for more flexible -configuration, and to simplify configuration for advanced usecases. +The configuration management for AST Otel Collector is being updated to allow for more flexible +configuration, and to simplify configuration for advanced use-cases. > Before you start, make sure to backup the /config/big-ips.json file! diff --git a/pages/config/manual_config/default_otel.md b/pages/config/manual_config/default_otel.md new file mode 100644 index 0000000..bb7af6b --- /dev/null +++ b/pages/config/manual_config/default_otel.md @@ -0,0 +1,59 @@ +--- +layout: page +title: Std Otel Collector Config +parent: Manual Configuration +grandparent: Configuration +nav_order: 2 +--- + +## Manual Configuration With Default Otel Collector Settings File + +1. TOC +{:toc} + +The Otel Collector container points to a single configuration file that is loaded at startup. +This config file, by default, loads a list of receivers and pipelines from elsewhere in the directory +structure. + +The other configuration managent options modify these other files, but the primary config +can also be managed however you see fit. + + +{: .important } +The OTEL Collector Container must be restarted before changes to these files will take effect. + +The main otel collector config file is located in `/services/otel_collector/defaults/bigip-scraper-config.yaml` + +Syntax for these files is the [Opentelemetry Collector Configuration](https://opentelemetry.io/docs/collector/configuration/) syntax. + +```yaml +receivers: ${file:/etc/otel-collector-config/receivers.yaml} + +processors: + batch/local: + +exporters: + otlphttp/metrics-bigip: + endpoint: http://prometheus:9090/api/v1/otlp + debug/bigip: + verbosity: basic + sampling_initial: 5 + sampling_thereafter: 200 + +service: + pipelines: ${file:/etc/otel-collector-config/pipelines.yaml} + +``` + +Any of these files can be modified directly to update settings as desired (e.g. adding additional +logging levels). + +## Restart The AST Containers +Whenever the AST Configuration Files are updated, the containers need to be restarted for the updates +to take effect. This can be accomplished in a few ways, but the simplest is typically: + +(From the project root directory) +```bash +$ docker-compose down +$ docker-compose up -d +``` \ No newline at end of file diff --git a/pages/config/manual_config/manual_config.md b/pages/config/manual_config/manual_config.md new file mode 100644 index 0000000..3188b9e --- /dev/null +++ b/pages/config/manual_config/manual_config.md @@ -0,0 +1,12 @@ +--- +layout: page +title: Manual Configuration +parent: Configuration +nav_order: 3 +has_children: true +permalink: /config/manual_config +--- + +Manual configuration of the Opentelemetry Collector settings files may be desirable in some cases, +such as very small installations or where other automation is in use. This section describes 2 methods +for handling configurations manually (without the python script). \ No newline at end of file diff --git a/pages/config/manual_config/receivers_pipelines.md b/pages/config/manual_config/receivers_pipelines.md new file mode 100644 index 0000000..d0612d2 --- /dev/null +++ b/pages/config/manual_config/receivers_pipelines.md @@ -0,0 +1,99 @@ +--- +layout: page +title: Receiver/Pipeline Config +parent: Manual Configuration +grandparent: Configuration +nav_order: 1 +--- + +## Manual Configuration With The Receivers and Pipelines Files + + +1. TOC +{:toc} + +As described in the Config Helper section, the output of the helper is 2 yaml files that are used by +the Otel Collector at run time. + +* `receivers.yaml` - The final list of scraper configs and their settings. +* `pipelines.yaml` - The final pipeline configs that map receievers to output destinations +(prometheus). + + +{: .important } +The OTEL Collector Container must be restarted before changes to these files will take effect. + +These files can be modified directly by users without the config helper script without any additional +changes to the AST docker-compose or other settings. + +Syntax for these files is the [Opentelemetry Collector Configuration](https://opentelemetry.io/docs/collector/configuration/) syntax. + +#### Receivers File +The `/services/otel_collector/receivers.yaml` file contains the OTEL Collector receivers configuration: +```yaml +bigip/1: + collection_interval: 30s + data_types: + f5.dns: + enabled: false + f5.gtm: + enabled: false + endpoint: https://10.0.0.1 + password: ${env:SOME_OTHER_ENV_VAR_WITH_ANOTHER_PASSWORD} + tls: + ca_file: "" + insecure_skip_verify: false + username: SOME_OVERRIDE_ACCOUNT_NAME +``` + +#### Pipelines File +The `/services/otel_collector/pipelines.yaml` file contains the OTEL Collector pipelines configuration +[/services/otel_collector/pipelines.yaml](https://github.com/f5devcentral/application-study-tool/blob/main/services/otel_collector/pipelines.yaml): +```yaml +metrics/bigip: + exporters: + - otlphttp/metrics-bigip + - debug/bigip + processors: + - batch/local + receivers: + - bigip/1 +``` + + +These are mapped into the final OTEL Collector config via the "file" directives in the +receivers and services.pipelines section of the +[/services/otel_collector/defaults/bigip-scraper-config.yaml](https://github.com/f5devcentral/application-study-tool/blob/main/services/otel_collector/defaults/bigip-scraper-config.yaml) file: + +``` +receivers: ${file:/etc/otel-collector-config/receivers.yaml} + +processors: + batch/local: + +exporters: + otlphttp/metrics-bigip: + endpoint: http://prometheus:9090/api/v1/otlp + debug/bigip: + verbosity: basic + sampling_initial: 5 + sampling_thereafter: 200 + +service: + pipelines: ${file:/etc/otel-collector-config/pipelines.yaml} + +``` + +Any of these files can be modified directly to update settings as desired (e.g. adding additional +logging levels). + + +## Restart The AST Containers +Whenever the AST Configuration Files are updated, the containers need to be restarted for the updates +to take effect. This can be accomplished in a few ways, but the simplest is typically: + +(From the project root directory) +```bash +$ docker-compose down +$ docker-compose up -d +``` \ No newline at end of file diff --git a/pages/config_management.md b/pages/config_management.md deleted file mode 100644 index e1d1e92..0000000 --- a/pages/config_management.md +++ /dev/null @@ -1,214 +0,0 @@ -# AST Configuration Management Overview - -## Config Management Options -In the post v0.6.0 management scheme, users can choose from one of the below options to manage -the AST Otel Collector configs: - -1. Using the [/src/config_helper.py](https://github.com/f5devcentral/application-study-tool/blob/main/src/config_helper.py) script to generate full Otel Collector -Config files from a small set of configuration for each device plus a set of defaults (recommended for -most users, and includes migration path from old big-ips.json configs) - -2. Manual maintenance of the Otel Collector Config files in -[/services/otel_collector/pipelines.yaml](https://github.com/f5devcentral/application-study-tool/blob/main/services/otel_collector/pipelines.yaml) -and [/services/otel_collector/receivers.yaml](https://github.com/f5devcentral/application-study-tool/blob/main/services/otel_collector/receivers.yaml) - -3. Manual maintenance of the Otel Collector Config files in -[/services/otel_collector/defaults](https://github.com/f5devcentral/application-study-tool/blob/main/services/otel_collector/defaults) - -### Using config_helper.py (Recommended For Most Users) -With the included python script in [/src/config_helper.py](https://github.com/f5devcentral/application-study-tool/blob/main/src/config_helper.py), AST collector -configuration is managed through 2 primary files: - -1. A default file which contains settings that should be applied for each BigIP unless overriden (see below) -in [/config/ast_defaults.yaml](https://github.com/f5devcentral/application-study-tool/blob/main/config/ast_defaults.yaml): - -```yaml -# These configs are applied to each entry in the bigip_receivers file -# where they don't contain an equivalent / overriding entry. -bigip_receiver_defaults: - # The time to wait between metric collection runs - collection_interval: 60s - # The username to login to the device with - username: admin - # The password (not recommended) or a reference to an env variable (recommended) - # Below tells the collector to look for an environment variable named BIGIP_PASSWORD_1 - password: "${env:BIGIP_PASSWORD_1}" - # The data_types that should be enabled or disabled. DNS and GTM users can enable those modules - # by setting the below to true. These will apply to all devices and may be better specified on the - # per-reciever settings file below. - data_types: - f5.dns: - enabled: false - f5.gtm: - enabled: false - # The TLS settings to use. Either a CA file must be specified or insecure_skip_verify - # set to true (not recommended) - tls: - insecure_skip_verify: false - ca_file: "" - - -# Set to true to enable periodic metric export to F5 DataFabric. -# Requires adding your Sensor ID and secret token to the container environment (see .env-example). -# Contact your F5 sales rep to obtain the ID / secret token. -f5_data_export: false - - -# Most people should not need to modify settings below this line - -# The default local pipeline to use if one isn't specified in the per-device configs. -pipeline_default: metrics/local -# The default pipeline to use if metric export to F5 is enabled (if f5_data_export.sensor_id field above is set) -f5_pipeline_default: metrics/f5-datafabric - -pipelines: - - # These pipeline configs are written to the OTEL config after having the configured receivers - # added to the dictionary in accordance with the "pipeline_default" field above and "pipeline" - # field on the per-receiver config file. Otel Collector documentation explains the syntax in more - # detail. - metrics/local: - #receivers list are generated via the config helper script - processors: [batch/local] - exporters: [otlphttp/metrics-local, debug/bigip] - - # These pipeline configs are written to the OTEL config after having the configured receivers - # added to the dictionary in accordance with the "f5_pipeline_default" field above and "f5_pipeline" - # field on the per-receiver config file. Otel Collector documentation explains the syntax in more - # detail. - metrics/f5-datafabric: - #receivers list are generated via the config helper script - processors: [interval/f5-datafabric, attributes/f5-datafabric, batch/f5-datafabric] - exporters: [otlp/f5-datafabric, debug/bigip] -``` - -2. A file which contains settings that should override (or have no default) at the individual bigip level -in [/config/bigip_receivers.yaml](https://github.com/f5devcentral/application-study-tool/blob/main/config/bigip_receivers.yaml): -```yaml -# This file contains the list of BigIP receivers (scrape jobs). -# Each item must have a unique key (e.g. bigip/1, bigip/2, etc). -# Values not explicitly configured here inherit values in ast_defaults.yaml. -bigip/1: - # Endpoint must be specified for each device because there's no rational default. - endpoint: https://10.0.0.1 - ## Pipeline is used to tell the config_helper script which pipeline to attach it to. - ## Most users shouldn't configure this (and it will inherit from the value in ast_defaults.yaml) - # pipeline: metrics/bigip - ## Anything below here could be uncommented to override the default value - # collection_interval: 30s - username: SOME_OVERRIDE_ACCOUNT_NAME - password: "${SOME_OTHER_ENV_VAR_WITH_ANOTHER_PASSWORD}" - # data_types: - # f5.dns: - # enabled: false - # f5.gtm: - # enabled: false - # tls: - # insecure_skip_verify: false - # ca_file: -``` - -When the config_helper script is run with the --generate-configs option, 2 new files are written out -to the [/services/otel_collector](https://github.com/f5devcentral/application-study-tool/blob/main/services/otel_collector) directory: - -1. The first contains the OTEL Collector pipelines configuration [/services/otel_collector/pipelines.yaml](https://github.com/f5devcentral/application-study-tool/blob/main/services/otel_collector/pipelines.yaml) which is basically the contents of the default config pipelines section plus the list of receivers (bigip scrape jobs): -```yaml -metrics/bigip: - exporters: - - otlphttp/metrics-bigip - - debug/bigip - processors: - - batch/local - receivers: - # This was inserted here because the pipeline / default_pipeline for this device was - # set to "metrics/bigip" - - bigip/1 -``` - -2. The second contains the OTEL Collector receivers configuration -[/services/otel_collector/receivers.yaml](https://github.com/f5devcentral/application-study-tool/blob/main/services/otel_collector/receivers.yaml) -which is the merged contents of the default config settings and the per-device settings: -```yaml -bigip/1: - collection_interval: 30s - data_types: - f5.dns: - enabled: false - f5.gtm: - enabled: false - endpoint: https://10.0.0.1 - password: ${SOME_OTHER_ENV_VAR_WITH_ANOTHER_PASSWORD} - tls: - ca_file: "" - insecure_skip_verify: false - username: SOME_OVERRIDE_ACCOUNT_NAME -``` - -When the OTEL container is run, the default configs in -[/services/otel_collector/defaults/](https://github.com/f5devcentral/application-study-tool/blob/main/services/otel_collector/defaults/) merge these files into -the final configuration the OTEL Collector needs to run correctly. - -#### Run The Configuration Helper -The config helper script can be run natively or via docker from the project root directory -to merge the default and device level configs into the final OTEL Collector config as follows: -```shell -# Run the configuration generator from the project root directory -docker run --rm -it -w /app -v ${PWD}:/app --entrypoint /app/src/bin/init_entrypoint.sh python:3.12.6-slim-bookworm --generate-config -``` - -This will write 2 new files in the services/otel_collector directory: - -* `receivers.yaml` - The final list of scraper configs and their settings. -* `pipelines.yaml` - The final pipeline configs that map receievers to output destinations -(prometheus). - -### Manual Maintenance Of Receiver and Pipeline Files -The files mentioned above can be managed directly by users if they want to skip the config_helper -script in favor of their own automation / templating. In this case, you just need to update the files: - -* [/services/otel_collector/receivers.yaml](https://github.com/f5devcentral/application-study-tool/blob/main/services/otel_collector/receivers.yaml) -* [/services/otel_collector/pipelines.yaml](https://github.com/f5devcentral/application-study-tool/blob/main/services/otel_collector/pipelines.yaml) - -These are mapped into the final OTEL Collector config via the "file" directives in the -receivers and services.pipelines section of the -[/services/otel_collector/defaults/bigip-scraper-config.yaml](https://github.com/f5devcentral/application-study-tool/blob/main/services/otel_collector/defaults/bigip-scraper-config.yaml) file: - -``` -receivers: ${file:/etc/otel-collector-config/receivers.yaml} - -processors: - batch/local: - -exporters: - otlphttp/metrics-bigip: - endpoint: http://prometheus:9090/api/v1/otlp - debug/bigip: - verbosity: basic - sampling_initial: 5 - sampling_thereafter: 200 - -service: - pipelines: ${file:/etc/otel-collector-config/pipelines.yaml} - -``` - -Any of these files can be modified directly to update settings as desired (e.g. adding additional -logging levels). - - -### Manual Maintenance Of The OTEL Collector Config -You can also forgo all of the above config structure in favor of your own management scheme. If you're -running with the base docker-compose file, you may need to modify the commands section to point at your -own config files: - -```yaml - otel-collector: - ... - # Update these as needed - volumes: - - ./services/otel_collector:/etc/otel-collector-config - # Update these as needed - command: - - "--config=/etc/otel-collector-config/defaults/bigip-scraper-config.yaml" - ... -``` \ No newline at end of file diff --git a/pages/datafabric/configuration.md b/pages/datafabric/configuration.md new file mode 100644 index 0000000..10e0b82 --- /dev/null +++ b/pages/datafabric/configuration.md @@ -0,0 +1,8 @@ +--- +layout: page +title: Export Configuration +parent: F5 Datafabric +nav_order: 1 +--- + +See [>Configuration > Configuration Helper (Recommended) > Configure Export To F5 Datafabric]({{site.baseurl}}/config/config_helper/config_datafabric.html) \ No newline at end of file diff --git a/pages/datafabric/datafabric.md b/pages/datafabric/datafabric.md new file mode 100644 index 0000000..1e41d6d --- /dev/null +++ b/pages/datafabric/datafabric.md @@ -0,0 +1,19 @@ +--- +layout: page +title: F5 Datafabric +nav_order: 4 +has_children: true +permalink: /datafabric +--- + +### F5 Data Fabric + +The **Application Study Tool (AST)** seamlessly integrates with the +[**F5 Data Fabric**](https://www.f5.com/company/blog/f5-unlocking-the-power-of-ai), a cutting-edge +data platform designed to harness the power of AI for enhanced application and API security. This +integration allows the AST to send collected metrics and insights to the F5 Data Fabric, enabling +advanced analytics and machine learning capabilities for a wide variety of use cases, including +more effective threat detection and response. + +By leveraging the capabilities of the F5 Data Fabric, the Application Study Tool empowers organizations +to stay ahead of emerging threats and capitalize on the potential of AI-driven security innovations. \ No newline at end of file diff --git a/pages/quickstart.md b/pages/getting_started.md similarity index 88% rename from pages/quickstart.md rename to pages/getting_started.md index 7c5574b..17e40a1 100644 --- a/pages/quickstart.md +++ b/pages/getting_started.md @@ -1,10 +1,29 @@ -# Quick Start +--- +layout: page +title: Getting Started +nav_order: 2 +--- +# Getting Started -## Getting Started +1. TOC +{:toc} + +## Background +This instructions in this file will get a new installation up and running in as little as a few minutes. +For more detailed information on AST config management options, see the +[Configuration Management Overview]({{site.baseurl}}/config/) and related sections. ### Prerequisites -docker (or compatible) - [Installation Instructions](https://docs.docker.com/engine/install/) +[Git Client](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) + +Docker (or compatible) container environment with compose. + +Installation Instructions: + * [General (docker engine)](https://docs.docker.com/engine/install/) + * [Ubuntu (docker engine)](https://docs.docker.com/engine/install/ubuntu/) + * [RHEL (docker engine)](https://docs.docker.com/engine/install/rhel/) + * [Podman](https://podman.io/docs/installation) ### Installation @@ -146,6 +165,8 @@ The config helper script can be run natively or via docker from the project root to merge the default and device level configs into the final OTEL Collector config as follows: ```shell # Run the configuration generator from the project root directory +# If `echo $PWD` doesn't give you the current directory on your system, +# replace the '-v ${PWD}' section with '-v /path/to/your/directory' docker run --rm -it -w /app -v ${PWD}:/app --entrypoint /app/src/bin/init_entrypoint.sh python:3.12.6-slim-bookworm --generate-config ``` diff --git a/pages/index.md b/pages/index.md index 57f13bf..3b3056e 100644 --- a/pages/index.md +++ b/pages/index.md @@ -1,14 +1,15 @@ --- layout: home +title: Home +nav_order: 1 --- - -# Application Study Took +# Application Study Tool ### Overview The Application Study Tool is intended to provide enhanced insights into (classic) BIG-IP products, leveraging best in class open source telemetry tools. The full installation includes: -* Custom Instance of OpenTelemetry Collector with enhanced BIG-IP data receivers (data fetched via iControlRest) [Receiver Metrics Info](./receiver_metrics.md). +* Custom Instance of OpenTelemetry Collector with enhanced BIG-IP data receivers (data fetched via iControlRest) [Receiver Metrics Info]({{site.baseurl}}/components/otel_collector/receiver_metrics.html). * Prometheus timeseries database for storing and querying collected data. * Grafana Instance with pre-configured dashboards for quick insights at the device and "fleet" levels. @@ -19,35 +20,4 @@ the Openetlemetry Collector can be configured to send data to existing productio ![](./assets/ui.gif) -### Table of Contents - -- [Quick Start](./quickstart.md) - - [Installation](./quickstart.md#installation) - - [Configuration](./quickstart.md#configure-default-device-settings) -- [Dashboards](./dashboard.md) - - [Fleet](./dashboard.md#fleet-dashboards) - - [Inventory](./dashboard.md#inventory) - - [Device Utilization](./dashboard.md#device-utilization) - - [Virtual Servers](./dashboard.md#virtual-servers) - - [SSL Certs](./dashboard.md#ssl-certs) - - [Device](./dashboard.md#device-dashboards) - - [Overview](./dashboard.md#overview) - - [Top N](./dashboard.md#top-n) - - [Pools](./dashboard.md#pools) - - [Virtual Servers](./dashboard.md#virtual-servers-1) - - [iRules](./dashboard.md#irules) - - [WAF](.dashboard.) - - [Profile](./dashboard.md#profile-dashboards) - - [HTTP](./dashboard.md#http) - - [Stats](./dashboard.md#collector-stats) -- [Config Management Overview](./config_management.md) - - [Recommended Method](./config_management.md#using-config_helperpy-recommended-for-most-users) - - [Manual Maintenance](./config_management.md#manual-maintenance-of-receiver-and-pipeline-files) -- [Config Migration (From v0.5.0 and Earlier)](./config_migration.md) -- [Receiver Metrics Info](./receiver_metrics.md) -- [Receiver Readme](./receiver_readme.md) -- [Metric Obfuscation](./metric_obfuscation.md) - - [Example](./metric_obfuscation.md#masking-attributes-example) - - \ No newline at end of file diff --git a/pages/troubleshooting/troubleshooting.md b/pages/troubleshooting/troubleshooting.md new file mode 100644 index 0000000..5ba77b6 --- /dev/null +++ b/pages/troubleshooting/troubleshooting.md @@ -0,0 +1,141 @@ +--- +layout: page +title: Troubleshooting +nav_order: 4 +has_children: true +permalink: /troubleshooting +--- + +# Troubleshooting AST + +1. TOC +{:toc} + +## Useful Commands + +### View Docker Container Status +You can view the state of the docker containers after they've been started with `docker ps`. +The STATUS field is a good indicator whether containers are running correctly or not. + +This output shows everything looks good: +```shell +$ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +cb4cf8867390 grafana/grafana:11.2.0 "/run.sh" About a minute ago Up 49 seconds 0.0.0.0:3000->3000/tcp grafana +bb8891f2cd47 prom/prometheus:v2.54.1 "/bin/prometheus --c…" About a minute ago Up 49 seconds 0.0.0.0:9090->9090/tcp prometheus +df2739cd67cb ghcr.io/f5devcentral/application-study-tool/otel_custom_collector:v0.6.0 "/otelcol-custom --c…" About a minute ago Up 49 seconds 4317/tcp, 55679-55680/tcp application-study-tool-otel-collector-1 +``` + +This output shows a problem (Restarting container) for the `application-study-tool-otel-collector-1` +container (the otel collector): +```shell +$ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +fdbde8a3ee16 ghcr.io/f5devcentral/application-study-tool/otel_custom_collector:v0.6.0 "/otelcol-custom --c…" 14 seconds ago Restarting (1) 5 seconds ago application-study-tool-otel-collector-1 +b7ef41accd46 grafana/grafana:11.2.0 "/run.sh" 14 seconds ago Up 13 seconds 0.0.0.0:3000->3000/tcp grafana +8edff3e8666e prom/prometheus:v2.54.1 "/bin/prometheus --c…" 14 seconds ago Up 13 seconds 0.0.0.0:9090->9090/tcp prometheus +``` + +### View Docker Container Logs +When containers aren't running correctly or other issues are present, the container logs are a good place +to look. Once you have the Container ID or Container Name field from the output of `docker ps`, you can +view the container logs using the `docker logs ` command. + +Logs for the broken container above (which show a parsing error with the config file) +can be gathered as shown: + +```shell +$ docker logs application-study-tool-otel-collector-1 +Error: failed to get config: cannot unmarshal the configuration: decoding failed due to the following error(s): + +'receivers' expected a map, got 'string' +2024/10/17 15:53:44 collector server run finished with error: failed to get config: cannot unmarshal the configuration: decoding failed due to the following error(s): + +'receivers' expected a map, got 'string' +``` + +### Save Logs To A File +The output of `docker logs` command for specific containers may be requested during troubleshooting. +The logs can be gathered and saved to file with a command similar to: + +```shell +$ docker logs application-study-tool-otel-collector-1 >& otellogs.txt && gzip otellogs.txt +``` + +### Stop and Start A Container +Individual containers can be started and stopped using the `docker stop ` command. +This will cause the container to restart with any modifications to config files picked up (alternative +to the `docker-compose down` and `docker-compose up` commands which restart all containers): + +```shell +$ docker stop fdbde8a3ee16 +fdbde8a3ee16 + +$ docker start fdbde8a3ee16 +fdbde8a3ee16 +``` + +## Grafana Not Reachable +Once containers are up and running, you should be able to view the Grafana instance by navigating to port +`3000` on the host machine. If this isn't working, things to check include: + +* Run `docker ps` and check the status of the Grafana container. +* Ensure port `3000` is open between the client browser and the instance running AST. +* Check grafana logs with `docker logs -f grafana` +* Check if the prometheus endpoint (avaialble on port `9090` of the AST instance) is reachable (as a + second data point assuming none of the above show issues). + + +## Dashboard Data Not Visible (All Dashboards) +If Grafana is loading but dashboard data is not populating in any dashboards, there are a few things to +check: + +### Check the Opentelemetry Collector Dashboard +The opentelemetry collector dashboard at the top level of the 'Dashboards' list in Grafana is a good +starting point. The data in this dashboard is collected through a different mechanism from the rest of +the dashboards. + +* If data is present in this dashboard (but no others), it indicates an issue with the collection of data +from the BigIPs. +* If no data is present in this dashboard, it indicates a problem with the connection between Prometheus +and Grafana. + +**If Data Is Present** + +If data is present here, there's very likely an issue with collection from the BigIPs themselves. +Things to check: + +* Is the Otel Collector container running correctly (`docker ps`) +* Do the otel collector container logs show any + 1. Authentication errors: + * `2024-10-17T16:23:50.658Z error scraperhelper/scrapercontroller.go:197 Error scraping metrics {"kind": "receiver", "name": "bigip/1", "data_type": "metrics", "error": "endpoint /mgmt/shared/authn/login returned code 401", "scraper": "bigip"}` + 2. Network connection problems: + * `2024-10-17T16:17:44.347Z error scraperhelper/scrapercontroller.go:197 Error scraping metrics {"kind": "receiver", "name": "bigip/1", "data_type": "metrics", "error": "failed to make http request: Post \"https://10.0.0.1/mgmt/shared/authn/login\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)", "scraper": "bigip"}` +* Do the `Metrics Points Receive Rate` and `Metrics Points Export Rate` in the dashboard show failures +or '0' (indicating nothing is being collected)? + +**If Data Is NOT Present** + +This indicates that there is an issue with the communication between Grafana and Prometheus. +Check: +* Is the prometheus container running `docker ps` +* Are there any error logs present in the prometheus container (if running or not) `docker logs prometheus` +* Can you reach the prometheus interface on port `9090` of the AST host (assuming firewalls are open) + +## Dashboard Data Not Visible (Specific Dashboards) + +If the dashboards are GTM or DNS profile dashboards, see below. + +For other dashboards, ensure that: +* The dashboard 'time range' in the upper right is set to a window where data is expected to exist. +* Any device, virtual server, or other selectors in the upper left are set to valid values where data +should exist. +* Check the responses for a specific broken panel by clicking the '3 dot icon' (upper right while +hovering) and selecting 'Inspect > Query' +* Are any queries to the BigIP timing or erroring out? Check the otel collector logs (`docker ps`) and +the 'BigIP Collector Stats' dashboard at the top level of the Dashboards section in Grafana + +## GTM and DNS Metrics Not Loading + +Metrics for DNS and GTM are disabled by default. See +[Configuration > Configuration Helper (Recommended) > Configure DNS & GTM]({{site.baseurl}}/config/config_helper/config_dns_gtm.html) for instructions to enable. \ No newline at end of file From dc7bac3b1d42d53ced880a7f7b16c1c4bfb407b5 Mon Sep 17 00:00:00 2001 From: Chris Hain Date: Thu, 17 Oct 2024 10:14:03 -0700 Subject: [PATCH 05/10] switch ordering on pictures / descriptions --- pages/components/grafana/dashboard.md | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/pages/components/grafana/dashboard.md b/pages/components/grafana/dashboard.md index 23b6e83..b98366a 100644 --- a/pages/components/grafana/dashboard.md +++ b/pages/components/grafana/dashboard.md @@ -30,49 +30,48 @@ You can update the Grafana user/pass by changing the values of the `GF_SECURITY_ The fleet dashboards provide top level information of your environment #### Inventory -![](../../assets/BigIP-Fleet-Inventory.png) *The inventory dashboard allows you to view the differences between BigIP devices in your inventory.* +![](../../assets/BigIP-Fleet-Inventory.png) #### Device Utilization -![](../../assets/BigIP-Fleet-Device-Utilization.png) *This dashboard provides a quick view on how much usage your box is receiving. To view more information about the device take a look at the [device overview dashboard](#overview).* +![](../../assets/BigIP-Fleet-Device-Utilization.png) #### Virtual Servers -![](../../assets/BigIP-Fleet-Virtual-Server.png) *The Virtual Server dashboard provides an overview of all the virtual servers present in your inventory.* +![](../../assets/BigIP-Fleet-Virtual-Server.png) #### SSL Certs -![](../../assets/BigIP-Fleet-SSL-Certs.png) *The SSL Certs view allows you to stay up to date on the status of your SSL certficates.* +![](../../assets/BigIP-Fleet-SSL-Certs.png) ### Device Dashboards The device dashboards provide more grainular information about how the device is configured and how it is performing #### Overview -![](../../assets/BipIP-Device-Overview.png) *This dashboard provides variety of metrics about the performance of the device.* +![](../../assets/BipIP-Device-Overview.png) #### Top N -![](../../assets/BigIP-Device-TopN.png) *The Top N dashboard provides statistical highlights by presenting stats in a Top N, by default N is set to 10 this can be changed at top of the dashboard* +![](../../assets/BigIP-Device-TopN.png) #### Pools -![](../../assets/BigIP-Device-Pools.png) *Here you can view stats pertaining to Pools and Pool members on a BigIP.* +![](../../assets/BigIP-Device-Pools.png) #### Virtual Servers -![](../../assets/BigIP-Device-Virtual-Server.png) *Here you can view stats pertaining to all the virtual servers on a BigIP.* +![](../../assets/BigIP-Device-Virtual-Server.png) #### iRules -![](../../assets/BigIP-Device-iRules.png) *This dashboard allows you to monitor iRule usage and performance.* +![](../../assets/BigIP-Device-iRules.png) #### WAF - -![](../../assets/BigIP-Device-WAF.png) *View the status of your WAF policies here.* +![](../../assets/BigIP-Device-WAF.png) @@ -82,8 +81,8 @@ The profile dashboards dive into the different profiles configured on the device #### HTTP -![](../../assets/BigIP-Device:Profile-HTTP.png) *This dashboard enables the user to monitor their HTTP profiles, view information pretaining to HTTP requests and responses* +![](../../assets/BigIP-Device:Profile-HTTP.png) ### Collector Stats From 8bffed73d31810c99b5ccdee60b86b648272ced4 Mon Sep 17 00:00:00 2001 From: Chris Hain Date: Thu, 17 Oct 2024 16:24:02 -0700 Subject: [PATCH 06/10] add virtual server drilldown. Update device overview with license and peer device state info. --- .../bigip/device/device-overview.json | 228 +- .../bigip/device/virtual-server-detail.json | 3454 +++++++++++++++++ 2 files changed, 3631 insertions(+), 51 deletions(-) create mode 100644 services/grafana/provisioning/dashboards/bigip/device/virtual-server-detail.json diff --git a/services/grafana/provisioning/dashboards/bigip/device/device-overview.json b/services/grafana/provisioning/dashboards/bigip/device/device-overview.json index 19ba957..3a2efdf 100644 --- a/services/grafana/provisioning/dashboards/bigip/device/device-overview.json +++ b/services/grafana/provisioning/dashboards/bigip/device/device-overview.json @@ -208,7 +208,7 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 9, "w": 6, "x": 3, "y": 1 @@ -297,6 +297,7 @@ }, { "datasource": { + "default": false, "type": "prometheus", "uid": "${datasource}" }, @@ -312,25 +313,52 @@ }, "inspect": false }, - "mappings": [], + "mappings": [ + { + "options": { + "licensed": { + "color": "green", + "index": 0, + "text": "Licensed" + }, + "unlicensed": { + "color": "#808080", + "index": 1, + "text": "Unlicensed" + } + }, + "type": "value" + } + ], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "#939393", "value": null - }, - { - "color": "red", - "value": 80 } ] } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "f5_module_license_state" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] }, "gridPos": { - "h": 6, + "h": 9, "w": 6, "x": 9, "y": 1 @@ -346,7 +374,13 @@ ], "show": false }, - "showHeader": false + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "License State" + } + ] }, "pluginVersion": "11.2.0", "targets": [ @@ -357,7 +391,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "f5_system_info{job=\"$device_name\"}", + "expr": "f5_module_license_info_ratio", "format": "table", "instant": true, "legendFormat": "__auto", @@ -365,7 +399,7 @@ "refId": "A" } ], - "title": "Platform", + "title": "License State", "transformations": [ { "id": "organize", @@ -377,10 +411,13 @@ "baseMac": true, "chassisId": false, "chassisType": false, + "dataType": true, "f5_device_management_ip": true, "f5_device_name": true, "f5_endpoint": true, "failover_state": true, + "instance": true, + "job": true, "marketingName": false, "partition": true, "platformId": false, @@ -390,31 +427,18 @@ "includeByName": {}, "indexByName": { "Time": 0, - "Value": 14, + "Value": 2, "__name__": 1, - "baseMac": 5, - "chassisId": 8, - "chassisType": 9, - "f5_device_management_ip": 3, - "f5_device_name": 2, - "f5_endpoint": 4, - "failover_state": 10, - "marketingName": 6, - "partition": 11, - "platformId": 7, - "product": 12, - "version": 13 + "dataType": 3, + "f5_module_license_state": 5, + "f5_module_name": 4, + "instance": 6, + "job": 7 }, - "renameByName": {} - } - }, - { - "id": "reduce", - "options": { - "labelsToFields": false, - "reducers": [ - "last" - ] + "renameByName": { + "f5_module_license_state": "License State", + "f5_module_name": "Module" + } } } ], @@ -454,7 +478,7 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 9, "w": 3, "x": 15, "y": 1 @@ -528,7 +552,7 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 9, "w": 3, "x": 18, "y": 1 @@ -607,7 +631,7 @@ ] }, "gridPos": { - "h": 6, + "h": 9, "w": 3, "x": 21, "y": 1 @@ -689,6 +713,108 @@ "title": "Disk Use", "type": "piechart" }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "index": 1, + "text": "Down" + }, + "1": { + "index": 0, + "text": "Up" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "transparent", + "index": 2, + "text": "None Configured" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 4 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "f5_system_failover_peer_active_state_ratio{job=\"$device_name\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{failover_peer_name}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Failover Peer", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -720,7 +846,7 @@ "h": 3, "w": 3, "x": 0, - "y": 4 + "y": 7 }, "id": 3, "options": { @@ -770,7 +896,7 @@ "h": 1, "w": 24, "x": 0, - "y": 7 + "y": 10 }, "id": 8, "panels": [], @@ -843,7 +969,7 @@ "h": 6, "w": 7, "x": 0, - "y": 8 + "y": 11 }, "id": 9, "options": { @@ -916,7 +1042,7 @@ "h": 6, "w": 5, "x": 7, - "y": 8 + "y": 11 }, "id": 12, "options": { @@ -1016,7 +1142,7 @@ "h": 12, "w": 6, "x": 12, - "y": 8 + "y": 11 }, "id": 10, "options": { @@ -1120,7 +1246,7 @@ "h": 12, "w": 6, "x": 18, - "y": 8 + "y": 11 }, "id": 13, "options": { @@ -1246,7 +1372,7 @@ "h": 6, "w": 7, "x": 0, - "y": 14 + "y": 17 }, "id": 11, "options": { @@ -1319,7 +1445,7 @@ "h": 6, "w": 5, "x": 7, - "y": 14 + "y": 17 }, "id": 14, "options": { @@ -1380,7 +1506,7 @@ "h": 1, "w": 24, "x": 0, - "y": 20 + "y": 23 }, "id": 15, "panels": [], @@ -1528,7 +1654,7 @@ "h": 8, "w": 9, "x": 0, - "y": 21 + "y": 24 }, "id": 16, "options": { @@ -1766,7 +1892,7 @@ "h": 8, "w": 8, "x": 9, - "y": 21 + "y": 24 }, "id": 17, "options": { @@ -1885,7 +2011,7 @@ "h": 8, "w": 7, "x": 17, - "y": 21 + "y": 24 }, "id": 18, "options": { @@ -1976,6 +2102,6 @@ "timezone": "browser", "title": "Device Overview", "uid": "adpz4tutl28e8f", - "version": 4, + "version": 7, "weekStart": "" } \ No newline at end of file diff --git a/services/grafana/provisioning/dashboards/bigip/device/virtual-server-detail.json b/services/grafana/provisioning/dashboards/bigip/device/virtual-server-detail.json new file mode 100644 index 0000000..179a065 --- /dev/null +++ b/services/grafana/provisioning/dashboards/bigip/device/virtual-server-detail.json @@ -0,0 +1,3454 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.2.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Drill-Down Details for a Single Virtual Server", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 5, + "panels": [], + "title": "Summary", + "type": "row" + }, + { + "datasource": { + "name": "${datasource}", + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Pool" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdu11thaxntvkf/device-pools?orgId=1&${device_name:queryparam}&var-pool=${__data.fields[\"Pool\"]}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Virtual Server" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/ddq2jbj2v588wa/device-virtual-servers?orgId=1&${device_name:queryparam}&var-virtual_server=${__data.fields[\"Virtual Server\"]}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 16, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "f5_virtual_server_info{job=\"$device_name\", f5_virtual_server_name=~\"$virtual_server\"}==1", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "B" + } + ], + "title": "VS Overview", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Value": true, + "Value #B": true, + "__name__": true, + "__name__ 1": true, + "__name__ 2": true, + "dataType": true, + "f5_pool_name 2": true, + "f5_virtual_server_destination 2": true, + "f5_virtual_server_name 2": true, + "instance": true, + "job": true, + "job 1": true, + "job 2": true + }, + "includeByName": {}, + "indexByName": { + "Time": 3, + "Value": 8, + "__name__": 4, + "dataType": 9, + "f5_pool_name": 6, + "f5_virtual_server_destination": 2, + "f5_virtual_server_name": 1, + "f5_virtual_server_protocol": 5, + "instance": 0, + "job": 7 + }, + "renameByName": { + "Time": "", + "Time 2": "", + "Value #A": "CPU Utilization", + "f5_pool_name": "Pool", + "f5_virtual_server_destination": "Destination", + "f5_virtual_server_destination 1": "Destination", + "f5_virtual_server_name": "Virtual Server", + "f5_virtual_server_name 1": "VS Name", + "f5_virtual_server_protocol": "Protocol", + "job 1": "", + "status": "Status" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "name": "${datasource}", + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "applyToRow": true, + "type": "color-background" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "super-light-orange", + "value": 1 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Virtual Server" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/ddq2jbj2v588wa/device-virtual-servers?orgId=1&${device_name:queryparam}&var-virtual_server=${__data.fields[\"Virtual Server\"]}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 19, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Config Generation Changes [24h]" + } + ] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "changes(f5_virtual_server_generation{job=\"$device_name\", f5_virtual_server_name=~\"$virtual_server\"}[24h])", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Generation Changes [24h]", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "dataType": true, + "instance": true, + "job": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Config Generation Changes [24h]", + "f5_pool_name": "Pool", + "f5_virtual_server_name": "Virtual Server", + "instance": "", + "job": "" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 23, + "panels": [], + "title": "Availability", + "type": "row" + }, + { + "datasource": { + "name": "${datasource}", + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 9 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "avg_over_time(f5_virtual_server_availability_ratio{job=\"$device_name\", availability_state=\"available\", f5_virtual_server_name=~\"$virtual_server\"}[$__interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_virtual_server_name}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Virtual Server State isAvailable", + "type": "timeseries" + }, + { + "datasource": { + "name": "${datasource}", + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 9 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "avg_over_time(f5_virtual_server_enabled_ratio{job=\"$device_name\", f5_virtual_server_name=~\"$virtual_server\"}[$__interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_virtual_server_name}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Virtual Server isEnabled", + "type": "timeseries" + }, + { + "datasource": { + "name": "${datasource}", + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "color-text" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Offline" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unknown" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + }, + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Enabled" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "#9a9aa7", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + { + "id": "color" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Virtual Server" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/ddq2jbj2v588wa/device-virtual-servers?orgId=1&${device_name:queryparam}&var-virtual_server=${__data.fields[\"Virtual Server\"]}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 9 + }, + "id": 20, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum_over_time(f5_virtual_server_availability_ratio{job=~\"$device_name\", availability_state=\"available\", f5_virtual_server_name=~\"$virtual_server\"}[24h]) / count_over_time(f5_virtual_server_availability_ratio{job=~\"$device_name\", f5_virtual_server_name=~\"$virtual_server\", availability_state=\"available\"}[24h])", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum_over_time(f5_virtual_server_availability_ratio{job=~\"$device_name\", f5_virtual_server_name=~\"$virtual_server\", availability_state=\"offline\"}[24h]) / count_over_time(f5_virtual_server_availability_ratio{job=~\"$device_name\", f5_virtual_server_name=~\"$virtual_server\", availability_state=\"offline\"}[24h])", + "format": "table", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum_over_time(f5_virtual_server_availability_ratio{job=~\"$device_name\", f5_virtual_server_name=~\"$virtual_server\", availability_state=\"unknown\"}[24h]) / count_over_time(f5_virtual_server_availability_ratio{job=~\"$device_name\", f5_virtual_server_name=~\"$virtual_server\", availability_state=\"unknown\"}[24h])", + "format": "table", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum_over_time(f5_virtual_server_enabled_ratio{job=~\"$device_name\", f5_virtual_server_name=~\"$virtual_server\"}[24h]) / count_over_time(f5_virtual_server_enabled_ratio{job=~\"$device_name\", f5_virtual_server_name=~\"$virtual_server\"}[24h])", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "D" + } + ], + "title": "Virtual Server Availability [24h]", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "f5_virtual_server_name", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "availability_state": false, + "availability_state 1": true, + "availability_state 2": true, + "availability_state 3": true, + "dataType 1": true, + "dataType 2": true, + "dataType 3": true, + "dataType 4": true, + "instance 1": true, + "instance 2": true, + "instance 3": true, + "instance 4": true, + "job 1": true, + "job 2": true, + "job 3": true, + "job 4": true + }, + "includeByName": {}, + "indexByName": { + "Time 1": 1, + "Time 2": 10, + "Time 3": 15, + "Time 4": 20, + "Value #A": 7, + "Value #B": 8, + "Value #C": 9, + "Value #D": 6, + "availability_state 1": 2, + "availability_state 2": 11, + "availability_state 3": 16, + "dataType 1": 3, + "dataType 2": 12, + "dataType 3": 17, + "dataType 4": 21, + "f5_virtual_server_name": 0, + "instance 1": 4, + "instance 2": 13, + "instance 3": 18, + "instance 4": 22, + "job 1": 5, + "job 2": 14, + "job 3": 19, + "job 4": 23 + }, + "renameByName": { + "Time 1": "", + "Value #A": "Available", + "Value #B": "Offline", + "Value #C": "Unknown", + "Value #D": "Enabled", + "f5_virtual_server_name": "Virtual Server", + "job 1": "" + } + } + } + ], + "type": "table" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 25, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Availability" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "available": { + "color": "green", + "index": 2, + "text": "Available" + }, + "offline": { + "color": "red", + "index": 1, + "text": "Offline" + }, + "unknown": { + "color": "orange", + "index": 0, + "text": "Unknown" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pool" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdu11thaxntvkf/device-pools?orgId=1&${device_name:queryparam}&var-pool=${__data.fields[\"Pool\"]}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Virtual Server" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/ddq2jbj2v588wa/device-virtual-servers?orgId=1&${device_name:queryparam}&var-virtual_server=${__data.fields[\"Virtual Server\"]}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 24, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "f5_virtual_server_info{job=\"$device_name\", f5_virtual_server_name=~\"$virtual_server\", f5_pool_name!=\"\"}", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "f5_pool_availability_ratio{job=\"$device_name\"}==1", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "B" + } + ], + "title": "Virtual Server Pool State", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "f5_pool_name", + "mode": "outer" + } + }, + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "greater", + "options": { + "value": 0 + } + }, + "fieldName": "Time 1" + } + ], + "match": "all", + "type": "include" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time 1": true, + "Time 2": true, + "Value #A": true, + "Value #B": true, + "__name__ 1": true, + "__name__ 2": true, + "dataType 1": true, + "dataType 2": true, + "f5_virtual_server_destination": true, + "f5_virtual_server_protocol": true, + "instance 1": true, + "instance 2": true, + "job 1": true, + "job 2": true + }, + "includeByName": {}, + "indexByName": { + "Time 1": 0, + "Time 2": 10, + "Value #A": 9, + "Value #B": 16, + "__name__ 1": 1, + "__name__ 2": 11, + "availability_state": 12, + "dataType 1": 2, + "dataType 2": 13, + "f5_pool_name": 5, + "f5_virtual_server_destination": 3, + "f5_virtual_server_name": 4, + "f5_virtual_server_protocol": 6, + "instance 1": 7, + "instance 2": 14, + "job 1": 8, + "job 2": 15 + }, + "renameByName": { + "availability_state": "Availability", + "f5_pool_name": "Pool", + "f5_virtual_server_name": "Virtual Server", + "f5_virtual_server_protocol": "" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Rule" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdrjs6lfzq2v4e/device-irules?orgId=1&${device_name:queryparam}&var-rule=${__data.fields[\"Rule\"]}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Virtual Server" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/ddq2jbj2v588wa/device-virtual-servers?orgId=1&${device_name:queryparam}&var-virtual_server=${__data.fields[\"Virtual Server\"]}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 28, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "f5_virtual_server_rule_enabled_ratio{job=\"$device_name\", f5_virtual_server_name=~\"$virtual_server\"}", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Applied iRules", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "dataType": true, + "instance": true, + "job": true + }, + "includeByName": {}, + "indexByName": { + "Time": 0, + "Value": 7, + "__name__": 1, + "dataType": 2, + "f5_rule_name": 4, + "f5_virtual_server_name": 3, + "instance": 5, + "job": 6 + }, + "renameByName": { + "f5_rule_name": "Rule", + "f5_virtual_server_name": "Virtual Server", + "instance": "" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Virtual Server" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/ddq2jbj2v588wa/device-virtual-servers?orgId=1&${device_name:queryparam}&var-virtual_server=${__data.fields[\"Virtual Server\"]}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dashboard" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "1": { + "index": 0, + "text": "Link" + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 27, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "f5_virtual_server_profile_info{job=\"$device_name\", f5_virtual_server_name=~\"$virtual_server\"}", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "interval": "", + "legendFormat": "__auto", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Applied Profiles", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "Value #A": true, + "Value #B": false, + "__name__": true, + "dataType": true, + "instance": true, + "job": true + }, + "includeByName": {}, + "indexByName": { + "Time": 0, + "Value": 9, + "__name__": 1, + "dataType": 2, + "f5_profile_family": 5, + "f5_profile_name": 4, + "f5_profile_type": 6, + "f5_virtual_server_name": 3, + "instance": 7, + "job": 8 + }, + "renameByName": { + "Value #B": "Dashboard", + "f5_profile_family": "Profile Class", + "f5_profile_name": "Profile Name", + "f5_profile_type": "Profile Type", + "f5_virtual_server_name": "Virtual Server", + "instance": "" + } + } + } + ], + "type": "table" + } + ], + "title": "Configured Objects", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 30, + "panels": [ + { + "datasource": { + "name": "${datasource}", + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "f5_virtual_server_cpu_utilization_5s{job=\"$device_name\", f5_virtual_server_name=~\"$virtual_server\"}", + "instant": false, + "legendFormat": "{{f5_virtual_server_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Virtual Server CPU Utilization", + "type": "timeseries" + }, + { + "datasource": { + "name": "${datasource}", + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "f5_virtual_server_asm_cpu_utilization_percent{job=\"$device_name\", f5_virtual_server_name=~\"$virtual_server\"}", + "instant": false, + "legendFormat": "{{f5_virtual_server_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Virtual Server ASM CPU Utilization", + "type": "timeseries" + } + ], + "title": "CPU Utilization", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 6, + "panels": [ + { + "datasource": { + "name": "${datasource}", + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(f5_virtual_server_requests_total{job=\"$device_name\", f5_virtual_server_name=~\"$virtual_server\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_virtual_server_name}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Virtual Server Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "name": "${datasource}", + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(f5_virtual_server_clientside_bytes_in_total{job=\"$device_name\", f5_virtual_server_name=~\"$virtual_server\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_virtual_server_name}} - In", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "-1 * rate(f5_virtual_server_clientside_bytes_out_total{job=\"$device_name\", f5_virtual_server_name=~\"$virtual_server\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{f5_virtual_server_name}} - Out", + "range": true, + "refId": "B" + } + ], + "title": "Virtual Server Traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(f5_virtual_server_packets_in_total{job=\"$device_name\", f5_virtual_server_name=~\"$virtual_server\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_virtual_server_name}} - In", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "-1 * rate(f5_virtual_server_packets_out_total{job=\"$device_name\", f5_virtual_server_name=~\"$virtual_server\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{f5_virtual_server_name}} - Out", + "range": true, + "refId": "B" + } + ], + "title": "Virtual Server Packets Rate", + "type": "timeseries" + } + ], + "title": "Traffic", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 10, + "panels": [ + { + "datasource": { + "name": "${datasource}", + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 20 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "f5_virtual_server_clientside_connection_count{job=\"$device_name\", f5_virtual_server_name=~\"$virtual_server\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_virtual_server_name}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Virtual Server Concurrent Connections", + "type": "timeseries" + }, + { + "datasource": { + "name": "${datasource}", + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "C/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 20 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(f5_virtual_server_clientside_connection_total{job=\"$device_name\", f5_virtual_server_name=~\"$virtual_server\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_virtual_server_name}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Virtual Server Connection Rate", + "type": "timeseries" + }, + { + "datasource": { + "name": "${datasource}", + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 20 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "f5_virtual_server_clientside_connection_duration_mean_milliseconds{job=\"$device_name\", f5_virtual_server_name=~\"$virtual_server\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_virtual_server_name}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Virtual Server Avg. Connection Duration", + "type": "timeseries" + } + ], + "title": "Connections", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 31, + "panels": [ + { + "datasource": { + "name": "${datasource}", + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "HTTP Requests by Virtual Server", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(f5_virtual_server_name) (rate(f5_virtual_server_profile_http_requests_by_method_total{job=\"$device_name\", f5_virtual_server_name=~\"$virtual_server\"}[$__rate_interval]))", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_virtual_server_name}} - Requests", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(f5_virtual_server_name) (rate(f5_virtual_server_profile_http_responses_by_status_total{job=~\"$device_name\", f5_virtual_server_name=~\"$virtual_server\"}[$__rate_interval]))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_virtual_server_name}} - Responses", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "HTTP Requests & Responses/Sec", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "HTTP Responses by Virtual Server", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(f5_virtual_server_profile_http_responses_by_status_total{job=~\"$device_name\", f5_virtual_server_name=~\"$virtual_server\", http_status_range=\"2xx\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_virtual_server_name}} - 2xx", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(f5_virtual_server_name) (rate(f5_virtual_server_profile_http_responses_by_status_total{job=~\"$device_name\", f5_virtual_server_name=~\"$virtual_server\", http_status_range!=\"2xx\"}[$__rate_interval]))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_virtual_server_name}} - Other", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "HTTP Responses/Sec", + "type": "timeseries" + } + ], + "title": "HTTP", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 39, + "panels": [ + { + "datasource": { + "name": "${datasource}", + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "minWidth": 110 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Inherits From" + }, + "properties": [ + { + "id": "custom.width", + "value": 134 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Profile Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 181 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 43, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "f5_profile_dns_info", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Panel Title", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "f5_defaults_from_name", + "f5_profile_dns_64", + "f5_profile_dns_app_service", + "f5_profile_dns_cache_enabled", + "f5_profile_dns_dns_express_enabled", + "f5_profile_dns_dns_firewall_enabled", + "f5_profile_dns_dnssec_enabled", + "f5_profile_dns_gtm_enabled", + "f5_profile_dns_hardware_query_validation_enabled", + "f5_profile_dns_hardware_response_cache_enabled", + "f5_profile_dns_local_bind", + "f5_profile_dns_name", + "f5_profile_dns_security", + "f5_profile_dns_unhandled_query_action", + "instance", + "Value" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Value": true, + "f5_profile_dns_app_service": true, + "f5_profile_dns_hardware_response_cache_enabled": false, + "f5_profile_dns_local_bind": false, + "f5_profile_dns_unhandled_query_action": true, + "instance": true + }, + "includeByName": {}, + "indexByName": { + "Value": 15, + "f5_defaults_from_name": 1, + "f5_profile_dns_64": 10, + "f5_profile_dns_app_service": 13, + "f5_profile_dns_cache_enabled": 2, + "f5_profile_dns_dns_express_enabled": 3, + "f5_profile_dns_dns_firewall_enabled": 8, + "f5_profile_dns_dnssec_enabled": 9, + "f5_profile_dns_gtm_enabled": 4, + "f5_profile_dns_hardware_query_validation_enabled": 5, + "f5_profile_dns_hardware_response_cache_enabled": 6, + "f5_profile_dns_local_bind": 11, + "f5_profile_dns_name": 0, + "f5_profile_dns_security": 7, + "f5_profile_dns_unhandled_query_action": 12, + "instance": 14 + }, + "renameByName": { + "f5_defaults_from_name": "Inherits From", + "f5_profile_dns_64": "DNS64 Enabled", + "f5_profile_dns_app_service": "DNS App Service", + "f5_profile_dns_cache_enabled": "Cache Enabled", + "f5_profile_dns_dns_express_enabled": "DNSExpress Enabled", + "f5_profile_dns_dns_firewall_enabled": "DNS Firewall Enabled", + "f5_profile_dns_dnssec_enabled": "DNSSEC Enabled", + "f5_profile_dns_gtm_enabled": "GTM Enabled", + "f5_profile_dns_hardware_query_validation_enabled": "HW Validation Enabled", + "f5_profile_dns_hardware_response_cache_enabled": "HW Response Cache Enabled", + "f5_profile_dns_local_bind": "Local BInd", + "f5_profile_dns_name": "Profile Name", + "f5_profile_dns_security": "DNS Security Enabled" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "name": "${datasource}", + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 42, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "f5_profile_dns_client_hits_response_time_milliseconds{job=~\"$device_name\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_profile_dns_name}} - Hits", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "f5_profile_dns_client_misses_response_time_milliseconds{job=~\"$device_name\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_profile_dns_name}} - Misses", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "DNS Latencies (msec.)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(f5_profile_dns_name) (rate(f5_profile_dns_request_total{job=~\"$device_name\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_profile_dns_name}} - Requests", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(f5_profile_dns_name) (rate(f5_profile_dns_response_total{job=~\"$device_name\"}[$__rate_interval]))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_profile_dns_name}} - Responses", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "DNS Queries & Responses/Sec", + "type": "timeseries" + }, + { + "datasource": { + "name": "${datasource}", + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(f5_profile_dns_name) (rate(f5_profile_dns_dnsx_queries_total{job=~\"$device_name\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_profile_dns_name}} - Queries", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(f5_profile_dns_name) (rate(f5_profile_dns_dnsx_responses_total{job=\"$device_name\"}[$__rate_interval]))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_profile_dns_name}} - Responses", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "DNS Express Queries & Responses/Sec", + "type": "timeseries" + } + ], + "title": "DNS", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 34, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(f5_virtual_server_name) (rate(f5_virtual_server_profile_client_ssl_bytes_in_total{f5_virtual_server_name=~\"$virtual_server\", job=~\"$device_name\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_virtual_server_name}} - Inbound", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": " - sum by(f5_virtual_server_name) (rate(f5_virtual_server_profile_client_ssl_bytes_out_total{f5_virtual_server_name=~\"$virtual_server\", job=~\"$device_name\"}[$__rate_interval]))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_virtual_server_name}} - Outbound", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "SSL Bytes/Sec", + "type": "timeseries" + }, + { + "datasource": { + "name": "${datasource}", + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(f5_virtual_server_name) (rate(f5_virtual_server_profile_client_ssl_renegotiations_total{f5_virtual_server_name=~\"$virtual_server\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_virtual_server_name}} - Renegotiiations", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(f5_virtual_server_name) (rate(f5_virtual_server_profile_client_ssl_secure_handshakes_total{f5_virtual_server_name=~\"$virtual_server\"}[$__rate_interval]))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{f5_virtual_server_name}} - Handshakes", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "SSL Handshakes & Renegotiations/Sec", + "type": "timeseries" + } + ], + "title": "SSL", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(f5_system_state_up_ratio,job)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "device_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(f5_system_state_up_ratio,job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(f5_virtual_server_info{job=\"$device_name\"},f5_virtual_server_name)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "virtual_server", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(f5_virtual_server_info{job=\"$device_name\"},f5_virtual_server_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Device Virtual Server Details", + "uid": "cdy11y7ws8miod", + "version": 9, + "weekStart": "" + } \ No newline at end of file From 03499dad69f3eed63d0b6ba2fddd8e275da58994 Mon Sep 17 00:00:00 2001 From: Chris Hain Date: Thu, 17 Oct 2024 16:32:07 -0700 Subject: [PATCH 07/10] add peer state to fleet and device overview dashboards --- .../bigip/device/device-overview.json | 8 +- .../bigip/fleet/fleet-inventory.json | 77 +++++++++++++++---- 2 files changed, 62 insertions(+), 23 deletions(-) diff --git a/services/grafana/provisioning/dashboards/bigip/device/device-overview.json b/services/grafana/provisioning/dashboards/bigip/device/device-overview.json index 3a2efdf..7ebfd30 100644 --- a/services/grafana/provisioning/dashboards/bigip/device/device-overview.json +++ b/services/grafana/provisioning/dashboards/bigip/device/device-overview.json @@ -755,12 +755,8 @@ "mode": "absolute", "steps": [ { - "color": "red", + "color": "transparent", "value": null - }, - { - "color": "green", - "value": 1 } ] } @@ -2102,6 +2098,6 @@ "timezone": "browser", "title": "Device Overview", "uid": "adpz4tutl28e8f", - "version": 7, + "version": 8, "weekStart": "" } \ No newline at end of file diff --git a/services/grafana/provisioning/dashboards/bigip/fleet/fleet-inventory.json b/services/grafana/provisioning/dashboards/bigip/fleet/fleet-inventory.json index 1ed515e..3a3c335 100644 --- a/services/grafana/provisioning/dashboards/bigip/fleet/fleet-inventory.json +++ b/services/grafana/provisioning/dashboards/bigip/fleet/fleet-inventory.json @@ -358,6 +358,7 @@ }, { "datasource": { + "default": false, "type": "prometheus", "uid": "${datasource}" }, @@ -430,49 +431,91 @@ "legendFormat": "__auto", "range": false, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "f5_system_failover_peer_active_state_ratio", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "B" } ], "title": "Inventory Detail", "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "job", + "mode": "outer" + } + }, { "id": "organize", "options": { "excludeByName": { "Time": true, + "Time 1": true, + "Time 2": true, "Value": true, + "Value #A": true, + "Value #B": true, "__name__": true, + "__name__ 1": true, + "__name__ 2": true, "dataType": true, + "dataType 1": true, + "dataType 2": true, "f5_endpoint": true, + "failover_peer_name": false, "instance": true, + "instance 1": true, + "instance 2": true, "job": true, + "job 1": true, "partition": true }, "includeByName": {}, "indexByName": { - "Time": 0, - "Value": 2, - "__name__": 1, - "dataType": 3, - "f5_instance_management_ip": 5, - "f5_system_baseMac": 7, - "f5_system_chassisType": 11, - "f5_system_marketing_name": 10, - "f5_system_partition": 12, - "failover_state": 13, - "instance": 14, - "job": 15, - "system_id": 6, - "system_name": 4, - "system_product_name": 9, - "system_version": 8 + "Time 1": 12, + "Time 2": 17, + "Value #A": 16, + "Value #B": 21, + "__name__ 1": 13, + "__name__ 2": 18, + "dataType 1": 14, + "dataType 2": 19, + "f5_instance_management_ip": 1, + "f5_system_baseMac": 3, + "f5_system_chassisType": 7, + "f5_system_marketing_name": 6, + "f5_system_partition": 8, + "failover_peer_name": 9, + "failover_state": 10, + "instance 1": 15, + "instance 2": 20, + "job 1": 11, + "system_id": 2, + "system_name": 0, + "system_product_name": 5, + "system_version": 4 }, "renameByName": { "dataType": "", + "dataType 2": "", "f5_instance_management_ip": "Management IP", "f5_system_baseMac": "Base MAC", "f5_system_chassisType": "Chassis Type", "f5_system_marketing_name": "Product Type", "f5_system_partition": "Partition", + "failover_peer_name": "Failover Peer", "failover_state": "Failover State", "status": "Failover Status", "system_id": "System ID", @@ -514,6 +557,6 @@ "timezone": "browser", "title": "Fleet Inventory", "uid": "bdpz4za42wnb4a", - "version": 4, + "version": 6, "weekStart": "" } \ No newline at end of file From 2579552e90ee39b04d49775a6da02a45bb1820ae Mon Sep 17 00:00:00 2001 From: Chris Hain Date: Fri, 18 Oct 2024 09:00:21 -0700 Subject: [PATCH 08/10] move legend on device overview disk use to bottom to prevent squish --- .../provisioning/dashboards/bigip/device/device-overview.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/grafana/provisioning/dashboards/bigip/device/device-overview.json b/services/grafana/provisioning/dashboards/bigip/device/device-overview.json index 7ebfd30..df9a92c 100644 --- a/services/grafana/provisioning/dashboards/bigip/device/device-overview.json +++ b/services/grafana/provisioning/dashboards/bigip/device/device-overview.json @@ -640,7 +640,7 @@ "options": { "legend": { "displayMode": "table", - "placement": "right", + "placement": "bottom", "showLegend": true }, "pieType": "donut", From 1f17cd58e8787ec8216e261fa86df2d5bad2f198 Mon Sep 17 00:00:00 2001 From: Chris Hain Date: Fri, 18 Oct 2024 11:47:47 -0700 Subject: [PATCH 09/10] add secret management section to docsite --- .../secret_management/env_file_based.md | 52 ++++++++++++ .../secret_management/secret_management.md | 18 +++++ pages/config/secret_management/vault_based.md | 79 +++++++++++++++++++ 3 files changed, 149 insertions(+) create mode 100644 pages/config/secret_management/env_file_based.md create mode 100644 pages/config/secret_management/secret_management.md create mode 100644 pages/config/secret_management/vault_based.md diff --git a/pages/config/secret_management/env_file_based.md b/pages/config/secret_management/env_file_based.md new file mode 100644 index 0000000..90c859f --- /dev/null +++ b/pages/config/secret_management/env_file_based.md @@ -0,0 +1,52 @@ +--- +layout: page +title: Via env File +parent: Secrets Management +grandparent: Configuration +nav_order: 1 +--- + +## Configure Device Secrets Via File + +The application study tool default configuration relies on environment variables +which contain device access credentials. There are a number of ways to manage and +inject secrets into a container environment (modifications to the docker-compose file +to support your preferred management process are encouraged), but for simplicity, +if there is a file named .env.device-secrets in the root project directory they will be +mounted. + +Create a file called .env.device-secrets, and add your BIP passwords like so: +``` +BIGIP_PASSWORD_1=foo-bar123! +BIGIP_PASSWORD_2=bar-foo123! +``` + +The variable name (the part on the left of the equal sign) must match the configured +value for the devices that use this password in config/ast_defaults.yaml or device specifc +cofig in config/bigip_receivers.yaml. In the following example, bigip/1 uses BIGIP_PASSWORD_1 +from the defaults and bigip/2 uses BIGIP_PASSWORD_2 from the device settings: + +``` +############################## +## config/ast_defaults.yaml +############################## + +bigip_receiver_defaults: + ... + password: "${env:BIGIP_PASSWORD_1}" + ... + +############################## +## config/bigip_receivers.yaml +############################## + +# This gets the default "${env:BIGIP_PASSWORD_1}" +bigip/1: + endpoint: https://10.0.0.1 + +# This overrides it with "${env:BIGIP_PASSWORD_2}" +bigip/2: + endpoint: https://10.0.0.1 + password: ${env:BIGIP_PASSWORD_2} + +``` \ No newline at end of file diff --git a/pages/config/secret_management/secret_management.md b/pages/config/secret_management/secret_management.md new file mode 100644 index 0000000..2c06618 --- /dev/null +++ b/pages/config/secret_management/secret_management.md @@ -0,0 +1,18 @@ +--- +layout: page +title: Secrets Management +parent: Configuration +nav_order: 4 +--- + +# BigIP Device Secrets Management + +Application Study Tool's Opentelemetry Collector requires credentials to access the BigIP devices +being monitored. The default configuration (management via environment variable file on disk) +is optimized for quick proof of concept style deployment, and is unlikely to meet the +"production" level use case requirements in many organizations. + +This section describes the default configuration (via env file), and outlines alternate +approaches which may be more appropriate for actual production use cases (at a high +level since the details will depend heavily on provider selection, deployment environment, +etc). diff --git a/pages/config/secret_management/vault_based.md b/pages/config/secret_management/vault_based.md new file mode 100644 index 0000000..a07cdb5 --- /dev/null +++ b/pages/config/secret_management/vault_based.md @@ -0,0 +1,79 @@ +--- +layout: page +title: Via Vault Provider +parent: Secrets Management +grandparent: Configuration +nav_order: 2 +--- + +## Configure Device Secrets Via Vault Provider + +Integration with a vault provider to inject secrets into the container environment can +be used to avoid storing secrets on disk. + +Implementation specifics will vary, but at a high level: + +* Device credentials are stored in the vault. +* A vault agent (or script, or similar) running on the AST host reads the keys from the vault and injects them into the +environment directly, or into file on a ramdisk. +* The configuration files for AST point devices at the appropriate environment variables +through the same mechanism as the static file based approach (e.g. ${env:BIGIP_PASSWORD_1}). + +For more, see e.g.: + +* [Hashicorp Vault Agent - secrets as environment variables](https://developer.hashicorp.com/vault/tutorials/vault-agent/agent-env-vars) +* [GCP Secret Manager](https://cloud.google.com/secret-manager/docs/create-secret-quickstart#secretmanager-quickstart-gcloud) +* [Azure Key Vault](https://learn.microsoft.com/en-us/azure/key-vault/secrets/quick-create-cli) +* [AWS Secret Manager](https://docs.aws.amazon.com/secretsmanager/latest/userguide/retrieving-secrets_cli.html) + +An extremely simplified example might look like: +```shell +$ sudo mkdir /mnt/secrets +$ sudo mount -o size=10M -t tmpfs none /mnt/secrets +$ echo -n "password1_value" | gcloud secrets create BIGIP_PASSWORD_1 --replication-policy="automatic" --data-file=- +$ echo -n "password2_value" | gcloud secrets create BIGIP_PASSWORD_2 --replication-policy="automatic" --data-file=- + +$ ./secretfetcher.sh +Secrets have been written to /mnt/secrets/.env.device-secrets + +$ cat /mnt/secrets/.env.device-secrets +BIGIP_PASSWORD_1=password1_value +BIGIP_PASSWORD_2=password2_value +``` + +And the shell script might look like: +```bash +#!/usr/bin/env bash + +# List of secret names +secret_names=("BIGIP_PASSWORD_1" "BIGIP_PASSWORD_2") + +# Output file +output_file="/mnt/secrets/.env.device-secrets" + +# Clear the output file if it already exists +> "$output_file" + +for secret in "${secret_names[@]}"; do + # Access the secret value + secret_value=$(gcloud secrets versions access 1 --secret="$secret" 2>/dev/null) + + # Check if the secret value was retrieved successfully + if [[ $? -eq 0 ]]; then + # Output in .env style format + echo "$secret=$secret_value" >> "$output_file" + else + echo "Failed to access secret: $secret" + fi +done +echo "Secrets have been written to $output_file" +``` + +You'd also need to point to the tmpfs secret file in the `docker-compose.yaml` file: +```yaml + otel-collector: + #... + env_file: + - ".env" + - "/mnt/secrets/.env.device-secrets" +``` \ No newline at end of file From 72877288d0fe955f1da8bb10af8cca166920b9e2 Mon Sep 17 00:00:00 2001 From: Chris Hain Date: Fri, 18 Oct 2024 14:07:26 -0700 Subject: [PATCH 10/10] release 0.7.0 --- VERSION | 2 +- docker-compose.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index e07d136..e7f5d1a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.6.0 \ No newline at end of file +v0.7.0 \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 7c121c7..b9fbd09 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -27,7 +27,7 @@ services: - 7lc_network otel-collector: - image: ghcr.io/f5devcentral/application-study-tool/otel_custom_collector:v0.6.0 + image: ghcr.io/f5devcentral/application-study-tool/otel_custom_collector:v0.7.0 restart: unless-stopped volumes: - ./services/otel_collector:/etc/otel-collector-config