From 1397aca81ce20454740d2c7940d95599a50ed24a Mon Sep 17 00:00:00 2001 From: smithclay Date: Mon, 18 Dec 2023 09:21:20 -0800 Subject: [PATCH] v0.0.5 --- .github/ISSUE_TEMPLATE/bug_report.md | 30 + .github/ISSUE_TEMPLATE/feature_request.md | 21 + .github/pull_request_template.md | 13 + .github/workflows/build.yml | 58 ++ .github/workflows/build_manual_msi.yml | 63 ++ .github/workflows/release.yaml | 56 ++ .github/workflows/test.yml | 24 + .gitignore | 3 +- .goreleaser.yaml | 136 ++++ CONTRIBUTING.md | 59 ++ LICENSE | 202 ++++++ collector/Dockerfile | 20 +- collector/Makefile | 16 + .../components/osqueryreceiver/factory.go | 102 --- collector/components/osqueryreceiver/go.mod | 11 - .../components/resourceapiextension/config.go | 27 - .../resourceapiextension/factory.go | 41 -- .../components/resourceapiextension/go.mod | 10 - .../components/resourceapiextension/readme.md | 3 - .../resource_api_extension.go | 79 --- .../resourcegraphprocessor/config.go | 22 - .../resourcegraphprocessor/factory.go | 81 --- .../components/resourcegraphprocessor/go.mod | 12 - .../resourcegraphprocessor/readme.md | 18 - .../resource-schema.yaml | 99 --- .../resource_graph_processor.go | 98 --- .../components/servicenowexporter/client.go | 132 ++++ .../components/servicenowexporter/config.go | 46 ++ .../components/servicenowexporter/factory.go | 52 +- .../servicenowexporter/factory_test.go | 14 + .../components/servicenowexporter/go.mod | 52 +- .../components/servicenowexporter/go.sum | 151 +++++ .../internal/metadata/generated_status.go | 23 + .../servicenowexporter/metadata.yaml | 5 + .../servicenowexporter/servicenow.go | 471 ++++++++++++++ .../servicenowexporter/servicenow_event.go | 17 + .../servicenowexporter/servicenow_log.go | 13 + .../servicenowexporter/servicenow_metric.go | 14 + .../config/otelcol-docker-hostmetrics.yaml | 88 +++ .../config/otelcol-linux-hostmetrics.yaml | 81 +++ .../config/otelcol-macos-hostmetrics.yaml | 81 +++ .../config/otelcol-windows-hostmetrics.yaml | 81 +++ collector/examples/k8s-deployment.yaml | 181 ++++++ collector/examples/otelcol-k8s.yaml | 75 +++ collector/otelcol-builder.yaml | 62 +- collector/otelcol.yaml | 35 - collector/scripts/install/install-macos.sh | 600 ++++++++++++++++++ collector/scripts/package/postinstall.sh | 77 +++ collector/scripts/package/preinstall.sh | 18 + .../service/com.servicenow.collector.plist | 31 + collector/service/sn-collector.service | 22 + docs/development.md | 30 + docs/monitor-kubernetes.md | 46 ++ docs/monitor-linux.md | 50 ++ docs/monitor-macos.md | 13 + docs/monitor-windows.md | 5 + readme.md | 133 +++- 57 files changed, 3306 insertions(+), 697 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/build_manual_msi.yml create mode 100644 .github/workflows/release.yaml create mode 100644 .github/workflows/test.yml create mode 100644 .goreleaser.yaml create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE delete mode 100644 collector/components/osqueryreceiver/factory.go delete mode 100644 collector/components/osqueryreceiver/go.mod delete mode 100644 collector/components/resourceapiextension/config.go delete mode 100644 collector/components/resourceapiextension/factory.go delete mode 100644 collector/components/resourceapiextension/go.mod delete mode 100644 collector/components/resourceapiextension/readme.md delete mode 100644 collector/components/resourceapiextension/resource_api_extension.go delete mode 100644 collector/components/resourcegraphprocessor/config.go delete mode 100644 collector/components/resourcegraphprocessor/factory.go delete mode 100644 collector/components/resourcegraphprocessor/go.mod delete mode 100644 collector/components/resourcegraphprocessor/readme.md delete mode 100644 collector/components/resourcegraphprocessor/resource-schema.yaml delete mode 100644 collector/components/resourcegraphprocessor/resource_graph_processor.go create mode 100644 collector/components/servicenowexporter/client.go create mode 100644 collector/components/servicenowexporter/config.go create mode 100644 collector/components/servicenowexporter/factory_test.go create mode 100644 collector/components/servicenowexporter/go.sum create mode 100644 collector/components/servicenowexporter/internal/metadata/generated_status.go create mode 100644 collector/components/servicenowexporter/metadata.yaml create mode 100644 collector/components/servicenowexporter/servicenow.go create mode 100644 collector/components/servicenowexporter/servicenow_event.go create mode 100644 collector/components/servicenowexporter/servicenow_log.go create mode 100644 collector/components/servicenowexporter/servicenow_metric.go create mode 100644 collector/config/otelcol-docker-hostmetrics.yaml create mode 100644 collector/config/otelcol-linux-hostmetrics.yaml create mode 100644 collector/config/otelcol-macos-hostmetrics.yaml create mode 100644 collector/config/otelcol-windows-hostmetrics.yaml create mode 100644 collector/examples/k8s-deployment.yaml create mode 100644 collector/examples/otelcol-k8s.yaml delete mode 100644 collector/otelcol.yaml create mode 100755 collector/scripts/install/install-macos.sh create mode 100644 collector/scripts/package/postinstall.sh create mode 100644 collector/scripts/package/preinstall.sh create mode 100644 collector/service/com.servicenow.collector.plist create mode 100644 collector/service/sn-collector.service create mode 100644 docs/development.md create mode 100644 docs/monitor-kubernetes.md create mode 100644 docs/monitor-linux.md create mode 100644 docs/monitor-macos.md create mode 100644 docs/monitor-windows.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..08efd5d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,30 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + + +**Describe the bug** +A clear and concise description of what the bug is. + +**Steps to reproduce** +If possible, provide a recipe for reproducing the error. + +**What did you expect to see?** +A clear and concise description of what you expected to see. + +**What did you see instead?** +A clear and concise description of what you saw instead. + +**Environment** +Describe any aspect of your environment relevant to the problem, +for example operating system or Kubernetes cluster version. +If this is related to a deployment of the collector please +provide your Collector config file. + +**Additional context** +Add any other context about the problem here. \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..a18462b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,21 @@ +--- +name: Feature request +about: Suggest an idea for the ServiceNow Collector +title: '' +labels: '' +assignees: '' + +--- + + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'd like to send data in a different format [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..a714acd --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,13 @@ +**Description:** + + +**Link to tracking Issue:** + +**Testing:** + +**Documentation:** + + + +By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..c83984a --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,58 @@ +# Build is responsible for testing builds on all supported platforms. +# It is broken up into three separate jobs with targeted builds so that each OS will +# build in parallel and speed up overall CI time. +name: Build +on: + pull_request: + +jobs: + build_linux: + runs-on: ubuntu-latest-4-cores + steps: + - name: Checkout Sources + uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '~1.21.5' + check-latest: true + - name: Install builder + working-directory: ./collector + run: | + go install go.opentelemetry.io/collector/cmd/builder@v0.95.0 + - name: Build + working-directory: ./collector + run: make build-linux + build_darwin: + runs-on: macos-14 + steps: + - name: Checkout Sources + uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: "1.20" + check-latest: true + - name: Install builder + working-directory: ./collector + run: | + go install go.opentelemetry.io/collector/cmd/builder@v0.95.0 + - name: Build + run: make build-darwin + build_windows: + runs-on: ubuntu-20.04 + steps: + - name: Checkout Sources + uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: "1.20" + check-latest: true + - name: Install builder + working-directory: ./collector + run: | + go install go.opentelemetry.io/collector/cmd/builder@v0.95.0 + - name: Build + working-directory: ./collector + run: make build-windows diff --git a/.github/workflows/build_manual_msi.yml b/.github/workflows/build_manual_msi.yml new file mode 100644 index 0000000..d2c65de --- /dev/null +++ b/.github/workflows/build_manual_msi.yml @@ -0,0 +1,63 @@ +name: Build MSI (Manual) +on: + workflow_dispatch: + inputs: + version: + description: "Collector Version" + required: true + default: "v0.0.1" + +jobs: + build-64bit-msi: + runs-on: windows-2022 + steps: + - name: Checkout Sources + uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '~1.21.5' + check-latest: true + - name: Generate distribution sources + run: | + go install go.opentelemetry.io/collector/cmd/builder@v0.95.0 + cd collector + builder --config otelcol-builder.yaml --skip-compilation + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v5 + with: + # either 'goreleaser' (default) or 'goreleaser-pro' + distribution: goreleaser + version: "v1.22.1" + args: build --single-target --skip=validate --clean --snapshot + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Copy Windows Collector Binary + run: cp dist/otelcol-servicenow_windows_amd64_v1/otelcol-servicenow.exe windows/otelcol-servicenow.exe + - name: Copy Example Config + run: cp collect/config/otelcol-windows-hostmetrics.yaml windows/config.yaml + - name: Copy LICENSE + run: cp LICENSE windows/LICENSE + # HACK: Copy build directory to C drive to avoid this error, since there must be a relative path from the tempdir that go-msi uses + # for the MSI to build properly + - name: Copy Build Dir + run: | + cp -r windows C:/build + echo "C:/build" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + # Installs go-msi and wix. + - name: Install Build Tools + run: | + curl -f -L -o go-msi.exe https://github.com/observIQ/go-msi/releases/download/v2.2.0/go-msi.exe + curl -f -L -o wix310-binaries.zip http://wixtoolset.org/downloads/v3.10.3.3007/wix310-binaries.zip + unzip wix310-binaries.zip + working-directory: C:/build + - name: "Build MSI from Tagged Release" + run: go-msi.exe make -m otelcol-servicenow.msi --version ${{ github.event.inputs.version }} --arch amd64 + working-directory: C:/build + - name: "Upload MSI" + uses: actions/upload-artifact@v4 + with: + name: otelcol-servicenow.msi + path: C:/build/otelcol-servicenow.msi + # Short lived because this is meant as an action for developers + retention-days: 1 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..72db794 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,56 @@ +# based on: https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/.github/workflows/release.yaml +name: Release + +on: + push: + tags: ["v*"] + +jobs: + release: + permissions: + id-token: write + packages: write + contents: write + + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: docker/setup-qemu-action@v3 + with: + platforms: arm64,linux/arm/v7 + + - uses: docker/setup-buildx-action@v3 + + - uses: actions/setup-go@v5 + with: + go-version: '~1.21.5' + check-latest: true + + - name: Generate distribution sources + run: | + go install go.opentelemetry.io/collector/cmd/builder@v0.95.0 + cd collector + builder --config otelcol-builder.yaml --skip-compilation + + - name: Login to GitHub Package Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - shell: bash + run: | + echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_ENV + + - uses: goreleaser/goreleaser-action@v5 + with: + distribution: goreleaser + version: latest + args: release --clean --skip=sign --timeout 2h + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..30a3230 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,24 @@ +name: Tests +on: + pull_request: + +jobs: + unit-tests: + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04, macos-14, windows-2022-8-cores] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout Sources + uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: "1.20" + check-latest: true + - name: Run tests + working-directory: ./collector + run: | + make test + \ No newline at end of file diff --git a/.gitignore b/.gitignore index 07cd005..6543033 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .DS_Store -collector/otelcol-servicenow/ \ No newline at end of file +collector/otelcol-servicenow/ +dist/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..0f6be51 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,136 @@ +# somewhat inspired from https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/.goreleaser.yaml +project_name: sn-collector-experimental + +builds: + - id: otelcol-servicenow + goos: + - darwin + - linux + - windows + goarch: + - amd64 + - arm + - arm64 + goarm: + - "7" + ignore: + - goos: darwin + goarch: "386" + - goos: darwin + goarch: arm + - goos: darwin + goarch: s390x + - goos: windows + goarch: arm + - goos: windows + goarch: arm64 + - goos: windows + goarch: s390x + dir: ./collector/otelcol-servicenow + binary: otelcol-servicenow + ldflags: + - -s + - -w + flags: + - -trimpath + env: + - CGO_ENABLED=0 + +dockers: + - goos: linux + goarch: amd64 + dockerfile: collector/Dockerfile + image_templates: + - ghcr.io/lightstep/sn-collector/sn-collector-experimental:{{ .Version }}-amd64 + - ghcr.io/lightstep/sn-collector/sn-collector-experimental:latest-amd64 + extra_files: + - collector/config/otelcol-docker-hostmetrics.yaml + build_flag_templates: + - --pull + - --platform=linux/amd64 + - goos: linux + goarch: arm64 + dockerfile: collector/Dockerfile + image_templates: + - ghcr.io/lightstep/sn-collector/sn-collector-experimental:{{ .Version }}-arm64 + - ghcr.io/lightstep/sn-collector/sn-collector-experimental:latest-arm64 + extra_files: + - collector/config/otelcol-docker-hostmetrics.yaml + build_flag_templates: + - --pull + - --platform=linux/arm64 + +docker_manifests: + - name_template: "ghcr.io/lightstep/sn-collector/sn-collector-experimental:latest" + image_templates: + - "ghcr.io/lightstep/sn-collector/sn-collector-experimental:latest-amd64" + - "ghcr.io/lightstep/sn-collector/sn-collector-experimental:latest-arm64" + skip_push: false + +archives: + - id: otelcol-servicenow + builds: + - otelcol-servicenow + name_template: '{{ .Binary }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}{{ if .Mips }}_{{ .Mips }}{{ end }}' + files: + - src: LICENSE + - src: collector/service/sn-collector.service + dst: install + strip_parent: true + - src: collector/service/com.servicenow.collector.plist + dst: install + strip_parent: true + # bundle default config files in packages + - src: collector/config/*.yaml + dst: config + strip_parent: true + format_overrides: + - goos: windows + format: zip + +nfpms: + - id: otelcol-servicenow + package_name: sn-collector + builds: + - otelcol-servicenow + vendor: ServiceNow, Inc. + homepage: https://servicenow.com/ + maintainer: ServiceNow Engineering + description: SerivceNow distribution of the OpenTelemetry Collector. + formats: + - apk + - deb + - rpm + bindir: /opt/sn-collector + scripts: + preinstall: ./collector/scripts/package/preinstall.sh + postinstall: ./collector/scripts/package/postinstall.sh + contents: + - dst: /opt/sn-collector + type: dir + file_info: + mode: 0755 + owner: sn-collector + group: sn-collector + - src: collector/config/otelcol-linux-hostmetrics.yaml + dst: /opt/sn-collector/config.yaml + file_info: + mode: 0640 + owner: sn-collector + group: sn-collector + - src: collector/service/sn-collector.service + dst: /usr/lib/systemd/system/sn-collector.service + type: config|noreplace + file_info: + mode: 0644 + owner: root + group: root + +checksum: + name_template: 'checksums.txt' + +release: + draft: false + prerelease: "true" + extra_files: + - glob: "./collector/scripts/install/install-macos.sh" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..7374979 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,59 @@ +# Contributing Guidelines + +Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional +documentation, we greatly value feedback and contributions from our community. + +Please read through this document before submitting any issues or pull requests to ensure we have all the necessary +information to effectively respond to your bug report or contribution. + + +## Reporting Bugs/Feature Requests + +We welcome you to use the GitHub issue tracker to report bugs or suggest features. + +When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already +reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: + +* A reproducible test case or series of steps +* The version of our code being used +* Any modifications you've made relevant to the bug +* Anything unusual about your environment or deployment + + +## New Component Contributions + +We are currently not supporting pull requests for new components from outside contributors. +If you would like to see a new component integrated please open a Feature Request issue. + + +## Contributing via Pull Requests +Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: + +1. You are working against the latest source on the *main* branch. +2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. +3. You open an issue to discuss any significant work - we would hate for your time to be wasted. + +To send us a pull request, please: + +1. Fork the repository. +2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. +3. Ensure local tests pass. +4. Commit to your fork using clear commit messages. +5. Send us a pull request, answering any default questions in the pull request interface. +6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. + +GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and +[creating a pull request](https://help.github.com/articles/creating-a-pull-request/). + + +## Finding contributions to work on +Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. + + +## Security issue or vulnerability notifications +If you discover a potential security issue in this project we ask that you notify ServiceNow via our [responsible disclosure page](https://www.servicenow.com/company/trust/privacy/responsible-disclosure.html). Please do **not** create a public Github issue. + + +## Licensing + +See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a7550aa --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright ServiceNow Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/collector/Dockerfile b/collector/Dockerfile index 5f3d362..a2af49c 100644 --- a/collector/Dockerfile +++ b/collector/Dockerfile @@ -1,16 +1,16 @@ -FROM golang:1-bullseye AS build-env +FROM alpine:3.16 as certs +RUN apk --update add ca-certificates -RUN go install go.opentelemetry.io/collector/cmd/builder@v0.88.0 +FROM scratch -RUN apt-get update && apt-get install libpcap-dev -y +ARG USER_UID=10001 +USER ${USER_UID} -WORKDIR /otelcol -COPY ./components ./components -COPY otelcol-builder.yaml otelcol-builder.yaml +COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --chmod=755 otelcol-servicenow /otelcol-servicenow -RUN builder --config otelcol-builder.yaml +COPY collector/config/otelcol-docker-hostmetrics.yaml /etc/otelcol/config.yaml -COPY ./otelcol.yaml /etc/otelcol/config.yaml - -ENTRYPOINT ["/otelcol/otelcol-servicenow/otelcol-servicenow"] +ENTRYPOINT ["/otelcol-servicenow"] CMD ["--config", "/etc/otelcol/config.yaml"] +EXPOSE 4317 55678 55679 \ No newline at end of file diff --git a/collector/Makefile b/collector/Makefile index e490d7d..b4521d8 100644 --- a/collector/Makefile +++ b/collector/Makefile @@ -5,10 +5,26 @@ all: build build: components/servicenowexporter/factory.go builder --config otelcol-builder.yaml +.PHONY: build-windows +build-windows: + GOOS=windows GOARCH=amd64 builder --config otelcol-builder.yaml + +.PHONY: build-linux +build-linux: + GOOS=linux GOARCH=amd64 builder --config otelcol-builder.yaml + +.PHONY: build-darwin +build-darwin: + GOOS=darwin GOARCH=amd64 builder --config otelcol-builder.yaml + .PHONY: test - Run tests for servicenowexporter test: cd components/servicenowexporter && go test -v ./... +.PHONY: install-tools +install-tools: + go install go.opentelemetry.io/collector/cmd/builder@v0.95.0 + .PHONY: docker - Build docker image docker: docker build . -t lightstep/servicenow-collector:latest \ No newline at end of file diff --git a/collector/components/osqueryreceiver/factory.go b/collector/components/osqueryreceiver/factory.go deleted file mode 100644 index 0fa139b..0000000 --- a/collector/components/osqueryreceiver/factory.go +++ /dev/null @@ -1,102 +0,0 @@ -package osqueryreceiver - -import ( - "context" - "errors" - "fmt" - "log" - "time" - - osquery "github.com/osquery/osquery-go" - "github.com/osquery/osquery-go/plugin/logger" - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/consumer" - "go.opentelemetry.io/collector/pdata/pcommon" - "go.opentelemetry.io/collector/pdata/plog" - "go.opentelemetry.io/collector/receiver" -) - -const ( - typeStr = "osqueryreceiver" - stability = component.StabilityLevelAlpha -) - -var errInvalidConfig = errors.New("invalid config for osqueryreceiver") - -type Config struct { -} - -func createDefaultConfig() component.Config { - return &Config{} -} - -type osQueryReceiver struct { - config *Config - logsConsumer consumer.Logs -} - -func newLog(body string) plog.Logs { - ld := plog.NewLogs() - rl := ld.ResourceLogs().AppendEmpty() - sl := rl.ScopeLogs().AppendEmpty() - lr := sl.LogRecords().AppendEmpty() - - resourceAttrs := rl.Resource().Attributes() - resourceAttrs.PutStr("foo", "bar") - lr.SetTimestamp(pcommon.NewTimestampFromTime(time.Now())) - - // The Message field contains description about the event, - // which is best suited for the "Body" of the LogRecordSlice. - lr.Body().SetStr(body) - return ld -} - -func (or *osQueryReceiver) Start(ctx context.Context, _ component.Host) error { - server, err := osquery.NewExtensionManagerServer("foobar", "/Users/clay.smith/.osquery/shell.em") - if err != nil { - log.Fatalf("Error creating extension: %s\n", err) - } - server.RegisterPlugin(logger.NewPlugin("example_logger", createLoggerFunction(or.logsConsumer))) - or.logsConsumer.ConsumeLogs(ctx, newLog("test")) - return nil -} - -func createLoggerFunction(lc consumer.Logs) func(context.Context, logger.LogType, string) error { - return func(ctx context.Context, typ logger.LogType, logText string) error { - lc.ConsumeLogs(ctx, newLog(logText)) - //log.Printf("%s: %s\n", typ, logText) - return nil - } -} - -func LogString(ctx context.Context, typ logger.LogType, logText string) error { - log.Printf("%s: %s\n", typ, logText) - return nil -} - -func (or *osQueryReceiver) Shutdown(context.Context) error { - return nil -} - -func createLogsReceiver( - ctx context.Context, - set receiver.CreateSettings, - cfg component.Config, - consumer consumer.Logs, -) (receiver.Logs, error) { - if err := component.ValidateConfig(cfg); err != nil { - return nil, fmt.Errorf("cannot configure servicenow logs exporter: %w", err) - } - return &osQueryReceiver{ - config: cfg.(*Config), - logsConsumer: consumer, - }, nil -} - -func NewFactory() receiver.Factory { - return receiver.NewFactory( - typeStr, - createDefaultConfig, - receiver.WithLogs(createLogsReceiver, stability), - ) -} diff --git a/collector/components/osqueryreceiver/go.mod b/collector/components/osqueryreceiver/go.mod deleted file mode 100644 index 216178b..0000000 --- a/collector/components/osqueryreceiver/go.mod +++ /dev/null @@ -1,11 +0,0 @@ -module github.com/lightstep/sn-collector/components/osqueryreceiver - -go 1.19 - -require ( - go.opentelemetry.io/collector/component v0.81.0 - go.opentelemetry.io/collector/consumer v0.81.0 - go.opentelemetry.io/collector/pdata v1.0.0-rcv0013 - go.opentelemetry.io/collector/receiver v0.81.0 - go.uber.org/zap v1.24.0 -) \ No newline at end of file diff --git a/collector/components/resourceapiextension/config.go b/collector/components/resourceapiextension/config.go deleted file mode 100644 index 8c6c15c..0000000 --- a/collector/components/resourceapiextension/config.go +++ /dev/null @@ -1,27 +0,0 @@ -package resourceapiextension - -import ( - "errors" - - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/config/confighttp" -) - -// Config has the configuration for the extension enabling the health check -// extension, used to report the health status of the service. -type Config struct { - confighttp.HTTPServerSettings `mapstructure:",squash"` -} - -var _ component.Config = (*Config)(nil) -var ( - errNoEndpointProvided = errors.New("bad config: endpoint must be specified") -) - -// Validate checks if the extension configuration is valid -func (cfg *Config) Validate() error { - if cfg.Endpoint == "" { - return errNoEndpointProvided - } - return nil -} diff --git a/collector/components/resourceapiextension/factory.go b/collector/components/resourceapiextension/factory.go deleted file mode 100644 index 459e05c..0000000 --- a/collector/components/resourceapiextension/factory.go +++ /dev/null @@ -1,41 +0,0 @@ -package resourceapiextension - -import ( - "context" - - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/config/confighttp" - "go.opentelemetry.io/collector/extension" -) - -const ( - // Use 0.0.0.0 to make the health check endpoint accessible - // in container orchestration environments like Kubernetes. - defaultEndpoint = "0.0.0.0:13133" - typeStr = "resourceapiextension" - stability = component.StabilityLevelAlpha -) - -// NewFactory creates a factory for HealthCheck extension. -func NewFactory() extension.Factory { - return extension.NewFactory( - typeStr, - createDefaultConfig, - createExtension, - stability, - ) -} - -func createDefaultConfig() component.Config { - return &Config{ - HTTPServerSettings: confighttp.HTTPServerSettings{ - Endpoint: defaultEndpoint, - }, - } -} - -func createExtension(_ context.Context, set extension.CreateSettings, cfg component.Config) (extension.Extension, error) { - config := cfg.(*Config) - - return newServer(*config, set.TelemetrySettings), nil -} diff --git a/collector/components/resourceapiextension/go.mod b/collector/components/resourceapiextension/go.mod deleted file mode 100644 index caa1620..0000000 --- a/collector/components/resourceapiextension/go.mod +++ /dev/null @@ -1,10 +0,0 @@ -module github.com/lightstep/sn-collector/components/resourceapiextension - -go 1.19 - -require ( - go.opentelemetry.io/collector/component v0.81.0 - go.opentelemetry.io/collector/extension v0.81.0 - - go.uber.org/zap v1.24.0 -) \ No newline at end of file diff --git a/collector/components/resourceapiextension/readme.md b/collector/components/resourceapiextension/readme.md deleted file mode 100644 index 931bd9c..0000000 --- a/collector/components/resourceapiextension/readme.md +++ /dev/null @@ -1,3 +0,0 @@ -## resourceapiextension - -Surfaces resource data as a REST-ful API. (WIP) \ No newline at end of file diff --git a/collector/components/resourceapiextension/resource_api_extension.go b/collector/components/resourceapiextension/resource_api_extension.go deleted file mode 100644 index 6ee7429..0000000 --- a/collector/components/resourceapiextension/resource_api_extension.go +++ /dev/null @@ -1,79 +0,0 @@ -package resourceapiextension - -import ( - "context" - "errors" - "fmt" - "net/http" - - "go.opentelemetry.io/collector/component" - "go.uber.org/zap" -) - -type resourceApiExtension struct { - config Config - logger *zap.Logger - server *http.Server - settings component.TelemetrySettings - stopCh chan struct{} -} - -func (re *resourceApiExtension) Start(_ context.Context, host component.Host) error { - - re.logger.Info("Starting resource_api extension", zap.Any("config", re.config)) - ln, err := re.config.ToListener() - if err != nil { - return fmt.Errorf("failed to bind to address %s: %w", re.config.Endpoint, err) - } - - re.server, err = re.config.ToServer(host, re.settings, nil) - if err != nil { - return err - } - re.stopCh = make(chan struct{}) - - mux := http.NewServeMux() - mux.Handle("/", re.baseHandler()) - re.server.Handler = mux - re.stopCh = make(chan struct{}) - go func() { - defer close(re.stopCh) - - // The listener ownership goes to the server. - if err = re.server.Serve(ln); !errors.Is(err, http.ErrServerClosed) && err != nil { - _ = re.settings.ReportComponentStatus(component.NewFatalErrorEvent(err)) - } - }() - - return nil -} - -func (re *resourceApiExtension) Shutdown(context.Context) error { - if re.server == nil { - return nil - } - err := re.server.Close() - if re.stopCh != nil { - <-re.stopCh - } - return err -} - -func (re *resourceApiExtension) baseHandler() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("TODO return resources")) - }) -} - -func newServer(config Config, settings component.TelemetrySettings) *resourceApiExtension { - hc := &resourceApiExtension{ - config: config, - logger: settings.Logger, - settings: settings, - } - - hc.logger = settings.Logger - - return hc -} diff --git a/collector/components/resourcegraphprocessor/config.go b/collector/components/resourcegraphprocessor/config.go deleted file mode 100644 index 1efd543..0000000 --- a/collector/components/resourcegraphprocessor/config.go +++ /dev/null @@ -1,22 +0,0 @@ -package resourcegraphprocessor - -import ( - "go.opentelemetry.io/collector/config/confignet" - "go.opentelemetry.io/collector/config/configopaque" - "go.opentelemetry.io/collector/config/configtls" -) - -type Config struct { - confignet.NetAddr `mapstructure:",squash"` - // Optional username. Use the specified Username to authenticate the current connection - // with one of the connections defined in the ACL list when connecting - // to a Redis 6.0 instance, or greater, that is using the Redis ACL system. - Username string `mapstructure:"username"` - - // Optional password. Must match the password specified in the - // requirepass server configuration option, or the user's password when connecting - // to a Redis 6.0 instance, or greater, that is using the Redis ACL system. - Password configopaque.String `mapstructure:"password"` - - TLS configtls.TLSClientSetting `mapstructure:"tls,omitempty"` -} diff --git a/collector/components/resourcegraphprocessor/factory.go b/collector/components/resourcegraphprocessor/factory.go deleted file mode 100644 index af9c33e..0000000 --- a/collector/components/resourcegraphprocessor/factory.go +++ /dev/null @@ -1,81 +0,0 @@ -package resourcegraphprocessor - -import ( - "context" - - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/config/confignet" - "go.opentelemetry.io/collector/config/configtls" - "go.opentelemetry.io/collector/consumer" - "go.opentelemetry.io/collector/processor" - "go.opentelemetry.io/collector/processor/processorhelper" -) - -var processorCapabilities = consumer.Capabilities{MutatesData: false} - -const ( - typeStr = "resourcegraphprocessor" - stability = component.StabilityLevelAlpha -) - -func createDefaultConfig() component.Config { - return &Config{ - NetAddr: confignet.NetAddr{ - Transport: "tcp", - }, - TLS: configtls.TLSClientSetting{ - Insecure: true, - }, - } -} -func createLogsProcessor( - ctx context.Context, - set processor.CreateSettings, - cfg component.Config, - nextConsumer consumer.Logs) (processor.Logs, error) { - proc, err := newResourceGraphProcessor(cfg.(*Config), set) - if err != nil { - return nil, err - } - - return processorhelper.NewLogsProcessor( - ctx, - set, - cfg.(*Config), - nextConsumer, - proc.processLogs, - processorhelper.WithCapabilities(processorCapabilities), - processorhelper.WithShutdown(proc.Shutdown), - processorhelper.WithStart(proc.Start)) -} - -func createMetricsProcessor( - ctx context.Context, - set processor.CreateSettings, - cfg component.Config, - nextConsumer consumer.Metrics) (processor.Metrics, error) { - proc, err := newResourceGraphProcessor(cfg.(*Config), set) - if err != nil { - return nil, err - } - - return processorhelper.NewMetricsProcessor( - ctx, - set, - cfg.(*Config), - nextConsumer, - proc.processMetrics, - processorhelper.WithCapabilities(processorCapabilities), - processorhelper.WithShutdown(proc.Shutdown), - processorhelper.WithStart(proc.Start)) -} - -func NewFactory() processor.Factory { - return processor.NewFactory( - typeStr, - createDefaultConfig, - //processor.WithTraces(createTracesProcessor, metadata.TracesStability), - processor.WithMetrics(createMetricsProcessor, stability), - processor.WithLogs(createLogsProcessor, stability), - ) -} diff --git a/collector/components/resourcegraphprocessor/go.mod b/collector/components/resourcegraphprocessor/go.mod deleted file mode 100644 index 022facc..0000000 --- a/collector/components/resourcegraphprocessor/go.mod +++ /dev/null @@ -1,12 +0,0 @@ -module github.com/lightstep/sn-collector/components/servicenowexporter - -go 1.19 - -require ( - go.opentelemetry.io/collector/component v0.81.0 - go.opentelemetry.io/collector/consumer v0.81.0 - go.opentelemetry.io/collector/pdata v1.0.0-rcv0013 - go.opentelemetry.io/collector/processor v0.81.0 - go.uber.org/zap v1.24.0 - github.com/redis/go-redis/v9 v9.3.0 -) \ No newline at end of file diff --git a/collector/components/resourcegraphprocessor/readme.md b/collector/components/resourcegraphprocessor/readme.md deleted file mode 100644 index 04b66cc..0000000 --- a/collector/components/resourcegraphprocessor/readme.md +++ /dev/null @@ -1,18 +0,0 @@ -## resourcegraphprocessor - -Infers resources and resource relationships from metrics, logs and traces. - -Uses redis for persistence. - -### usage - -``` - # build - make - - # run - ./otelcol-dev/otelcol-dev --config otelcol.yaml - - # generate some fake telemetry with resource attributes - telemetrygen metrics --duration 1s --otlp-insecure -``` \ No newline at end of file diff --git a/collector/components/resourcegraphprocessor/resource-schema.yaml b/collector/components/resourcegraphprocessor/resource-schema.yaml deleted file mode 100644 index 5ea7c1c..0000000 --- a/collector/components/resourcegraphprocessor/resource-schema.yaml +++ /dev/null @@ -1,99 +0,0 @@ -# TODO: should probably allow all of these to scope to a specific instrumentation library -apiVersion: v0.0.1 -name: otel-to-servicenow-cis -description: Description of inferred resources and relationships from telemetry attributes -# -# Resources -# These get turned into CI Classes -# -resources: -# CloudObs service -- name: service - sources: - - metrics - - logs - - spans - attributes: - - service.name - -# Kubernetes cluster -- name: k8s-cluster - sources: - - metrics - - logs - - spans - attributes: - - k8s.cluster.name - - k8s.cluster.uid - -# Kubernetes namespace -- name: k8s-namespace - sources: - - metrics - - logs - - spans - dependent_relations: - - k8s-cluster - attributes: - - k8s.cluster.name - - k8s.cluster.uid - - k8s.namespace.name -# Kubernetes node -- name: k8s-node - sources: - - metrics - - logs - - spans - dependent_relations: - - k8s-cluster - attributes: - - k8s.cluster.name - - k8s.cluster.uid - - k8s.node.name -- name: k8s-pod - sources: - - metrics - - logs - - spans - attributes: - - k8s.cluster.name - - k8s.cluster.uid - - k8s.namespace.name - - k8s.pod.name - - k8s.pod.uid - -# -# Relationships -# These get turned into CI relationships -# -relationships: -# service -> service -- name: service-to-service - sources: - - metrics - attributes: - - service.name - - peer.service - -# service -> service (following Grafana Tempo convention) -# https://grafana.com/docs/tempo/latest/metrics-generator/service_graphs/ -- name: service-to-service-grafana - metric_name: tempo_service_graph_request_total - sources: - - metrics - attributes: - - client - - server - -# deployment workload -> service -- name: deployment-to-service - sources: - - metrics - - logs - - spans - attributes: - - k8s.cluster.uid - - k8s.cluster.name - - k8s.namespace.name - - k8s.deployment.name - - service.name \ No newline at end of file diff --git a/collector/components/resourcegraphprocessor/resource_graph_processor.go b/collector/components/resourcegraphprocessor/resource_graph_processor.go deleted file mode 100644 index 547967b..0000000 --- a/collector/components/resourcegraphprocessor/resource_graph_processor.go +++ /dev/null @@ -1,98 +0,0 @@ -package resourcegraphprocessor - -import ( - "context" - - "github.com/redis/go-redis/v9" - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/pdata/plog" - "go.opentelemetry.io/collector/pdata/pmetric" - - "go.opentelemetry.io/collector/processor" - "go.uber.org/zap" -) - -type resourceGraphProcessor struct { - logger *zap.Logger - client *redis.Client -} - -func newResourceGraphProcessor(cfg *Config, set processor.CreateSettings) (*resourceGraphProcessor, error) { - opts := &redis.Options{ - Addr: cfg.Endpoint, - Username: cfg.Username, - Password: string(cfg.Password), - Network: cfg.Transport, - } - - var err error - if opts.TLSConfig, err = cfg.TLS.LoadTLSConfig(); err != nil { - return nil, err - } - - return &resourceGraphProcessor{ - logger: set.Logger, - client: redis.NewClient(opts), - }, nil -} - -// redis sets -// resources:project-id -// services:project-id -// hosts:project-id -// service-dependencies:project-id -// k8s-clusters:project-id -// k8s-workloads:project-id -// k8s-namespaces:project-id -// k8s-nodes:project-id -// k8s-pods:project-id -// k8s-containers:project-id -// software:project-id - -func (rpg *resourceGraphProcessor) processLogs(ctx context.Context, ld plog.Logs) (plog.Logs, error) { - return ld, nil -} - -func (rpg *resourceGraphProcessor) processMetrics(ctx context.Context, md pmetric.Metrics) (pmetric.Metrics, error) { - // Batches redis commands into a single request - pipe := rpg.client.Pipeline() - for i := 0; i < md.ResourceMetrics().Len(); i++ { - rm := md.ResourceMetrics().At(i) - v, exists := rm.Resource().Attributes().Get("service.name") - if exists { - rpg.logger.Info("found service name", zap.String("value", v.AsString())) - rpg.client.SAdd(ctx, "resources:project-id", "services:project-id") - rpg.client.SAdd(ctx, "services:project-id", v.AsString()) - } - - v, exists = rm.Resource().Attributes().Get("host.name") - if exists { - rpg.logger.Info("found host name", zap.String("value", v.AsString())) - rpg.client.SAdd(ctx, "resources:project-id", "hosts:project-id") - rpg.client.SAdd(ctx, "hosts:project-id", v.AsString()) - } - } - _, err := pipe.Exec(ctx) - if err != nil { - return md, err - } - - return md, nil -} - -func (rpg *resourceGraphProcessor) Start(ctx context.Context, host component.Host) error { - status := rpg.client.Ping(ctx) - if status.Err() != nil { - rpg.logger.Error("could not connect to redis", zap.Any("status", status)) - } else { - rpg.logger.Info("connected to redis", zap.Any("status", status)) - } - return nil -} - -func (rpg *resourceGraphProcessor) Shutdown(_ context.Context) error { - if rpg.client != nil { - return rpg.client.Close() - } - return nil -} diff --git a/collector/components/servicenowexporter/client.go b/collector/components/servicenowexporter/client.go new file mode 100644 index 0000000..e11e6a7 --- /dev/null +++ b/collector/components/servicenowexporter/client.go @@ -0,0 +1,132 @@ +package servicenowexporter + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + + "go.uber.org/zap" +) + +type midClient struct { + config *Config + httpClient *http.Client + logger *zap.Logger +} + +func newMidClient(config *Config, l *zap.Logger) *midClient { + return &midClient{ + config: config, + logger: l, + httpClient: &http.Client{ + Timeout: config.TimeoutSettings.Timeout, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: config.InsecureSkipVerify, + }, + }, + }, + } +} + +func handleNon200Response(res *http.Response) error { + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return err + } + + return fmt.Errorf("ServiceNow API returned non-200 status code: %d (%s)", res.StatusCode, string(bodyBytes)) +} + +func (c *midClient) Close() { + c.httpClient.CloseIdleConnections() +} + +func (c *midClient) sendEvents(payload []ServiceNowEvent) error { + url := c.config.PushEventsURL + request := ServiceNowEventRequestBody{Records: payload} + c.logger.Info("Sending events to ServiceNow", zap.String("url", url), zap.Any("request", request)) + body, err := json.Marshal(request) + if err != nil { + return err + } + r, err := http.NewRequest("POST", url, bytes.NewBuffer(body)) + if err != nil { + return err + } + r.Header.Set("Content-Type", "application/json") + r.SetBasicAuth(c.config.Username, string(c.config.Password)) + + res, err := c.httpClient.Do(r) + if err != nil { + return err + } + defer res.Body.Close() + + if res.StatusCode != 200 { + return handleNon200Response(res) + } + + return nil +} + +func (c *midClient) sendLogs(payload []ServiceNowLog) error { + url := c.config.PushLogsURL + body, err := json.Marshal(payload) + if err != nil { + return err + } + r, err := http.NewRequest("POST", url, bytes.NewBuffer(body)) + r.Header.Set("Content-Type", "application/json") + + if len(c.config.Username) > 0 { + r.SetBasicAuth(c.config.Username, string(c.config.Password)) + } else if len(c.config.ApiKey) > 0 { + r.Header.Set("Authorization", "key "+string(c.config.ApiKey)) + } + + if err != nil { + return err + } + + res, err := c.httpClient.Do(r) + if err != nil { + return err + } + defer res.Body.Close() + + if res.StatusCode != 200 { + return handleNon200Response(res) + } + + return nil +} + +func (c *midClient) sendMetrics(payload []ServiceNowMetric) error { + url := c.config.PushMetricsURL + body, err := json.Marshal(payload) + if err != nil { + return err + } + r, err := http.NewRequest("POST", url, bytes.NewBuffer(body)) + r.Header.Set("Content-Type", "application/json") + r.SetBasicAuth(c.config.Username, string(c.config.Password)) + if err != nil { + return err + } + + res, err := c.httpClient.Do(r) + if err != nil { + return err + } + defer res.Body.Close() + + if res.StatusCode != 200 { + return handleNon200Response(res) + } + + return nil +} diff --git a/collector/components/servicenowexporter/config.go b/collector/components/servicenowexporter/config.go new file mode 100644 index 0000000..d430f81 --- /dev/null +++ b/collector/components/servicenowexporter/config.go @@ -0,0 +1,46 @@ +package servicenowexporter + +import ( + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/configopaque" + "go.opentelemetry.io/collector/config/configretry" + "go.opentelemetry.io/collector/exporter/exporterhelper" +) + +type Config struct { + exporterhelper.TimeoutSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct. + exporterhelper.QueueSettings `mapstructure:"sending_queue"` + configretry.BackOffConfig `mapstructure:"retry_on_failure"` + + // PushMetricsURL is the full url of the ServiceNow instance to send push metrics to. Ex: http://127.0.0.1:8090/api/mid/sa/metrics + PushMetricsURL string `mapstructure:"instance_metrics_url"` + + // PushLogsURL is the full url of the ServiceNow instance to send push logs to. Ex: http://127.0.0.1:8090/api/mid/hla/raw + PushLogsURL string `mapstructure:"instance_logs_url"` + + // PushEventsURL is the full url of the ServiceNow instance to send push events to. Ex: http://127.0.0.1:8090/api/sn_em_connector/em/inbound_event?source=snotel + PushEventsURL string `mapstructure:"instance_events_url"` + + // ApiKey is used to set an Authorization header with a bearer token + ApiKey configopaque.String `mapstructure:"api_key"` + + // InsecureSkipVerify disables TLS certificate verification + InsecureSkipVerify bool `mapstructure:"insecure_skip_verify"` + + // Username is used to optionally specify the basic auth username + Username string `mapstructure:"username"` + // Password is used to optionally specify the basic auth password + Password configopaque.String `mapstructure:"password"` +} + +func createDefaultConfig() component.Config { + return &Config{ + PushMetricsURL: "http://localhost:8090/api/mid/sa/metrics", + PushLogsURL: "", + PushEventsURL: "http://localhost:8090/api/sn_em_connector/em/inbound_event?source=snotel", + InsecureSkipVerify: false, + TimeoutSettings: exporterhelper.NewDefaultTimeoutSettings(), + BackOffConfig: configretry.NewDefaultBackOffConfig(), + QueueSettings: exporterhelper.NewDefaultQueueSettings(), + } +} diff --git a/collector/components/servicenowexporter/factory.go b/collector/components/servicenowexporter/factory.go index c4ffc06..3218c20 100644 --- a/collector/components/servicenowexporter/factory.go +++ b/collector/components/servicenowexporter/factory.go @@ -5,24 +5,37 @@ import ( "errors" "fmt" + "github.com/lightstep/sn-collector/collector/servicenowexporter/internal/metadata" + "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/exporter" "go.opentelemetry.io/collector/exporter/exporterhelper" - "go.opentelemetry.io/collector/pdata/plog" -) - -const ( - typeStr = "servicenowexporter" - stability = component.StabilityLevelAlpha ) -var errInvalidConfig = errors.New("invalid config for tcpstatsreceiver") +var errInvalidConfig = errors.New("invalid config for servicenowexporter") -type Config struct { -} +func createMetricsExporter( + ctx context.Context, + set exporter.CreateSettings, + cfg component.Config, +) (exporter.Metrics, error) { + if err := component.ValidateConfig(cfg); err != nil { + return nil, fmt.Errorf("cannot configure servicenow metrics exporter: %w", err) + } + oCfg := cfg.(*Config) + me := newServiceNowProducer(set.Logger, oCfg) -func createDefaultConfig() component.Config { - return &Config{} + return exporterhelper.NewMetricsExporter( + ctx, + set, + cfg, + me.metricsDataPusher, + // disable timeout + exporterhelper.WithTimeout(exporterhelper.TimeoutSettings{Timeout: 0}), + exporterhelper.WithRetry(oCfg.BackOffConfig), + exporterhelper.WithQueue(oCfg.QueueSettings), + exporterhelper.WithShutdown(me.Close), + ) } func createLogsExporter( @@ -31,20 +44,29 @@ func createLogsExporter( cfg component.Config, ) (exporter.Logs, error) { if err := component.ValidateConfig(cfg); err != nil { - return nil, fmt.Errorf("cannot configure servicenow logs exporter: %w", err) + return nil, fmt.Errorf("cannot configure servicenow metrics exporter: %w", err) } + oCfg := cfg.(*Config) + me := newServiceNowProducer(set.Logger, oCfg) + return exporterhelper.NewLogsExporter( ctx, set, cfg, - func(_ context.Context, _ plog.Logs) error { return nil }, + me.logDataPusher, + // disable timeout + exporterhelper.WithTimeout(exporterhelper.TimeoutSettings{Timeout: 0}), + exporterhelper.WithRetry(oCfg.BackOffConfig), + exporterhelper.WithQueue(oCfg.QueueSettings), + exporterhelper.WithShutdown(me.Close), ) } func NewFactory() exporter.Factory { return exporter.NewFactory( - typeStr, + metadata.Type, createDefaultConfig, - exporter.WithLogs(createLogsExporter, stability), + exporter.WithMetrics(createMetricsExporter, metadata.MetricsStability), + exporter.WithLogs(createLogsExporter, metadata.LogsStability), ) } diff --git a/collector/components/servicenowexporter/factory_test.go b/collector/components/servicenowexporter/factory_test.go new file mode 100644 index 0000000..8c1951d --- /dev/null +++ b/collector/components/servicenowexporter/factory_test.go @@ -0,0 +1,14 @@ +package servicenowexporter + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFactory(t *testing.T) { + f := NewFactory() + assert.EqualValues(t, "servicenow", f.Type()) + cfg := f.CreateDefaultConfig() + assert.NotNil(t, cfg) +} diff --git a/collector/components/servicenowexporter/go.mod b/collector/components/servicenowexporter/go.mod index ded06d0..ec6a850 100644 --- a/collector/components/servicenowexporter/go.mod +++ b/collector/components/servicenowexporter/go.mod @@ -1,11 +1,47 @@ -module github.com/lightstep/sn-collector/components/servicenowexporter +module github.com/lightstep/sn-collector/collector/servicenowexporter -go 1.19 +go 1.20 require ( - go.opentelemetry.io/collector/component v0.81.0 - go.opentelemetry.io/collector/consumer v0.81.0 - go.opentelemetry.io/collector/pdata v1.0.0-rcv0013 - go.opentelemetry.io/collector/receiver v0.81.0 - go.uber.org/zap v1.24.0 -) \ No newline at end of file + github.com/stretchr/testify v1.8.4 + go.opentelemetry.io/collector/component v0.95.0 + go.opentelemetry.io/collector/config/configopaque v1.2.0 + go.opentelemetry.io/collector/config/configretry v0.95.0 + go.opentelemetry.io/collector/exporter v0.95.0 + go.opentelemetry.io/collector/pdata v1.2.0 + go.opentelemetry.io/otel/metric v1.23.1 + go.opentelemetry.io/otel/trace v1.23.1 + go.uber.org/zap v1.26.0 +) + +require ( + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.1.0 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.opentelemetry.io/collector v0.95.0 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.95.0 // indirect + go.opentelemetry.io/collector/confmap v0.95.0 // indirect + go.opentelemetry.io/collector/consumer v0.95.0 // indirect + go.opentelemetry.io/collector/extension v0.95.0 // indirect + go.opentelemetry.io/otel v1.23.1 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/sys v0.17.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect + google.golang.org/grpc v1.61.0 // indirect + google.golang.org/protobuf v1.32.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/collector/components/servicenowexporter/go.sum b/collector/components/servicenowexporter/go.sum new file mode 100644 index 0000000..ffe140b --- /dev/null +++ b/collector/components/servicenowexporter/go.sum @@ -0,0 +1,151 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsMBaUOKXq6HSv655U1c= +github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= +github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= +github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= +github.com/knadh/koanf/v2 v2.1.0 h1:eh4QmHHBuU8BybfIJ8mB8K8gsGCD/AUQTdwGq/GzId8= +github.com/knadh/koanf/v2 v2.1.0/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c h1:cqn374mizHuIWj+OSJCajGr/phAmuMug9qIX3l9CflE= +github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= +github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/collector v0.95.0 h1:DFW0BkF2sOocpA3NUPrbMeuPSN3PWxFBrLqs/Cxn3vo= +go.opentelemetry.io/collector v0.95.0/go.mod h1:Lc+VkOkSBppKqR/cAevS5oPrbtUO3JUbYzo91niRkG0= +go.opentelemetry.io/collector/component v0.95.0 h1:68tI7KVy1bfpaR83+LxUvjd9/hjDh78utklGl2t6eVM= +go.opentelemetry.io/collector/component v0.95.0/go.mod h1:SMt7r9zm0OOEKJF/ZUy8agD92OAXq2Xhq1FaTcfIWHw= +go.opentelemetry.io/collector/config/configopaque v1.2.0 h1:ncnAuq4px3yREsirivGUbwr36xXEKa3K6JTOBNGlbtc= +go.opentelemetry.io/collector/config/configopaque v1.2.0/go.mod h1:6BAnSe6wok2Sg3tiNuapBbLnrduyMwzsBzbfgUSbDnI= +go.opentelemetry.io/collector/config/configretry v0.95.0 h1:YBLly9WRjLCnB91feTshPNCj3z91Yf+akLWRNiUNxps= +go.opentelemetry.io/collector/config/configretry v0.95.0/go.mod h1:Nq7hp4nk+zeH0LYYsx348NHl02O89FnV45hcCCmqdtg= +go.opentelemetry.io/collector/config/configtelemetry v0.95.0 h1:HabJZqbOAbNQ52L3v6usXoGXg1UKA1Ofs4Ytp5sGXEo= +go.opentelemetry.io/collector/config/configtelemetry v0.95.0/go.mod h1:tl8sI2RE3LSgJ0HjpadYpIwsKzw/CRA0nZUXLzMAZS0= +go.opentelemetry.io/collector/confmap v0.95.0 h1:0oZwSUaeKTDCP7eewFpQSD+9SxXspiaJWjZDQYGGars= +go.opentelemetry.io/collector/confmap v0.95.0/go.mod h1:L3djzwpt+jL06wxnHAuy1jPUFcM+MdKGQAsz3B1d6pk= +go.opentelemetry.io/collector/consumer v0.95.0 h1:M/N5RDx8/6Hz5L1qWUXdtirtdoV8BEjIxCSdt6cCx+I= +go.opentelemetry.io/collector/consumer v0.95.0/go.mod h1:tM5aOolWS1zAByMbne2xVOkmVvZrF3VEKY6TkxhmkOs= +go.opentelemetry.io/collector/exporter v0.95.0 h1:orUpjMVsD2IYRzrofE0tBhEBGHjr7W4wfltERxnWhP4= +go.opentelemetry.io/collector/exporter v0.95.0/go.mod h1:5wyZskvf4T1Zmm414a7JM1xw8EPhrFZeCe5kCff6VRg= +go.opentelemetry.io/collector/extension v0.95.0 h1:amE7zV/lfJRdCmZ4cqWmvBzZB5aQmIFIQqIuXmhaCjI= +go.opentelemetry.io/collector/extension v0.95.0/go.mod h1:IDt4B5GJxh/uJ/mUWxYZ+eHrJ49k4B8s8gJhuI0TRVI= +go.opentelemetry.io/collector/pdata v1.2.0 h1:N6VdyEFYJyoHIKqHd0F372eNVD5b+AbH0ZQf7Z2jJ9I= +go.opentelemetry.io/collector/pdata v1.2.0/go.mod h1:mKXb6527Syb8PT4P9CZOJNbkuHOHjjGTZNNwSKESJhc= +go.opentelemetry.io/collector/receiver v0.95.0 h1:9gA8/ceXiYiPheGQUt+EmCEKTjkGpUdPkiG7iey3Md0= +go.opentelemetry.io/collector/receiver v0.95.0/go.mod h1:kQrMBxcrgZfmtvjVQa6jStYG7c2L1c8UiHe/JNb7M+E= +go.opentelemetry.io/otel v1.23.1 h1:Za4UzOqJYS+MUczKI320AtqZHZb7EqxO00jAHE0jmQY= +go.opentelemetry.io/otel v1.23.1/go.mod h1:Td0134eafDLcTS4y+zQ26GE8u3dEuRBiBCTUIRHaikA= +go.opentelemetry.io/otel/exporters/prometheus v0.45.2 h1:pe2Jqk1K18As0RCw7J08QhgXNqr+6npx0a5W4IgAFA8= +go.opentelemetry.io/otel/exporters/prometheus v0.45.2/go.mod h1:B38pscHKI6bhFS44FDw0eFU3iqG3ASNIvY+fZgR5sAc= +go.opentelemetry.io/otel/metric v1.23.1 h1:PQJmqJ9u2QaJLBOELl1cxIdPcpbwzbkjfEyelTl2rlo= +go.opentelemetry.io/otel/metric v1.23.1/go.mod h1:mpG2QPlAfnK8yNhNJAxDZruU9Y1/HubbC+KyH8FaCWI= +go.opentelemetry.io/otel/sdk v1.23.1 h1:O7JmZw0h76if63LQdsBMKQDWNb5oEcOThG9IrxscV+E= +go.opentelemetry.io/otel/sdk v1.23.1/go.mod h1:LzdEVR5am1uKOOwfBWFef2DCi1nu3SA8XQxx2IerWFk= +go.opentelemetry.io/otel/sdk/metric v1.23.1 h1:T9/8WsYg+ZqIpMWwdISVVrlGb/N0Jr1OHjR/alpKwzg= +go.opentelemetry.io/otel/sdk/metric v1.23.1/go.mod h1:8WX6WnNtHCgUruJ4TJ+UssQjMtpxkpX0zveQC8JG/E0= +go.opentelemetry.io/otel/trace v1.23.1 h1:4LrmmEd8AU2rFvU1zegmvqW7+kWarxtNOPyeL6HmYY8= +go.opentelemetry.io/otel/trace v1.23.1/go.mod h1:4IpnpJFwr1mo/6HL8XIPJaE9y0+u1KcVmuW7dwFSVrI= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 h1:6G8oQ016D88m1xAKljMlBOOGWDZkes4kMhgGFlf8WcQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917/go.mod h1:xtjpI3tXFPP051KaWnhvxkiubL/6dJ18vLVf7q2pTOU= +google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0= +google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/collector/components/servicenowexporter/internal/metadata/generated_status.go b/collector/components/servicenowexporter/internal/metadata/generated_status.go new file mode 100644 index 0000000..b194764 --- /dev/null +++ b/collector/components/servicenowexporter/internal/metadata/generated_status.go @@ -0,0 +1,23 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" +) + +const ( + Type = "servicenow" + MetricsStability = component.StabilityLevelAlpha + LogsStability = component.StabilityLevelAlpha +) + +func Meter(settings component.TelemetrySettings) metric.Meter { + return settings.MeterProvider.Meter("otelcol/servicenow") +} + +func Tracer(settings component.TelemetrySettings) trace.Tracer { + return settings.TracerProvider.Tracer("otelcol/servicenow") +} diff --git a/collector/components/servicenowexporter/metadata.yaml b/collector/components/servicenowexporter/metadata.yaml new file mode 100644 index 0000000..2b4de10 --- /dev/null +++ b/collector/components/servicenowexporter/metadata.yaml @@ -0,0 +1,5 @@ +type: servicenow +status: + class: exporter + stability: + alpha: [metrics, logs] \ No newline at end of file diff --git a/collector/components/servicenowexporter/servicenow.go b/collector/components/servicenowexporter/servicenow.go new file mode 100644 index 0000000..00d7583 --- /dev/null +++ b/collector/components/servicenowexporter/servicenow.go @@ -0,0 +1,471 @@ +package servicenowexporter + +import ( + "bytes" + "context" + "encoding/json" + "strconv" + "strings" + + "go.uber.org/zap" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/pdata/pmetric" +) + +const ( + // sanitizedRune is used to replace any invalid char per Carbon format. + sanitizedRune = '_' + + // Tag related constants per Carbon plaintext protocol. + tagPrefix = ";" + tagKeyValueSeparator = "=" + tagValueEmptyPlaceholder = "" + + // Constants used when converting from distribution metrics to Carbon format. + distributionBucketSuffix = ".bucket" + distributionUpperBoundTagKey = "upper_bound" + distributionUpperBoundTagBeforeValue = tagPrefix + distributionUpperBoundTagKey + tagKeyValueSeparator + + // Constants used when converting from summary metrics to Carbon format. + summaryQuantileSuffix = ".quantile" + summaryQuantileTagKey = "quantile" + summaryQuantileTagBeforeValue = tagPrefix + summaryQuantileTagKey + tagKeyValueSeparator + + // Suffix to be added to original metric name for a Carbon metric representing + // a count metric for either distribution or summary metrics. + countSuffix = ".count" + + // Textual representation for positive infinity valid in Carbon, ie.: + // positive infinity as represented in Python. + infinityCarbonValue = "inf" + + midSource = "sn-otel-collector" +) + +type serviceNowProducer struct { + logger *zap.Logger + config *Config + client *midClient +} + +func newServiceNowProducer(logger *zap.Logger, config *Config) *serviceNowProducer { + return &serviceNowProducer{ + logger: logger, + config: config, + client: newMidClient(config, logger), + } +} + +func (e *serviceNowProducer) logDataPusher(_ context.Context, md plog.Logs) error { + snLogs := make([]ServiceNowLog, 0) + snEvents := make([]ServiceNowEvent, 0) + + useLogs := e.config.PushLogsURL != "" + + for i := 0; i < md.ResourceLogs().Len(); i++ { + rl := md.ResourceLogs().At(i) + resourceAttrs := rl.Resource().Attributes() + for j := 0; j < rl.ScopeLogs().Len(); j++ { + sl := rl.ScopeLogs().At(j) + scope := sl.Scope().Name() + + for k := 0; k < sl.LogRecords().Len(); k++ { + log := sl.LogRecords().At(k) + if useLogs { + newLog := ServiceNowLog{ + Body: log.Body().AsString(), + ResourcePath: buildPath("", log.Attributes()), + Ci2LogID: ci2metricAttrs(resourceAttrs), + Timestamp: formatTimestamp(log.Timestamp()), + Severity: log.SeverityText(), + Node: formatNode(ci2metricAttrs(resourceAttrs)), + Source: midSource, + } + snLogs = append(snLogs, newLog) + } else { + additionalInfo, err := formatAdditionalInfo(ci2metricAttrs(log.Attributes()), ci2metricAttrs(resourceAttrs)) + if err != nil { + e.logger.Error("Failed to format additional info", zap.Error(err)) + continue + } + + newEvent := ServiceNowEvent{ + Type: scope, + Description: log.Body().AsString(), + Resource: buildPath("", log.Attributes()), + Severity: "5", // TODO: figure out this mapping + Timestamp: formatEventTimestamp(log.Timestamp()), + Node: formatNode(ci2metricAttrs(resourceAttrs)), + Source: midSource, + AdditionalInfo: additionalInfo, + } + snEvents = append(snEvents, newEvent) + } + } + } + } + + if useLogs { + e.logger.Info("Sending logs to MID Server...", zap.Any("logs", snLogs)) + err := e.client.sendLogs(snLogs) + if err != nil { + e.logger.Error("Failed to send logs to MID Server", zap.Int("logCount", len(snLogs)), zap.Error(err)) + return err + } + + return nil + } + + e.logger.Info("Sending events to instance...", zap.Any("events", snEvents)) + err := e.client.sendEvents(snEvents) + if err != nil { + e.logger.Error("Failed to send events to MID Server", zap.Int("logCount", len(snEvents)), zap.Error(err)) + return err + } + + return nil +} + +// based on: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/carbonexporter/metricdata_to_plaintext.go#L82 +func (e *serviceNowProducer) metricsDataPusher(_ context.Context, md pmetric.Metrics) error { + snMetrics := make([]ServiceNowMetric, 0) + + for i := 0; i < md.ResourceMetrics().Len(); i++ { + rm := md.ResourceMetrics().At(i) + resourceAttrs := rm.Resource().Attributes() + for j := 0; j < rm.ScopeMetrics().Len(); j++ { + sm := rm.ScopeMetrics().At(j) + scope := sm.Scope().Name() + + for k := 0; k < sm.Metrics().Len(); k++ { + metric := sm.Metrics().At(k) + if metric.Name() == "" { + // TODO: log error info + continue + } + switch metric.Type() { + case pmetric.MetricTypeGauge: + snMetrics = append(snMetrics, e.writeNumberDataPoints(metric.Name(), scope, resourceAttrs, metric.Gauge().DataPoints())...) + case pmetric.MetricTypeSum: + snMetrics = append(snMetrics, e.writeNumberDataPoints(metric.Name(), scope, resourceAttrs, metric.Sum().DataPoints())...) + case pmetric.MetricTypeHistogram: + snMetrics = append(snMetrics, e.formatHistogramDataPoints(metric.Name(), scope, resourceAttrs, metric.Histogram().DataPoints())...) + case pmetric.MetricTypeSummary: + snMetrics = append(snMetrics, e.formatSummaryDataPoints(metric.Name(), scope, resourceAttrs, metric.Summary().DataPoints())...) + } + } + } + } + + e.logger.Info("Sending metrics to MID Server...", zap.Any("metrics", len(snMetrics))) + err := e.client.sendMetrics(snMetrics) + + if err != nil { + e.logger.Error("Failed to send metric to MID Server", zap.Int("metricCount", len(snMetrics)), zap.Error(err)) + return err + } + return nil +} + +func (e *serviceNowProducer) Close(context.Context) error { + e.client.Close() + return nil +} + +func (e *serviceNowProducer) writeNumberDataPoints(metricName string, scope string, rAttrs pcommon.Map, dps pmetric.NumberDataPointSlice) []ServiceNowMetric { + snm := make([]ServiceNowMetric, 0) + for i := 0; i < dps.Len(); i++ { + dp := dps.At(i) + var val float64 + switch dp.ValueType() { + case pmetric.NumberDataPointValueTypeEmpty: + continue // skip this data point - otherwise an empty string will be used as the value and the backend will use the timestamp as the metric value + case pmetric.NumberDataPointValueTypeInt: + val = float64(dp.IntValue()) + case pmetric.NumberDataPointValueTypeDouble: + val = float64(dp.DoubleValue()) + } + snm = append(snm, e.createMetric( + metricName, + scope, + ci2metricAttrs(rAttrs), + buildPath(metricName, dp.Attributes()), + val, + formatTimestamp(dp.Timestamp()))) + } + return snm +} + +// Converts resource attributes to a map of string key/value pairs +// for use in ci2metric_id in the push metric API +func ci2metricAttrs(rAttrs pcommon.Map) map[string]string { + attrs := make(map[string]string) + rAttrs.Range(func(k string, v pcommon.Value) bool { + attrs[k] = v.AsString() + return true + }) + return attrs +} + +// formatHistogramDataPoints transforms a slice of histogram data points into a series +// of Carbon metrics and injects them into the string builder. +// +// Carbon doesn't have direct support to distribution metrics they will be +// translated into a series of Carbon metrics: +// +// 1. The total count will be represented by a metric named ".count". +// +// 2. The total sum will be represented by a metric with the original "". +// +// 3. Each histogram bucket is represented by a metric named ".bucket" +// and will include a dimension "upper_bound" that specifies the maximum value in +// that bucket. This metric specifies the number of events with a value that is +// less than or equal to the upper bound. +func (e *serviceNowProducer) formatHistogramDataPoints( + metricName string, + scope string, + rAttrs pcommon.Map, + dps pmetric.HistogramDataPointSlice, +) []ServiceNowMetric { + snm := make([]ServiceNowMetric, 0) + + for i := 0; i < dps.Len(); i++ { + dp := dps.At(i) + + e.formatCountAndSum(metricName, scope, rAttrs, dp.Attributes(), dp.Count(), dp.Sum(), dp.Timestamp()) + if dp.ExplicitBounds().Len() == 0 { + continue + } + + bounds := dp.ExplicitBounds().AsRaw() + carbonBounds := make([]string, len(bounds)+1) + for i := 0; i < len(bounds); i++ { + carbonBounds[i] = formatFloatForLabel(bounds[i]) + } + carbonBounds[len(carbonBounds)-1] = infinityCarbonValue + + bucketPath := buildPath(metricName+distributionBucketSuffix, dp.Attributes()) + for j := 0; j < dp.BucketCounts().Len(); j++ { + snm = append(snm, e.createMetric( + metricName+distributionBucketSuffix, + scope, + ci2metricAttrs(rAttrs), + bucketPath+distributionUpperBoundTagBeforeValue+carbonBounds[j], + float64(dp.BucketCounts().At(j)), + formatTimestamp(dp.Timestamp()))) + } + } + return snm +} + +// formatSummaryDataPoints transforms a slice of summary data points into a series +// of Carbon metrics and injects them into the string builder. +// +// Carbon doesn't have direct support to summary metrics they will be +// translated into a series of Carbon metrics: +// +// 1. The total count will be represented by a metric named ".count". +// +// 2. The total sum will be represented by a metric with the original "". +// +// 3. Each quantile is represented by a metric named ".quantile" +// and will include a tag key "quantile" that specifies the quantile value. +func (e *serviceNowProducer) formatSummaryDataPoints( + metricName string, + scope string, + rAttrs pcommon.Map, + dps pmetric.SummaryDataPointSlice, +) []ServiceNowMetric { + snm := make([]ServiceNowMetric, 0) + for i := 0; i < dps.Len(); i++ { + dp := dps.At(i) + + e.formatCountAndSum(metricName, scope, rAttrs, dp.Attributes(), dp.Count(), dp.Sum(), dp.Timestamp()) + + if dp.QuantileValues().Len() == 0 { + continue + } + + quantilePath := buildPath(metricName+summaryQuantileSuffix, dp.Attributes()) + for j := 0; j < dp.QuantileValues().Len(); j++ { + snm = append(snm, e.createMetric( + metricName+summaryQuantileSuffix, + scope, + ci2metricAttrs(rAttrs), + quantilePath+summaryQuantileTagBeforeValue+formatFloatForLabel(dp.QuantileValues().At(j).Quantile()*100), + dp.QuantileValues().At(j).Value(), + formatTimestamp(dp.Timestamp()))) + } + } + return snm +} + +// Carbon doesn't have direct support to distribution or summary metrics in both +// cases it needs to create a "count" and a "sum" metric. This function creates +// both, as follows: +// +// 1. The total count will be represented by a metric named ".count". +// +// 2. The total sum will be represented by a metruc with the original "". +func (e *serviceNowProducer) formatCountAndSum( + metricName string, + scope string, + rAttrs pcommon.Map, + attributes pcommon.Map, + count uint64, + sum float64, + timestamp pcommon.Timestamp, +) []ServiceNowMetric { + snm := make([]ServiceNowMetric, 0, 2) + // Write count and sum metrics. + snm = append(snm, e.createMetric( + metricName, + scope, + ci2metricAttrs(rAttrs), + buildPath(metricName+countSuffix, attributes), + float64(count), + formatTimestamp(timestamp))) + + snm = append(snm, e.createMetric( + metricName, + scope, + ci2metricAttrs(rAttrs), + buildPath(metricName, attributes), + sum, + formatTimestamp(timestamp))) + return snm +} + +// buildPath is used to build the per description above. +func buildPath(name string, attributes pcommon.Map) string { + if attributes.Len() == 0 { + return name + } + + buf := new(bytes.Buffer) + + buf.WriteString(name) + attributes.Range(func(k string, v pcommon.Value) bool { + value := v.AsString() + if value == "" { + value = tagValueEmptyPlaceholder + } + buf.WriteString(tagPrefix) + buf.WriteString(sanitizeTagKey(k)) + buf.WriteString(tagKeyValueSeparator) + buf.WriteString(value) + return true + }) + + return buf.String() +} + +func formatAdditionalInfo(attrs map[string]string, resourceAttrs map[string]string) (string, error) { + // merge attrs + resource attrs + newAttrs := make(map[string]string) + for k, v := range resourceAttrs { + newAttrs[k] = v + } + + for k, v := range attrs { + newAttrs[k] = v + } + + // convert resoruceAttrs + bytes, err := json.Marshal(newAttrs) + if err != nil { + return "", err + } + return string(bytes), nil +} + +func formatNode(resourceAttrs map[string]string) string { + // TODO: make this mapping support more than host.name + return resourceAttrs["host.name"] +} + +func (e *serviceNowProducer) createMetric(name string, scope string, resourceAttrs map[string]string, path string, value float64, timestamp uint64) ServiceNowMetric { + if scope != "" { + resourceAttrs["otel.scope"] = scope + } + + snm := ServiceNowMetric{ + MetricType: name, + ResourcePath: path, + Value: value, + Timestamp: timestamp, + Source: midSource, + Ci2MetricID: resourceAttrs, + } + + // set by a processor (does not exist yet) + ciClass := resourceAttrs["servicenow.ci.sys_id"] + if ciClass != "" { + snm.CiSysId = ciClass + snm.Ci2MetricID = nil + } + + snm.Node = formatNode(resourceAttrs) + + return snm +} + +// sanitizeTagKey removes any invalid character from the tag key, the invalid +// characters are ";!^=". +func sanitizeTagKey(key string) string { + mapRune := func(r rune) rune { + switch r { + case ';', '!', '^', '=': + return sanitizedRune + default: + return r + } + } + + return strings.Map(mapRune, key) +} + +// sanitizeTagValue removes any invalid character from the tag value, the invalid +// characters are ";~". +func sanitizeTagValue(value string) string { + mapRune := func(r rune) rune { + switch r { + case ';', '~': + return sanitizedRune + default: + return r + } + } + + return strings.Map(mapRune, value) +} + +// Formats a float64 per Prometheus label value. This is an attempt to keep other +// the label values with different formats of metrics. +func formatFloatForLabel(f float64) string { + return strconv.FormatFloat(f, 'g', -1, 64) +} + +// Formats a float64 per Carbon plaintext format. +func formatFloatForValue(f float64) string { + return strconv.FormatFloat(f, 'f', -1, 64) +} + +func formatUint64(i uint64) string { + return strconv.FormatUint(i, 10) +} + +func formatInt64(i int64) string { + return strconv.FormatInt(i, 10) +} + +func formatEventTimestamp(timestamp pcommon.Timestamp) string { + ts := timestamp.AsTime() + return ts.Format("2006-01-02 15:04:05") +} + +func formatTimestamp(timestamp pcommon.Timestamp) uint64 { + return uint64(timestamp) / 1e6 +} diff --git a/collector/components/servicenowexporter/servicenow_event.go b/collector/components/servicenowexporter/servicenow_event.go new file mode 100644 index 0000000..9acbc98 --- /dev/null +++ b/collector/components/servicenowexporter/servicenow_event.go @@ -0,0 +1,17 @@ +package servicenowexporter + +type ServiceNowEventRequestBody struct { + Records []ServiceNowEvent `json:"records"` +} + +// https://docs.servicenow.com/bundle/vancouver-it-operations-management/page/product/event-management/task/send-events-via-web-service.html +type ServiceNowEvent struct { + Resource string `json:"resource"` + Node string `json:"node"` + Severity string `json:"severity"` + Type string `json:"type"` + Description string `json:"description"` + Timestamp string `json:"time_of_event"` // yyyy-MM-dd HH:mm:ss + AdditionalInfo string `json:"additional_info,omitempty"` // actually a json string + Source string `json:"source"` +} diff --git a/collector/components/servicenowexporter/servicenow_log.go b/collector/components/servicenowexporter/servicenow_log.go new file mode 100644 index 0000000..bce788a --- /dev/null +++ b/collector/components/servicenowexporter/servicenow_log.go @@ -0,0 +1,13 @@ +package servicenowexporter + +// https://docs.servicenow.com/bundle/vancouver-it-operations-management/page/product/health-log-analytics-admin/task/hla-data-input-rest-api.html +type ServiceNowLog struct { + ResourcePath string `json:"resource_path"` + Node string `json:"node"` + Body string `json:"body"` + CiSysId string `json:"ci,omitempty"` + Timestamp uint64 `json:"timestamp"` + Severity string `json:"severity"` + Ci2LogID map[string]string `json:"ci2log_id,omitempty"` + Source string `json:"source"` +} diff --git a/collector/components/servicenowexporter/servicenow_metric.go b/collector/components/servicenowexporter/servicenow_metric.go new file mode 100644 index 0000000..60fb7f9 --- /dev/null +++ b/collector/components/servicenowexporter/servicenow_metric.go @@ -0,0 +1,14 @@ +package servicenowexporter + +// https://docs.servicenow.com/bundle/vancouver-api-reference/page/integrate/inbound-rest/concept/push-metrics-MID-server.html +// https://support.servicenow.com/kb?id=kb_article_view&sysparm_article=KB0853084 +type ServiceNowMetric struct { + MetricType string `json:"metric_type"` + ResourcePath string `json:"resource_path"` + Node string `json:"node"` + CiSysId string `json:"ci,omitempty"` + Value float64 `json:"value"` + Timestamp uint64 `json:"timestamp"` + Ci2MetricID map[string]string `json:"ci2metric_id,omitempty"` + Source string `json:"source"` +} diff --git a/collector/config/otelcol-docker-hostmetrics.yaml b/collector/config/otelcol-docker-hostmetrics.yaml new file mode 100644 index 0000000..13aca8a --- /dev/null +++ b/collector/config/otelcol-docker-hostmetrics.yaml @@ -0,0 +1,88 @@ +# +# Basic OpenTelemetry host metrics collection running inside a Docker container. +# Validated with v0.95.0 on 2/21/24 +# +receivers: + prometheus/self: + config: + scrape_configs: + - job_name: otel-collector + scrape_interval: 5s + static_configs: + - labels: + collector_name: sn-collector + targets: + - 0.0.0.0:8888 + hostmetrics: + # needed to read host metrics from Docker cntainer + # host path ostmust be mounted to the container + root_path: /hostfs + collection_interval: 5s + scrapers: + cpu: + disk: + load: + processes: + filesystem: + memory: + paging: + network: + +processors: + batch: + send_batch_size: 1000 + send_batch_max_size: 1500 + timeout: 1s + resourcedetection: + detectors: ["system"] + +exporters: + # Send to Cloud Observability + # Create an access token at https://docs.lightstep.com/docs/create-and-manage-access-tokens + otlp/lightstep: + endpoint: ingest.lightstep.com:443 + headers: + 'lightstep-access-token': '${LS_TOKEN}' + sending_queue: + enabled: true + num_consumers: 4 + queue_size: 100 + # Send to HLA + servicenow/logs: + instance_logs_url: ${MID_INSTANCE_LOGS_URL} + insecure_skip_verify: true + api_key: ${MID_INSTANCE_LOGS_API_KEY} + # Send to Metric Intelligence + servicenow/metrics: + instance_metrics_url: ${MID_INSTANCE_METRICS_URL} + username: ${MID_INSTANCE_USERNAME} + password: ${MID_INSTANCE_PASSWORD} + # Send to Event Management + servicenow/events: + instance_events_url: ${MID_INSTANCE_EVENTS_URL} + username: ${MID_INSTANCE_USERNAME} + password: ${MID_INSTANCE_PASSWORD} + debug/detailed: + verbosity: detailed + debug: + verbosity: normal + +extensions: + health_check: + opamp: + server: + ws: + endpoint: https://opamp.lightstep.com/v1/opamp + headers: + "Authorization": "Bearer ${LS_OPAMP_API_KEY}" + +service: + pipelines: + metrics/collector: + receivers: [prometheus/self] + processors: [batch, resourcedetection] + exporters: [debug] + metrics/host: + receivers: [hostmetrics] + processors: [batch, resourcedetection] + exporters: [debug] diff --git a/collector/config/otelcol-linux-hostmetrics.yaml b/collector/config/otelcol-linux-hostmetrics.yaml new file mode 100644 index 0000000..0fef7a4 --- /dev/null +++ b/collector/config/otelcol-linux-hostmetrics.yaml @@ -0,0 +1,81 @@ +# +# Basic OpenTelemetry host metrics for Linux (non-Dockerized). +# Validated with v0.95.0 on 2/21/24 +# +receivers: + prometheus/self: + config: + scrape_configs: + - job_name: otel-collector + scrape_interval: 5s + static_configs: + - labels: + collector_name: sn-collector + targets: + - 0.0.0.0:8888 + hostmetrics: + collection_interval: 5s + scrapers: + cpu: + disk: + load: + processes: + filesystem: + memory: + paging: + network: + +processors: + batch: + resourcedetection: + detectors: ["system"] + +exporters: + # Send to Cloud Observability + # Create an access token at https://docs.lightstep.com/docs/create-and-manage-access-tokens + otlp/lightstep: + endpoint: ingest.lightstep.com:443 + headers: + 'lightstep-access-token': '${LS_TOKEN}' + sending_queue: + enabled: true + num_consumers: 4 + queue_size: 100 + # Send to HLA + servicenow/logs: + instance_logs_url: ${MID_INSTANCE_LOGS_URL} + insecure_skip_verify: true + api_key: ${MID_INSTANCE_LOGS_API_KEY} + # Send to Metric Intelligence + servicenow/metrics: + instance_metrics_url: ${MID_INSTANCE_METRICS_URL} + username: ${MID_INSTANCE_USERNAME} + password: ${MID_INSTANCE_PASSWORD} + # Send to Event Management + servicenow/events: + instance_events_url: ${MID_INSTANCE_EVENTS_URL} + username: ${MID_INSTANCE_USERNAME} + password: ${MID_INSTANCE_PASSWORD} + debug/detailed: + verbosity: detailed + debug: + verbosity: normal + +extensions: + health_check: + opamp: + server: + ws: + endpoint: https://opamp.lightstep.com/v1/opamp + headers: + "Authorization": "Bearer ${LS_OPAMP_API_KEY}" +service: + pipelines: + metrics/collector: + receivers: [prometheus/self] + processors: [batch, resourcedetection] + exporters: [debug] + metrics/host: + receivers: [hostmetrics] + processors: [batch, resourcedetection] + exporters: [debug] diff --git a/collector/config/otelcol-macos-hostmetrics.yaml b/collector/config/otelcol-macos-hostmetrics.yaml new file mode 100644 index 0000000..5720069 --- /dev/null +++ b/collector/config/otelcol-macos-hostmetrics.yaml @@ -0,0 +1,81 @@ +# +# Basic OpenTelemetry host metrics for macOS. +# Validated with v0.95.0 on 2/21/24 +# +receivers: + prometheus/self: + config: + scrape_configs: + - job_name: otel-collector + scrape_interval: 5s + static_configs: + - labels: + collector_name: sn-collector + targets: + - 0.0.0.0:8888 + hostmetrics: + collection_interval: 5s + # cpu and disk not supported by macOS + scrapers: + load: + processes: + filesystem: + memory: + paging: + network: + +processors: + batch: + resourcedetection: + detectors: ["system"] + +exporters: + # Send to Cloud Observability + # Create an access token at https://docs.lightstep.com/docs/create-and-manage-access-tokens + otlp/lightstep: + endpoint: ingest.lightstep.com:443 + headers: + 'lightstep-access-token': '${LS_TOKEN}' + sending_queue: + enabled: true + num_consumers: 4 + queue_size: 100 + # Send to HLA + servicenow/logs: + instance_logs_url: ${MID_INSTANCE_LOGS_URL} + insecure_skip_verify: true + api_key: ${MID_INSTANCE_LOGS_API_KEY} + # Send to Metric Intelligence + servicenow/metrics: + instance_metrics_url: ${MID_INSTANCE_METRICS_URL} + username: ${MID_INSTANCE_USERNAME} + password: ${MID_INSTANCE_PASSWORD} + # Send to Event Management + servicenow/events: + instance_events_url: ${MID_INSTANCE_EVENTS_URL} + username: ${MID_INSTANCE_USERNAME} + password: ${MID_INSTANCE_PASSWORD} + debug/detailed: + verbosity: detailed + debug: + verbosity: normal + +extensions: + health_check: + opamp: + server: + ws: + endpoint: https://opamp.lightstep.com/v1/opamp + headers: + "Authorization": "Bearer ${LS_OPAMP_API_KEY}" + +service: + pipelines: + metrics/collector: + receivers: [prometheus/self] + processors: [batch, resourcedetection] + exporters: [debug] + metrics/host: + receivers: [hostmetrics] + processors: [batch, resourcedetection] + exporters: [debug] diff --git a/collector/config/otelcol-windows-hostmetrics.yaml b/collector/config/otelcol-windows-hostmetrics.yaml new file mode 100644 index 0000000..f4f9e2e --- /dev/null +++ b/collector/config/otelcol-windows-hostmetrics.yaml @@ -0,0 +1,81 @@ +# +# Basic OpenTelemetry host metrics for Windows (non-Dockerized). +# Validated with v0.95.0 on 2/21/24 +# +receivers: + prometheus/self: + config: + scrape_configs: + - job_name: otel-collector + scrape_interval: 5s + static_configs: + - labels: + collector_name: sn-collector + targets: + - 0.0.0.0:8888 + hostmetrics: + collection_interval: 5s + scrapers: + cpu: + disk: + load: + processes: + filesystem: + memory: + paging: + network: + +processors: + batch: + resourcedetection: + detectors: ["system"] + +exporters: + # Send to Cloud Observability + # Create an access token at https://docs.lightstep.com/docs/create-and-manage-access-tokens + otlp/lightstep: + endpoint: ingest.lightstep.com:443 + headers: + 'lightstep-access-token': '${LS_TOKEN}' + sending_queue: + enabled: true + num_consumers: 4 + queue_size: 100 + # Send to HLA + servicenow/logs: + instance_logs_url: ${MID_INSTANCE_LOGS_URL} + insecure_skip_verify: true + api_key: ${MID_INSTANCE_LOGS_API_KEY} + # Send to Metric Intelligence + servicenow/metrics: + instance_metrics_url: ${MID_INSTANCE_METRICS_URL} + username: ${MID_INSTANCE_USERNAME} + password: ${MID_INSTANCE_PASSWORD} + # Send to Event Management + servicenow/events: + instance_events_url: ${MID_INSTANCE_EVENTS_URL} + username: ${MID_INSTANCE_USERNAME} + password: ${MID_INSTANCE_PASSWORD} + debug/detailed: + verbosity: detailed + debug: + verbosity: normal + +extensions: + health_check: + opamp: + server: + ws: + endpoint: https://opamp.lightstep.com/v1/opamp + headers: + "Authorization": "Bearer ${LS_OPAMP_API_KEY}" +service: + pipelines: + metrics/collector: + receivers: [prometheus/self] + processors: [batch, resourcedetection] + exporters: [debug] + metrics/host: + receivers: [hostmetrics] + processors: [batch, resourcedetection] + exporters: [debug] diff --git a/collector/examples/k8s-deployment.yaml b/collector/examples/k8s-deployment.yaml new file mode 100644 index 0000000..b32b16f --- /dev/null +++ b/collector/examples/k8s-deployment.yaml @@ -0,0 +1,181 @@ +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector +metadata: + name: sn-collector +spec: + mode: deployment + image: ghcr.io/lightstep/sn-collector/sn-collector-experimental:latest-amd64 + imagePullPolicy: Always + env: + # 1/ Edit this to specify your Kubernetes cluster name. + - name: K8S_CLUSTER_NAME + value: "undefined-cluster" + # 2/ Edit this to point to your instance JSON event endpoint. + - name: MID_INSTANCE_EVENTS_URL + value: "https://itompmx.service-now.com/api/global/em/jsonv2" + # 3/ Edit this to use a user with em_event permission. + - name: MID_INSTANCE_EVENTS_USERNAME + value: "cobnow" + # *Do not* directly edit, set as a Kubernetes secret. + - name: MID_INSTANCE_EVENTS_PASSWORD + valueFrom: + secretKeyRef: + key: MID_INSTANCE_EVENTS_PASSWORD + name: mid-instance-events-password + # *Do not* edit directly, set as a Kubernetes secret. + - name: LS_TOKEN + valueFrom: + secretKeyRef: + key: LS_TOKEN + name: ls-token-secret + config: | + receivers: + k8s_events: + k8s_cluster: + node_conditions_to_report: [Ready, MemoryPressure] + + processors: + resource/clustername: + attributes: + - key: k8s.cluster.name + value: ${K8S_CLUSTER_NAME} + action: upsert + + k8sattributes: + passthrough: false + auth_type: "kubeConfig" + extract: + metadata: + - k8s.cluster.uid + - k8s.namespace.name + - k8s.pod.name + - k8s.pod.uid + - k8s.node.name + - k8s.pod.start_time + - k8s.deployment.name + - k8s.replicaset.name + - k8s.replicaset.uid + - k8s.daemonset.name + - k8s.daemonset.uid + - k8s.job.name + - k8s.job.uid + - k8s.cronjob.name + - k8s.statefulset.name + - k8s.statefulset.uid + - container.image.tag + - container.image.name + + exporters: + servicenow/logs: + instance_logs_url: ${MID_INSTANCE_LOGS_URL} + insecure_skip_verify: true + api_key: ${MID_INSTANCE_LOGS_API_KEY} + servicenow/metrics: + instance_metrics_url: ${MID_INSTANCE_METRICS_URL} + username: ${MID_INSTANCE_METRICS_USERNAME} + password: ${MID_INSTANCE_METRICS_PASSWORD} + servicenow/events: + instance_events_url: "${MID_INSTANCE_EVENTS_URL}" + username: "${MID_INSTANCE_EVENTS_USERNAME}" + password: "${MID_INSTANCE_EVENTS_PASSWORD}" + otlp/lightstep: + endpoint: ingest.lightstep.com:443 + headers: + 'lightstep-access-token': '${LS_TOKEN}' + debug/detailed: + verbosity: detailed + debug: + verbosity: normal + + service: + pipelines: + # metrics/in: + # receivers: [k8s_cluster] + # processors: [resource/clustername, k8sattributes] + # exporters: [debug, servicenow/metrics] + logs/k8s_events: + receivers: [k8s_events] + processors: [resource/clustername, k8sattributes] + # Event Management: + exporters: [debug, servicenow/events, otlp/lightstep] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: sn-collector +rules: +- apiGroups: [""] + resources: + - events + - namespaces + - namespaces/status + - nodes + - nodes/stats + - nodes/spec + - nodes/proxy + - nodes/metrics + - services + - resourcequotas + - replicationcontrollers + - replicationcontrollers/status + - endpoints + - persistentvolumes + - persistentvolumeclaims + - pods + - pods/status + verbs: ["get", "list", "watch"] +- apiGroups: ["monitoring.coreos.com"] + resources: + - servicemonitors + - podmonitors + verbs: ["get", "list", "watch"] +- apiGroups: + - extensions + resources: + - ingresses + verbs: ["get", "list", "watch"] +- apiGroups: + - apps + resources: + - daemonsets + - deployments + - replicasets + - statefulsets + verbs: ["get", "list", "watch"] +- apiGroups: + - batch + resources: + - cronjobs + - jobs + verbs: ["get", "list", "watch"] +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: ["get", "list", "watch"] +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: ["get", "list", "watch"] +- apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics", "/metrics/cadvisor"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: sn-collector +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: sn-collector +subjects: +- kind: ServiceAccount + # quirk of the Operator + name: sn-collector-collector + namespace: default +--- \ No newline at end of file diff --git a/collector/examples/otelcol-k8s.yaml b/collector/examples/otelcol-k8s.yaml new file mode 100644 index 0000000..424d05d --- /dev/null +++ b/collector/examples/otelcol-k8s.yaml @@ -0,0 +1,75 @@ +#s +# This is a work in progress. +# Collect k8s metrics via kubeconfig and send to MetricBase +# docker run --rm --name sn-collector-experimental ~/.kube/config:/kube/config -e KUBECONFIG=/kube/config -e MID_INSTANCE_URL=http://172.31.55.194:8090 -e MID_INSTANCE_USERNAME='...' -e MID_INSTANCE_PASSWORD='...' smithclay/sn-collector-experimental +# +receivers: + k8s_events: + auth_type: kubeConfig + + k8s_cluster: + auth_type: kubeConfig + node_conditions_to_report: [Ready, MemoryPressure] + +processors: + resource/clustername: + attributes: + - key: k8s.cluster.name + value: "k8s-test-cluster" + action: upsert + + k8sattributes: + passthrough: false + auth_type: "kubeConfig" + extract: + metadata: + - k8s.cluster.uid + - k8s.namespace.name + - k8s.pod.name + - k8s.pod.uid + - k8s.node.name + - k8s.pod.start_time + - k8s.deployment.name + - k8s.replicaset.name + - k8s.replicaset.uid + - k8s.daemonset.name + - k8s.daemonset.uid + - k8s.job.name + - k8s.job.uid + - k8s.cronjob.name + - k8s.statefulset.name + - k8s.statefulset.uid + - container.image.tag + - container.image.name + +exporters: + servicenow/logs: + instance_logs_url: ${MID_INSTANCE_LOGS_URL} + insecure_skip_verify: true + api_key: ${MID_INSTANCE_LOGS_API_KEY} + servicenow/metrics: + instance_metrics_url: ${MID_INSTANCE_METRICS_URL} + username: ${MID_INSTANCE_USERNAME} + password: ${MID_INSTANCE_PASSWORD} + servicenow/events: + instance_events_url: ${MID_INSTANCE_EVENTS_URL} + username: ${MID_INSTANCE_USERNAME} + password: ${MID_INSTANCE_PASSWORD} + debug/detailed: + verbosity: detailed + debug: + verbosity: normal + +service: + pipelines: + # metrics/in: + # receivers: [k8s_cluster] + # processors: [resource/clustername, k8sattributes] + # exporters: [debug, servicenow/metrics] + logs/k8s_events: + receivers: [k8s_events] + processors: [resource/clustername, k8sattributes] + # HLA: + # exporters: [debug, servicenow/logs] + # Event Management: + exporters: [debug, servicenow/events] \ No newline at end of file diff --git a/collector/otelcol-builder.yaml b/collector/otelcol-builder.yaml index ba7902a..614e0b9 100644 --- a/collector/otelcol-builder.yaml +++ b/collector/otelcol-builder.yaml @@ -2,36 +2,66 @@ dist: name: otelcol-servicenow description: ServiceNow-flavored OpenTelemetry Collector distro output_path: ./otelcol-servicenow - otelcol_version: 0.88.0 + otelcol_version: 0.95.0 + version: 0.0.5 exporters: - gomod: - go.opentelemetry.io/collector/exporter/loggingexporter v0.88.0 + go.opentelemetry.io/collector/exporter/debugexporter v0.95.0 - gomod: - go.opentelemetry.io/collector/exporter/otlpexporter v0.88.0 + go.opentelemetry.io/collector/exporter/otlpexporter v0.95.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/exporter/kafkaexporter v0.95.0 - gomod: "github.com/lightstep/sn-collector/collector/servicenowexporter v0.0.1" name: "servicenowexporter" path: "./components/servicenowexporter" processors: - - gomod: "github.com/lightstep/sn-collector/collector/resourcegraphprocessor v0.0.1" - name: "resourcegraphprocessor" - path: "./components/resourcegraphprocessor" + - gomod: github.com/open-telemetry/otel-arrow/collector v0.17.0 + import: github.com/open-telemetry/otel-arrow/collector/processor/concurrentbatchprocessor + - gomod: + go.opentelemetry.io/collector/processor/batchprocessor v0.95.0 - gomod: - github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.88.0 + github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.95.0 - gomod: - github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.88.0 + github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.95.0 - gomod: - github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.88.0 + github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.95.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.95.0 receivers: - - gomod: "github.com/lightstep/sn-collector/collector/osqueryreceiver v0.0.1" - name: "osqueryreceiver" - path: "./components/osqueryreceiver" + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/osqueryreceiver v0.95.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver v0.95.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/filelogreceiver v0.95.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.95.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/azuremonitorreceiver v0.95.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/azureeventhubreceiver v0.95.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/httpcheckreceiver v0.95.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8seventsreceiver v0.95.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver v0.95.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8sclusterreceiver v0.95.0 + - gomod: + go.opentelemetry.io/collector/receiver/otlpreceiver v0.95.0 + +connectors: - gomod: - go.opentelemetry.io/collector/receiver/otlpreceiver v0.88.0 + github.com/open-telemetry/opentelemetry-collector-contrib/connector/exceptionsconnector v0.95.0 extensions: - - gomod: "github.com/lightstep/sn-collector/collector/resourceapiextension v0.0.1" - name: "resourceapiextension" - path: "./components/resourceapiextension" \ No newline at end of file + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.95.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/extension/opampextension v0.95.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension v0.95.0 \ No newline at end of file diff --git a/collector/otelcol.yaml b/collector/otelcol.yaml deleted file mode 100644 index 58296ea..0000000 --- a/collector/otelcol.yaml +++ /dev/null @@ -1,35 +0,0 @@ -receivers: - osqueryreceiver: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - -processors: - resourcedetection: - detectors: ["system"] - resource: - attributes: - - key: host.name - value: "localhost" - action: upsert - resourcegraphprocessor: - -exporters: - logging: - verbosity: normal - -extensions: - resourceapiextension: - -service: - extensions: [resourceapiextension] - pipelines: - metrics: - receivers: [otlp] - processors: [resourcedetection, resourcegraphprocessor] - exporters: [logging] - #logs: - # receivers: [osqueryreceiver] - # processors: [] - # exporters: [logging] \ No newline at end of file diff --git a/collector/scripts/install/install-macos.sh b/collector/scripts/install/install-macos.sh new file mode 100755 index 0000000..8706d74 --- /dev/null +++ b/collector/scripts/install/install-macos.sh @@ -0,0 +1,600 @@ +#!/bin/sh + +set -e + +# fork of: https://github.com/observIQ/bindplane-agent/blob/release/v1.45.0/scripts/install/install_macos.sh + +# Collector Constants +SERVICE_NAME="com.servicenow.collector" +DOWNLOAD_BASE="https://github.com/lightstep/sn-collector/releases/download" + +# Script Constants +PREREQS="printf sed uname tr find grep" +TMP_DIR="${TMPDIR:-"/tmp/"}sn-collector" # Allow this to be overriden by cannonical TMPDIR env var +INSTALL_DIR="/opt/sn-collector" +MANAGEMENT_YML_PATH="$INSTALL_DIR/manager.yaml" +SCRIPT_NAME="$0" +INDENT_WIDTH=' ' +indent="" + +# Colors +num_colors=$(tput colors 2>/dev/null) +if test -n "$num_colors" && test "$num_colors" -ge 8; then + bold="$(tput bold)" + underline="$(tput smul)" + # standout can be bold or reversed colors dependent on terminal + standout="$(tput smso)" + reset="$(tput sgr0)" + bg_black="$(tput setab 0)" + bg_blue="$(tput setab 4)" + bg_cyan="$(tput setab 6)" + bg_green="$(tput setab 2)" + bg_magenta="$(tput setab 5)" + bg_red="$(tput setab 1)" + bg_white="$(tput setab 7)" + bg_yellow="$(tput setab 3)" + fg_black="$(tput setaf 0)" + fg_blue="$(tput setaf 4)" + fg_cyan="$(tput setaf 6)" + fg_green="$(tput setaf 2)" + fg_magenta="$(tput setaf 5)" + fg_red="$(tput setaf 1)" + fg_white="$(tput setaf 7)" + fg_yellow="$(tput setaf 3)" +fi + +if [ -z "$reset" ]; then + sed_ignore='' +else + sed_ignore="/^[$reset]+$/!" +fi + +# Helper Functions +printf() { + if command -v sed >/dev/null; then + command printf -- "$@" | sed -E "$sed_ignore s/^/$indent/g" # Ignore sole reset characters if defined + else + # Ignore $* suggestion as this breaks the output + # shellcheck disable=SC2145 + command printf -- "$indent$@" + fi +} + +increase_indent() { indent="$INDENT_WIDTH$indent" ; } +decrease_indent() { indent="${indent#*"$INDENT_WIDTH"}" ; } + +# Color functions reset only when given an argument +bold() { command printf "$bold$*$(if [ -n "$1" ]; then command printf "$reset"; fi)" ; } +underline() { command printf "$underline$*$(if [ -n "$1" ]; then command printf "$reset"; fi)" ; } +standout() { command printf "$standout$*$(if [ -n "$1" ]; then command printf "$reset"; fi)" ; } +# Ignore "parameters are never passed" +# shellcheck disable=SC2120 +reset() { command printf "$reset$*$(if [ -n "$1" ]; then command printf "$reset"; fi)" ; } +bg_black() { command printf "$bg_black$*$(if [ -n "$1" ]; then command printf "$reset"; fi)" ; } +bg_blue() { command printf "$bg_blue$*$(if [ -n "$1" ]; then command printf "$reset"; fi)" ; } +bg_cyan() { command printf "$bg_cyan$*$(if [ -n "$1" ]; then command printf "$reset"; fi)" ; } +bg_green() { command printf "$bg_green$*$(if [ -n "$1" ]; then command printf "$reset"; fi)" ; } +bg_magenta() { command printf "$bg_magenta$*$(if [ -n "$1" ]; then command printf "$reset"; fi)" ; } +bg_red() { command printf "$bg_red$*$(if [ -n "$1" ]; then command printf "$reset"; fi)" ; } +bg_white() { command printf "$bg_white$*$(if [ -n "$1" ]; then command printf "$reset"; fi)" ; } +bg_yellow() { command printf "$bg_yellow$*$(if [ -n "$1" ]; then command printf "$reset"; fi)" ; } +fg_black() { command printf "$fg_black$*$(if [ -n "$1" ]; then command printf "$reset"; fi)" ; } +fg_blue() { command printf "$fg_blue$*$(if [ -n "$1" ]; then command printf "$reset"; fi)" ; } +fg_cyan() { command printf "$fg_cyan$*$(if [ -n "$1" ]; then command printf "$reset"; fi)" ; } +fg_green() { command printf "$fg_green$*$(if [ -n "$1" ]; then command printf "$reset"; fi)" ; } +fg_magenta() { command printf "$fg_magenta$*$(if [ -n "$1" ]; then command printf "$reset"; fi)" ; } +fg_red() { command printf "$fg_red$*$(if [ -n "$1" ]; then command printf "$reset"; fi)" ; } +fg_white() { command printf "$fg_white$*$(if [ -n "$1" ]; then command printf "$reset"; fi)" ; } +fg_yellow() { command printf "$fg_yellow$*$(if [ -n "$1" ]; then command printf "$reset"; fi)" ; } + +# Intentionally using variables in format string +# shellcheck disable=SC2059 +info() { printf "$*\\n" ; } +# Intentionally using variables in format string +# shellcheck disable=SC2059 +warn() { + increase_indent + printf "$fg_yellow$*$reset\\n" + decrease_indent +} +# Intentionally using variables in format string +# shellcheck disable=SC2059 +error() { + increase_indent + printf "$fg_red$*$reset\\n" + decrease_indent +} +# Intentionally using variables in format string +# shellcheck disable=SC2059 +success() { printf "$fg_green$*$reset\\n" ; } +# Ignore 'arguments are never passed' +# shellcheck disable=SC2120 +prompt() { + if [ "$1" = 'n' ]; then + command printf "y/$(fg_red '[n]'): " + else + command printf "$(fg_green '[y]')/n: " + fi +} + +separator() { printf "===================================================\\n" ; } + +banner() +{ + printf "\\n" + separator + printf "| %s\\n" "$*" ; + separator +} + +usage() +{ + increase_indent + USAGE=$(cat <&2 + shift 2 + if [ -n "$0" ]; then + increase_indent + error "$*" + decrease_indent + fi + force_exit +} + +print_prereq_line() +{ + if [ -n "$2" ]; then + command printf "\\n${indent} - " + command printf "[$1]: $2" + fi +} + +check_failure() +{ + if [ "$indent" != '' ]; then increase_indent; fi + command printf "${indent}${fg_red}ERROR: %s check failed!${reset}" "$1" + + print_prereq_line "Issue" "$2" + print_prereq_line "Resolution" "$3" + print_prereq_line "Help Link" "$4" + print_prereq_line "Rerun" "$5" + + command printf "\\n" + if [ "$indent" != '' ]; then decrease_indent; fi + force_exit +} + +succeeded() +{ + increase_indent + success "Succeeded!" + decrease_indent +} + +failed() +{ + error "Failed!" +} + +# This will check all prerequisites before running an installation. +check_prereqs() +{ + banner "Checking Prerequisites" + increase_indent + root_check + os_check + dependencies_check + success "Prerequisite check complete!" + decrease_indent +} + +# This will check if the operating system is supported. +os_check() +{ + info "Checking that the operating system is supported..." + os_type=$(uname -s) + case "$os_type" in + Darwin) + succeeded + ;; + *) + failed + error_exit "$LINENO" "The operating system $(fg_yellow "$os_type") is not supported by this script." + ;; + esac +} + +# This checks to see if the user who is running the script has root permissions. +root_check() +{ + system_user_name=$(id -un) + if [ "${system_user_name}" != 'root' ] + then + failed + error_exit "$LINENO" "Script needs to be run as root or with sudo" + fi +} + +# This will check if the current environment has +# all required shell dependencies to run the installation. +dependencies_check() +{ + info "Checking for script dependencies..." + FAILED_PREREQS='' + for prerequisite in $PREREQS; do + if command -v "$prerequisite" >/dev/null; then + continue + else + if [ -z "$FAILED_PREREQS" ]; then + FAILED_PREREQS="${fg_red}$prerequisite${reset}" + else + FAILED_PREREQS="$FAILED_PREREQS, ${fg_red}$prerequisite${reset}" + fi + fi + done + + if [ -n "$FAILED_PREREQS" ]; then + failed + error_exit "$LINENO" "The following dependencies are required by this script: [$FAILED_PREREQS]" + fi + succeeded +} + +# This will set all installation variables +# at the beginning of the script. +setup_installation() +{ + banner "Configuring Installation Variables" + increase_indent + + set_os_arch + set_download_urls + set_ingest_endpoint + set_ingest_token + + success "Configuration complete!" + decrease_indent +} + +set_os_arch() +{ + os_arch=$(uname -m) + case "$os_arch" in + # arm64 strings. These are from https://stackoverflow.com/questions/45125516/possible-values-for-uname-m + aarch64|arm64|aarch64_be|armv8b|armv8l) + os_arch="arm64" + ;; + x86_64) + os_arch="amd64" + ;; + *) + # We only support arm64/amd64 architectures for macOS + error_exit "$LINENO" "Unsupported os arch: $os_arch" + ;; + esac +} + +# This will set the urls to use when downloading the agent and its plugins. +# These urls are constructed based on the --version flag or COLLECTOR_VERSION env variable. +# If not specified, the version defaults to whatever the latest release on github is. +set_download_urls() +{ + if [ -z "$url" ] ; then + if [ -z "$version" ] ; then + # shellcheck disable=SC2153 + version=$COLLECTOR_VERSION + fi + + if [ -z "$version" ] ; then + version=$(latest_version) + fi + + if [ -z "$version" ] ; then + error_exit "$LINENO" "Could not determine version to install" + fi + + if [ -z "$base_url" ] ; then + base_url=$DOWNLOAD_BASE + fi + + collector_download_url="$base_url/v$version/otelcol-servicenow_v${version}_darwin_${os_arch}.tar.gz" + else + collector_download_url="$url" + fi +} + +set_ingest_endpoint() +{ + if [ -z "$ingest_endpoint" ] ; then + ingest_endpoint="$ENDPOINT" + fi + + INGEST_ENDPOINT="$ingest_endpoint" +} + +set_ingest_token() +{ + if [ -z "$ingest_token" ] ; then + ingest_token=$INGEST_TOKEN + fi + + INGEST_TOKEN="$ingest_token" + + if [ -n "$INGEST_TOKEN" ] && [ -z "$INGEST_ENDPOINT" ]; then + error_exit "$LINENO" "An endpoint must be specified when providing an ingest token" + fi +} + +# latest_version gets the tag of the latest release, without the v prefix. +latest_version() +{ + curl -sSL -H"Accept: application/vnd.github.v3+json" \ https://api.github.com/repos/lightstep/sn-collector/releases/latest | \ + grep "\"tag_name\"" | \ + sed -E 's/ *"tag_name": "v([0-9]+\.[0-9]+\.[0-9+])",/\1/' +} + +# This will install the package by downloading & unpacking the tarball into the install directory +install_package() +{ + banner "Installing ServiceNow Collector" + increase_indent + + # Remove temporary directory, if it exists + rm -rf "$TMP_DIR" + mkdir -p "$TMP_DIR/artifacts" + + # Download into tmp dir + info "Downloading tarball from $collector_download_url into temporary directory..." + curl -L "$collector_download_url" -o "$TMP_DIR/otelcol-servicenow.tar.gz" --progress-bar --fail || error_exit "$LINENO" "Failed to download package" + succeeded + + # unpack + info "Unpacking tarball..." + tar -xzf "$TMP_DIR/otelcol-servicenow.tar.gz" -C "$TMP_DIR/artifacts" || error_exit "$LINENO" "Failed to unack archive $TMP_DIR/otelcol-servicenow.tar.gz" + succeeded + + mkdir -p "$INSTALL_DIR" || error_exit "$LINENO" "Failed to create directory $INSTALL_DIR" + + info "Creating install directory structure..." + increase_indent + # Find all directorys in the unpackaged tar + DIRS=$(cd "$TMP_DIR/artifacts"; find "." -type d) + for d in $DIRS + do + mkdir -p "$INSTALL_DIR/$d" || error_exit "$LINENO" "Failed to create directory $INSTALL_DIR/$d" + done + + # Create the storage dir; This dir is necessary for filelog based plugins + mkdir -p "$INSTALL_DIR/storage" || error_exit "$LINENO" "Failed to create directory $INSTALL_DIR/storage" + + decrease_indent + succeeded + + info "Copying artifacts to install directory..." + increase_indent + + # This find command gets a list of all artifacts paths except config.yaml, logging.yaml, or opentelemetry-java-contrib-jmx-metrics.jar + FILES=$(cd "$TMP_DIR/artifacts"; find "." -type f -not \( -name config.yaml -or -name logging.yaml -or -name opentelemetry-java-contrib-jmx-metrics.jar \)) + # Move files to install dir + for f in $FILES + do + rm -rf "$INSTALL_DIR/${f:?}" + cp "$TMP_DIR/artifacts/${f:?}" "$INSTALL_DIR/${f:?}" || error_exit "$LINENO" "Failed to copy artifact $f to install dir" + done + decrease_indent + succeeded + + if [ ! -f "$INSTALL_DIR/config.yaml" ]; then + info "Copying default config.yaml..." + cp "$TMP_DIR/artifacts/config/otelcol-macos-hostmetrics.yaml" "$INSTALL_DIR/config.yaml" || error_exit "$LINENO" "Failed to copy default config.yaml to install dir" + succeeded + fi + + # TODO: If an endpoint was specified, we need to update the config + #if [ -n "$OPAMP_ENDPOINT" ]; then + # create_manager_yml "$MANAGEMENT_YML_PATH" + #fi + + if [ -f "/Library/LaunchDaemons/$SERVICE_NAME.plist" ]; then + # Existing service file, we should stop & unload first. + info "Uninstalling existing service file..." + launchctl unload -w "/Library/LaunchDaemons/$SERVICE_NAME.plist" > /dev/null 2>&1 || error_exit "$LINENO" "Failed to unload service file /Library/LaunchDaemons/$SERVICE_NAME.plist" + succeeded + fi + + # Install service file + info "Installing service file..." + sed "s|\\[INSTALLDIR\\]|${INSTALL_DIR}/|g" "$INSTALL_DIR/install/$SERVICE_NAME.plist" | tee "/Library/LaunchDaemons/$SERVICE_NAME.plist" > /dev/null 2>&1 || error_exit "$LINENO" "Failed to install service file" + launchctl load -w "/Library/LaunchDaemons/$SERVICE_NAME.plist" > /dev/null 2>&1 || error_exit "$LINENO" "Failed to load service file /Library/LaunchDaemons/$SERVICE_NAME.plist" + succeeded + + info "Starting service..." + launchctl start "$SERVICE_NAME" || error_exit "$LINENO" "Failed to start service file $SERVICE_NAME" + succeeded + + info "Removing temporary files..." + rm -rf "$TMP_DIR" || error_exit "$LINENO" "Failed to remove temp dir: $TMP_DIR" + succeeded + + success "ServiceNow Collector installation complete!" + decrease_indent +} + +# create_manager_yml creates the manager.yml at the specified path, containing opamp information. +create_manager_yml() +{ + manager_yml_path="$1" + if [ ! -f "$manager_yml_path" ]; then + info "Creating manager yaml..." + command printf 'endpoint: "%s"\n' "$OPAMP_ENDPOINT" > "$manager_yml_path" + [ -n "$OPAMP_LABELS" ] && command printf 'labels: "%s"\n' "$OPAMP_LABELS" >> "$manager_yml_path" + [ -n "$OPAMP_SECRET_KEY" ] && command printf 'secret_key: "%s"\n' "$OPAMP_SECRET_KEY" >> "$manager_yml_path" + succeeded + fi +} + + +# This will display the results of an installation +display_results() +{ + banner 'Information' + increase_indent + info "Collector Home: $(fg_cyan "$INSTALL_DIR")$(reset)" + info "Collector Config: $(fg_cyan "$INSTALL_DIR/config.yaml")$(reset)" + info "Start Command: $(fg_cyan "sudo launchctl load /Library/LaunchDaemons/$SERVICE_NAME.plist")$(reset)" + info "Stop Command: $(fg_cyan "sudo launchctl unload /Library/LaunchDaemons/$SERVICE_NAME.plist")$(reset)" + info "Logs Command: $(fg_cyan "sudo tail -F $INSTALL_DIR/log/collector.log")$(reset)" + decrease_indent + + banner 'Support' + increase_indent + info "For more information on configuring the agent, see the docs:" + increase_indent + info "$(fg_cyan "https://github.com/lightstep/sn-collector/tree/main")$(reset)" + decrease_indent + info "If you have any other questions please contact us at $(fg_cyan support@lightstep.com)$(reset)" + decrease_indent + + banner "$(fg_green Installation Complete!)" + return 0 +} + +uninstall() +{ + banner "Uninstalling ServiceNow Collector" + increase_indent + + if [ ! -f "$INSTALL_DIR/otelcol-servicenow" ]; then + # If the agent binary is not present, we assume that the agent is not installed + # In this case, do nothing. + info "No install detected, skipping..." + decrease_indent + banner "$(fg_green Uninstallation Complete!)" + return 0 + fi + + info "Uninstalling service file..." + launchctl unload -w "/Library/LaunchDaemons/$SERVICE_NAME.plist" || error_exit "$LINENO" "Failed to unload service file /Library/LaunchDaemons/$SERVICE_NAME.plist" + rm -f "/Library/LaunchDaemons/$SERVICE_NAME.plist" || error_exit "$LINENO" "Failed to remove service file /Library/LaunchDaemons/$SERVICE_NAME.plist" + succeeded + + info "Backing up config.yaml to config.bak.yaml" + cp "$INSTALL_DIR/config.yaml" "$INSTALL_DIR/config.bak.yaml" || error_exit "$LINENO" "Failed to backup config.yaml to config.bak.yaml" + succeeded + + # Removes the whole install directory, including configs. + info "Removing installed artifacts..." + # This find command gets a list of all artifacts paths except config.yaml or the root directory. + FILES=$(cd "$INSTALL_DIR"; find "." -not \( -name config.bak.yaml -or -name "." \)) + for f in $FILES + do + rm -rf "${INSTALL_DIR:?}/$f" || error_exit "$LINENO" "Failed to remove artifact ${INSTALL_DIR:?}/$f" + done + + # Copy the old config to a backup. This is similar to how RPM handles this. + # This way, the file is recoverable if the uninstall was somehow accidental, + # but if a new install occurs, the default config will still be used. + succeeded + + decrease_indent + banner "$(fg_green Uninstallation Complete!)" +} + +main() +{ + # We do these checks before we process arguments, because + # some of these options bail early, and we'd like to be sure that those commands + # (e.g. uninstall) can run + + + check_prereqs + + if [ $# -ge 1 ]; then + while [ -n "$1" ]; do + case "$1" in + -l|--url) + url=$2 ; shift 2 ;; + -v|--version) + version=$2 ; shift 2 ;; + -r|--uninstall) + uninstall + exit 0 + ;; + -h|--help) + usage + exit 0 + ;; + -e|--endpoint) + ingest_endpoint=$2 ; shift 2 ;; + -i|--ingest-token) + ingest_token=$2 ; shift 2 ;; + -b|--base-url) + base_url=$2 ; shift 2 ;; + --) + shift; break ;; + *) + error "Invalid argument: $1" + usage + force_exit + ;; + esac + done + fi + + setup_installation + install_package + display_results +} + +main "$@" \ No newline at end of file diff --git a/collector/scripts/package/postinstall.sh b/collector/scripts/package/postinstall.sh new file mode 100644 index 0000000..fd3c40f --- /dev/null +++ b/collector/scripts/package/postinstall.sh @@ -0,0 +1,77 @@ +#!/bin/sh + +set -e + +manage_systemd_service() { + systemctl daemon-reload + + echo "configured systemd service" + + cat << EOF + +The "sn-collector" service has been configured! + +The collector's config file can be found here: + /opt/sn-collector/config.yaml + +To view logs from the collector, run: + sudo journalctl --unit=sn-collector + +For more information on configuring the collector, see the docs: + https://github.com/lightstep/sn-collector + +To stop the sn-collector service, run: + sudo systemctl stop sn-collector + +To start the sn-collector service, run: + sudo systemctl start sn-collector + +To restart the sn-collector service, run: + sudo systemctl restart sn-collector + +To enable the service on startup, run: + sudo systemctl enable sn-collector + +If you have any other questions please contact us at support@lightstep.com +EOF +} + +init_type() { + systemd_test="$(systemctl 2>/dev/null || : 2>&1)" + if command printf "$systemd_test" | grep -q '\-.mount'; then + command printf "systemd" + return + fi + + command printf "unknown" + return +} + +manage_service() { + service_type="$(init_type)" + case "$service_type" in + systemd) + manage_systemd_service + ;; + *) + echo "could not detect init system, skipping service configuration" + esac +} + +finish_permissions() { + # Goreleaser does not set plugin file permissions, so do them here + # We also change the owner of the binary to sn-collector + chown -R sn-collector:sn-collector /opt/sn-collector/otelcol-servicenow + + # Initialize the log file to ensure it is owned by sn-collector. + # This prevents the service (running as root) from assigning ownership to + # the root user. By doing so, we allow the user to switch to sn-collector + # user for 'non root' installs. + mkdir -p /opt/sn-collector/log + touch /opt/sn-collector/log/collector.log + chown sn-collector:sn-collector /opt/sn-collector/log/collector.log +} + + +finish_permissions +manage_service \ No newline at end of file diff --git a/collector/scripts/package/preinstall.sh b/collector/scripts/package/preinstall.sh new file mode 100644 index 0000000..a13480b --- /dev/null +++ b/collector/scripts/package/preinstall.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +set -e + +username="sn-collector" + +if getent group "$username" > /dev/null 2>&1; then + echo "Group ${username} already exists." +else + groupadd "$username" +fi + +if id "$username" > /dev/null 2>&1; then + echo "User ${username} already exists" + exit 0 +else + useradd --shell /sbin/nologin --system "$username" -g "$username" +fi diff --git a/collector/service/com.servicenow.collector.plist b/collector/service/com.servicenow.collector.plist new file mode 100644 index 0000000..a4543c3 --- /dev/null +++ b/collector/service/com.servicenow.collector.plist @@ -0,0 +1,31 @@ + + + + + Label + com.servicenow.collector + EnvironmentVariables + + SN_OTEL_COLLECTOR_HOME + [INSTALLDIR] + SN_OTEL_COLLECTOR_STORAGE + [INSTALLDIR]storage + + ProgramArguments + + [INSTALLDIR]otelcol-servicenow + --config + [INSTALLDIR]config.yaml + + RunAtLoad + + KeepAlive + + WorkingDirectory + [INSTALLDIR] + ExitTimeOut + 20 + StandardErrorPath + /var/log/sn_collector.err + + diff --git a/collector/service/sn-collector.service b/collector/service/sn-collector.service new file mode 100644 index 0000000..2066f4a --- /dev/null +++ b/collector/service/sn-collector.service @@ -0,0 +1,22 @@ +[Unit] +Description=ServiceNow OpenTelemetry collector +After=network.target +StartLimitIntervalSec=120 +StartLimitBurst=5 +[Service] +Type=simple +User=root +Group=sn-collector +Environment=PATH=/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin +Environment=SN_OTEL_COLLECTOR_HOME=/opt/sn-collector +Environment=SN_OTEL_COLLECTOR_STORAGE=/opt/sn-collector/storage +WorkingDirectory=/opt/sn-collector +ExecStart=/opt/sn-collector/otelcol-servicenow --config config.yaml +SuccessExitStatus=0 +TimeoutSec=20 +StandardOutput=journal +Restart=on-failure +RestartSec=5s +KillMode=process +[Install] +WantedBy=multi-user.target diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..a3d6a5d --- /dev/null +++ b/docs/development.md @@ -0,0 +1,30 @@ +## Development + +### Build + + +The OpenTelemetry Collector builder is required to build. + +```sh +cd collector/ +make +goreleaser release --snapshot --rm-dist +``` + +### Release + +`goreleaser` is used to package multi-platform builds of the collector. + +```sh +cd collector/ +make +goreleaser release --snapshot --rm-dist +``` + +To build for multiple platforms, this repository runs goreleaser automatically in a Github Action when a tag starting with `v` is pushed to the repository. + +```sh + git tag v0.0.1 + git push origin --tags + # ... Github Action to build and released kicked off remotely +``` \ No newline at end of file diff --git a/docs/monitor-kubernetes.md b/docs/monitor-kubernetes.md new file mode 100644 index 0000000..5e28677 --- /dev/null +++ b/docs/monitor-kubernetes.md @@ -0,0 +1,46 @@ +## Monitor Kubernetes with the ServiceNow Collector + +| Kuberenetes Distibution | Support Status | Architecture | +| ---------------------------------------------- | ------------------ ------ | ------------ | +| GKE (Google Cloud) | last three major versions | ARM, AMD | +| EKS (AWS) | last three major versions | ARM, AMD | +| AKS (Azure) | last three major versions | ARM, AMD | +| Kubernetes | last three major versions | ARM, AMD | + +* **Note:** We recommend Red Hat OpenShift customers should use the [Red Hat OpenTelemetry Distribution](https://docs.openshift.com/container-platform/4.12/otel/otel-using.html). + +### Deploy for Kubernetes monitoring with The OpenTelemetry Operator and Helm + +> This is an example only. We recommend using the official OpenTelemetry Operator Helm chart for deploying to production. + +All of the following assume you are installing the Operator in the `default` cluster namespace and you have `helm` v3 installed. + +1. Add OpenTelemetry Helm Charts. + - ```sh + helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts + helm repo update + ``` + +2. Install charts. This installs the Operator with an automatically-generated self-signed certificate. For other options, see ["TLS Certificate Requirement"](https://github.com/open-telemetry/opentelemetry-helm-charts/tree/main/charts/opentelemetry-operator#tls-certificate-requirement) in OpenTelemetry Operator documentation. + - ```sh + helm install \ + --set admissionWebhooks.certManager.enabled=false \ + --set admissionWebhooks.certManager.autoGenerateCert=true \ + opentelemetry-operator open-telemetry/opentelemetry-operator + ``` + +3. Set credentials for your ServiceNow instance and Cloud Observability. + - ```sh + export LS_TOKEN='' + kubectl create secret generic ls-token-secret -n default --from-literal='LS_TOKEN=$LS_TOKEN' + + # Set password for Event Manangement user on your instances + export MID_INSTANCE_EVENTS_PASSWORD='' + kubectl create secret generic mid-instance-events-password -n default --from-literal='MID_INSTANCE_EVENTS_PASSWORD=$MID_INSTANCE_EVENTS_PASSWORD' + ``` + +4. Deploy an OpenTelemetry Collector. The following uses the deployment example from the `collector/examples/` directory. Before applying you *must* edit the `k8s-deployment.yaml` file and set your username, instance URL, and cluster name. + + - ```sh + kubectl apply -f collector/examples/k8s-deployment.yaml + ``` diff --git a/docs/monitor-linux.md b/docs/monitor-linux.md new file mode 100644 index 0000000..d7ab464 --- /dev/null +++ b/docs/monitor-linux.md @@ -0,0 +1,50 @@ +## Monitor Linux with the ServiceNow Collector + +| Linux Distibution | Support Status | Architecture | +| ---------------------------------------------- | ------------------------- | ------------ | +| Red Hat Enterprise Linux (RHEL), Amazon Linux | last three major versions | ARM, AMD | +| Ubuntu | last three major versions | ARM, AMD | +| Alpine | last three major versions | ARM, AMD | +| Debian | last three major versions | ARM, AMD | + +### Install for for Linux server monitoring (no containers) + +Gather system metrics from a Linux system using an installed software package. Use this for servers and hosts that **do not** have Docker or a container runtime. + +1. Download the appropriate package for your system and architecture from the Releases page of this repository. + +2. Install the downloaded package using the appropriate package manager for your Linux distribution. + - RPM (RHEL, CentOS, Amazon Linux) package with `yum`: + ```sh + sudo yum install -y otelcol-servicenow_version_linux_arch.rpm + ``` + - Debian (Ubuntu) package with `apt-get`: + ```sh + sudo apt-get install -y otelcol-servicenow_version_linux_arch.deb + ``` + - APK (Alpine Linux) package with `apk`: + ```sh + sudo apk add --allow-untrusted otelcol-servicenow_version_linux_arch.apk + ``` +3. Follow the post-install instructions on starting the collector service. + +### Install for for Linux host monitoring with Docker + +Gather host system metrics from a Linux using a Docker image. + +1. Pull the latest Docker image for the collector. + - ```sh + docker pull ghcr.io/lightstep/sn-collector/sn-collector-experimental:latest + ``` + +2. Run the collector as a container, but mount the host filesystem to gather host metrics. + - ```sh + docker run --rm --name sn-collector-experimental \ + -v ./collector/config/otelcol-docker-hostmetrics.yaml:/etc/otelcol/config.yaml \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /:/hostfs + -e LS_TOKEN=your-cloud-obs-token + ghcr.io/lightstep/sn-collector/sn-collector-experimental:latest + ``` + +3. View the container logs and verify data is being sent. \ No newline at end of file diff --git a/docs/monitor-macos.md b/docs/monitor-macos.md new file mode 100644 index 0000000..c49634f --- /dev/null +++ b/docs/monitor-macos.md @@ -0,0 +1,13 @@ +## Monitor macOS with the ServiceNow Collector + +The last three macOS versions are supported on both Intel (AMD64) and Apple Silicon (ARM-based: M1, M2, etc) + +### Install for macOS Monitoring + +To install using an automated script, run: + +```sh +sudo sh -c "$(curl -fsSlL https://github.com/lightstep/sn-collector/releases/latest/download/install_macos.sh)" install_macos.sh +``` + +By default, the configuration for the collector will be installed in `/opt/sn-collector/config.yaml`. diff --git a/docs/monitor-windows.md b/docs/monitor-windows.md new file mode 100644 index 0000000..f41a59d --- /dev/null +++ b/docs/monitor-windows.md @@ -0,0 +1,5 @@ +## Monitor Windows with the ServiceNow Collector + +Windows 2022 64-bit is supported. + +Currently, the Windows collector is only available as an executable file from this repository's Releases page. diff --git a/readme.md b/readme.md index 6fbf237..ae7cba3 100644 --- a/readme.md +++ b/readme.md @@ -1,13 +1,128 @@ -## sn-collector +## ServiceNow OpenTelemetry Collector (Experimental) -ServiceNow-flavored OpenTelemetry collector experiments. +
-### data in -* `components/osqueryreceiver` turn osquery requests into OTLP logs +[![Action Status](https://github.com/lightstep/sn-collector/workflows/Build/badge.svg)](https://github.com/lightstep/sn-collector/actions) +[![Action Test Status](https://github.com/lightstep/sn-collector/workflows/Tests/badge.svg)](https://github.com/lightstep/sn-collector/actions) +[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -### data out -* `components/servicenowexporter` write metrics to MID servers +
-### data insights -* `components/resourcegraphprocessor` turn telemetry into CIs and CI relationships (stored in redis) -* `components/resourceapiextension` expose detected resources as an HTTP API +> ⚠️ **Important**: This is pre-release, experimental software under active development. There will be breaking changes between releases and it has not been tested on all platforms. Please contact your ServiceNow account team before installing to review eligibility. **We recommend customers build their own collectors or use [OpenTelemetry Collector Contrib](https://github.com/open-telemetry/opentelemetry-collector-contrib/) for production software.** + +ServiceNow OpenTelemetry Collector is an experimental distribution of the [OpenTelemetry +Collector](https://github.com/open-telemetry/opentelemetry-collector). It +provides a unified way to receive, process, and export metric, trace, and log +data for [ServiceNow Cloud Observability](https://www.lightstep.com) and various services running on ServiceNow instances. + +| Feature | Status | Docs | +| ---------------------------------------------- | ---------- | ------------------------ | +| Telemetry routing and processing ("gateway") | 🟢 stable | 📔 [Community docs][14] | +| Kubernetes cluster and workload monitoring | 🟢 stable | 📒 [Install guide][10] | +| Linux server monitoring | 🛠️ alpha | 📒 [Install guide][11] | +| Windows server monitoring | 🛠️ alpha | 📒 [Install guide][12] | +| macOS monitoring | 🛠️ alpha | 📒 [Install guide][13] | + +[10]: /docs/monitor-kubernetes.md +[11]: /docs/monitor-linux.md +[12]: /docs/monitor-windows.md +[13]: /docs/monitor-macos.md +[14]: https://opentelemetry.io/docs/collector/ + +### Supported ServiceNow destinations + +Native OTLP exporters can be used to send metrics, logs, and traces to ServiceNow Cloud Observability. + +| Destination | Metrics | Logs | Traces | Events | +| ------------------------ | ------------- | ---------------- | ------ | ---------------------- | +| Cloud Observability | OTLP | OTLP | OTLP | OTLP (Logs) | +| ServiceNow MID Server | [Push API][6] | [HLA REST API][8]| - | [Web Service API][7] | +| ServiceNow Instance | - | - | - | [Web Service API][7] | + +[6]: https://docs.servicenow.com/bundle/vancouver-api-reference/page/integrate/inbound-rest/concept/push-metrics-MID-server.html +[7]: https://docs.servicenow.com/bundle/vancouver-it-operations-management/page/product/event-management/task/send-events-via-web-service.html +[8]: https://docs.servicenow.com/bundle/vancouver-it-operations-management/page/product/health-log-analytics-admin/task/hla-data-input-rest-api.html + +### Supported ServiceNow sources + +| Source | Metrics | Logs | Traces | Events | +| ------------------------ | -------- | ------------------------------------ | ------- | ------ | +| ServiceNow Instance | - | [Log Export Service to OTLP Logs][5] | - | - | + +[5]: https://support.servicenow.com/kb?id=kb_article_view&sysparm_article=KB1575051 + +### ServiceNow Collector Built-in Components + +The following tables represent the supported components of the ServiceNow Collector. Our goal is to upstream all in-house developed components (marked with `*`), where possible, to the -contrib distribution of the OpenTelemetry Collector. + +#### Receivers + +| Receiver | Status | +| ---------------------------------------------------------------- | ---------------------------- | +| otlp | [contrib][1] | +| prometheus | [contrib][1] | +| hostmetrics | [contrib][1] | +| k8sevents | [contrib][1] | +| k8scluster | [contrib][1] | +| kubeletstats | [contrib][1] | +| filelog | [contrib][1] | +| httpcheck | [contrib][1] | +| `azuremonitor`* | [contrib][1] | +| azureeventhub | [contrib][1] | +| `osquery`* | [contrib][1] | + +[1]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver + +#### Processors + +| Processor | Status | +| ---------------------------------------------------------------- | -------------------- | +| k8sattributes | [contrib][2] | +| resource | [contrib][2] | +| resourcedetection | [contrib][2] | +| transform | [contrib][2] | +| `concurrentbatch`* | pending contribution | + +[2]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor + +#### Exporters + +| Exporter | Status | +| ---------------------------------------------------------------- | -------------------- | +| otlp | in core | +| `servicenow`* | pending contribution | +| debug | in core | +| kafka | [contrib][3] | + +[3]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter + +#### Extensions + +| Extension | Status | +| ---------------------------------------------------------------- | --------------- | +| healthcheck | [contrib][4] | +| opamp | [contrib][4] | +| pprof | [contrib][4] | + +[4]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension + +#### Connectors + +| Extension | Status | +| ---------------------------------------------------------------- | --------------- | +| exceptionsconnector | in [contrib][9] | + +[9]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/connector + +### Getting help + +We are providing support via GitHub on a best effort basis. ServiceNow customers should also open a case on [https://support.servicenow.com/now](https://support.servicenow.com/now). + +### Development and Contributing + +* For contributing guidelines, refer to [CONTRIBUTING.md](CONTRIBUTING.md). +* For getting started with development, see our development docs at [docs/development.md](/docs/development.md). + +### Acknowledgements + +* Thank you to the many open-source distributions for OpenTelemetry collectors for providing patterns for deploying, building, and releasing this software. Where possible, we've tried to follow best practices and align with community standards and conventions.