diff --git a/.dev.vars.example b/.dev.vars.example new file mode 100644 index 0000000..609e462 --- /dev/null +++ b/.dev.vars.example @@ -0,0 +1 @@ +INFLUX_TOKEN="example" diff --git a/.env.example b/.env.example deleted file mode 100644 index 77a8c8c..0000000 --- a/.env.example +++ /dev/null @@ -1,3 +0,0 @@ -INFLUX_HOST=http://localhost:8086 -INFLUX_TOKEN=example -INFLUX_DATABASE=example \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..5b92ccf --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,42 @@ +name: CI +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: cp wrangler.toml.example wrangler.toml + - uses: actions/setup-node@v4 + with: + node-version: 20 + - run: npm ci + - run: npm test + + deploy: + if: github.ref == 'refs/heads/main' + needs: [test] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Deploy + uses: cloudflare/wrangler-action@v3 + with: + apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} + preCommands: | + cp wranger.toml.example wrangler.toml + envsubst < wrangler.toml.example > wrangler.toml + postCommands: | + rm -f wrangler.toml + environment: production + secrets: | + INFLUX_TOKEN + env: + INFLUX_METRIC_NAME: ${{ env.INFLUX_METRIC }} + INFLUX_URL: ${{ env.INFLUX_URL }} + INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }} + INFLUX_DATABASE: ${{ env.INFLUX_DATABASE }} diff --git a/.gitignore b/.gitignore index f2e9ec2..1a7eae6 100644 --- a/.gitignore +++ b/.gitignore @@ -2,14 +2,14 @@ /dist **/*.rs.bk Cargo.lock -bin/ pkg/ wasm-pack.log worker/ node_modules/ .cargo-ok .env +.dev.vars .wrangler data/ config/ -wrangler.toml \ No newline at end of file +wrangler.toml diff --git a/README.md b/README.md index 4607253..759a9d6 100644 --- a/README.md +++ b/README.md @@ -1,34 +1,37 @@ # Cloudflare metrics worker -Send your page views from [Cloudflare worker](https://developers.cloudflare.com/workers/) to InfluxDB. +[Couldflare worker](https://developers.cloudflare.com/workers/) used to proxy requests to your application and send metrics to InfluxDB. -![Dashboard views](static/dashboard.png) +## Development -## Requirements - -1. Your site need to be setup behind Cloudflare CDN. -2. You need to setup InfluxDB with external access (make sure you have set [authentication](https://docs.influxdata.com/influxdb/v1.7/administration/authentication_and_authorization/#set-up-authentication)) - 1. Make sure InfluxDB is hosted under [supported port](https://blog.cloudflare.com/cloudflare-now-supporting-more-ports/) for Workers. Best option is 80 or 443. - -## How to use - -1. Install wrangler package +1. Install dependencies ``` -npm i @cloudflare/wrangler -g +npm install ``` 2. Copy example files ``` -cp .env.example .env +cp .dev.vars.example .dev.vars cp wrangler.toml.example wrangler.toml ``` -3. Deploy your worker to a site with wrangler +3. Edit secrets inside `.dev.vars` and environment variables inside `wrangler.toml` files + +3. Run your worker ``` -wrangler publish --env production +npm run dev ``` -4. (Optional) If you're using Grafana with InfluxDB, then you can import [example Dashboard](static/dashboard.json) from first screen. +## Deployment to production + +In order to deploy your worker via Github Actions, you need to have a [Cloudflare API token](https://developers.cloudflare.com/fundamentals/api/get-started/create-token/) and running instance of InfluxDB. + +Add generated API token to Github secrets as `CLOUDFLARE_API_TOKEN` and authentication token under `INFLUX_TOKEN`. + +Other required environment variables include the following: +- `INFLUX_URL` - InfluxDB URL +- `INFLUX_DATABASE` - InfluxDB database (bucket) name +- `INFLUX_METRIC_NAME` - InfluxDB metric name \ No newline at end of file diff --git a/bin/worker.js b/bin/worker.js new file mode 100644 index 0000000..e9064c2 --- /dev/null +++ b/bin/worker.js @@ -0,0 +1,9 @@ +import { reportRequestMetrics as reportRequestMetricsToInflux } from '../lib/influx.js' + +export default { + async fetch(request, env, ctx, { reportRequestMetrics = reportRequestMetricsToInflux } = {}) { + const response = await fetch(request) + ctx.waitUntil(reportRequestMetrics(request, env)) + return response + }, +} diff --git a/index.js b/index.js deleted file mode 100644 index 001e21c..0000000 --- a/index.js +++ /dev/null @@ -1,66 +0,0 @@ -import zlib from 'node:zlib'; -import { promisify } from 'node:util'; - -function getRequestData(request, response, startTime, endTime) { - const timestamp = Date.now(); - const originResponse = response || {}; - const url = new URL(request.url); - const apiKey = request.headers.get('api-key') || url.searchParams.get('api-key') || (request.headers.get('Authorization')?.startsWith('Bearer ') ? request.headers.get('Authorization').substring(7) : 'unknown'); - - return { - 'timestamp': timestamp, - 'url': request.url, - 'method': request.method, - 'status': originResponse.status, - 'originTime': (endTime - startTime), - 'cfCache': (originResponse) ? (response.headers.get('CF-Cache-Status') || 'miss') : 'miss', - 'apiKey': apiKey, - }; -} - -function formatRequestData(data, env) { - const url = new URL(data.url); - const formattedUrl = url.toString().replaceAll('=', '\\='); - // We're setting field value to 1 to count the number of requests - return `${env.INFLUX_METRIC},status_code=${data.status},url=${formattedUrl},hostname=${url.hostname},pathname="${url.pathname}",method=${data.method},cf_cache=${data.cfCache},api_key=${data.apiKey} value=1 ${data.timestamp}` -} - -async function reportMetric(request, response, startTime, endTime, env) { - const compress = promisify(zlib.gzip); - const reqData = getRequestData(request, response, startTime, endTime); - const line = formatRequestData(reqData, env); - console.log(line); - - // Define API endpoint and headers - const url = `${env.INFLUX_URL}/api/v2/write?&bucket=${env.INFLUX_DATABASE}&precision=ms`; - - // Compress the string using gzip - const compressedData = await compress(line); - - // Make the POST request - return fetch(url, { - method: 'POST', - headers: { - 'Authorization': `Token ${env.INFLUX_TOKEN}`, - 'Content-Encoding': 'gzip', - 'Content-Type': 'text/plain; charset=utf-8', - }, - body: compressedData, - }) -} - -export default { - async fetch(request, env, ctx) { - const url = new URL(request.url); - const newUrl = `${env.REQUEST_URL}${url.pathname}${url.search}`; - // override the request with the new URL - const newRequest = new Request(newUrl, request); - const reqStartTime = Date.now(); - const response = await fetch(newRequest); - const reqEndTime = Date.now(); - - // report metrics for original request - ctx.waitUntil(reportMetric(request, response, reqStartTime, reqEndTime, env)); - return response; - } -} \ No newline at end of file diff --git a/lib/influx.js b/lib/influx.js new file mode 100644 index 0000000..eb36515 --- /dev/null +++ b/lib/influx.js @@ -0,0 +1,47 @@ +/* +* Reports request metrics to InfluxDB +* @param {Request} request - incoming request +* @param {object} env - environment variables +*/ +export const reportRequestMetrics = async (request, env) => { + await writeMetrics(createMetricsFromRequest(request, env), env) +} + +/* +* Returns request metrics in InfluxDB line protocol format +* https://docs.influxdata.com/influxdb/cloud/reference/syntax/line-protocol/ +* @param {Request} request - incoming request +* @param {object} env - environment variables +*/ +export const createMetricsFromRequest = (request, env) => { + const url = new URL(request.url); + const timestamp = Date.now(); + const apiKey = request.headers.get('api-key') || + url.searchParams.get('api-key') || ( + request.headers.get('Authorization')?.startsWith('Bearer ') + ? request.headers.get('Authorization').substring(7) + : 'unknown' + ); + + return `${env.INFLUX_METRIC_NAME} api_key="${apiKey}" ${timestamp}` +} + +/* +* Sends request metrics to InfluxDB +* https://docs.influxdata.com/enterprise_influxdb/v1/guides/write_data/ +* @param {string} lineProtocolData - InfluxDB line protocol formatted data +* @param {object} env - environment variables +*/ +export const writeMetrics = async (lineProtocolData, env) => { + // Define API endpoint and headers + const url = `${env.INFLUX_URL}/api/v2/write?&bucket=${env.INFLUX_DATABASE}&precision=ms`; + + return fetch(url, { + method: 'POST', + headers: { + 'Authorization': `Token ${env.INFLUX_TOKEN}`, + 'Content-Type': 'application/octet-stream' + }, + body: lineProtocolData, + }) +} diff --git a/package.json b/package.json index 9dc3f35..34edb4e 100644 --- a/package.json +++ b/package.json @@ -6,9 +6,11 @@ "main": "index.js", "scripts": { "deploy": "wrangler deploy", - "dev": "wrangler dev", + "deploy:prod": "wrangler deploy --env production", + "dev": "wrangler dev --env dev", "start": "wrangler dev", - "test": "vitest" + "test": "vitest run", + "test:watch": "vitest" }, "author": "vanadium23 ", "license": "MIT", diff --git a/static/dashboard.json b/static/dashboard.json deleted file mode 100644 index 2846e12..0000000 --- a/static/dashboard.json +++ /dev/null @@ -1,1094 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_REQUESTS", - "label": "requests", - "description": "", - "type": "datasource", - "pluginId": "influxdb", - "pluginName": "InfluxDB" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "6.6.0" - }, - { - "type": "panel", - "id": "grafana-clock-panel", - "name": "Clock", - "version": "1.0.3" - }, - { - "type": "panel", - "id": "grafana-piechart-panel", - "name": "Pie Chart", - "version": "1.4.0" - }, - { - "type": "panel", - "id": "grafana-worldmap-panel", - "name": "Worldmap Panel", - "version": "0.2.1" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "datasource", - "id": "influxdb", - "name": "InfluxDB", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": null, - "iteration": 1581572429214, - "links": [], - "panels": [ - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_REQUESTS}", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 0 - }, - "hiddenSeries": false, - "id": 2, - "interval": "1h", - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "total", - "sortDesc": true, - "total": true, - "values": true - }, - "lines": false, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_url", - "groupBy": [ - { - "params": [ - "$Interval" - ], - "type": "time" - }, - { - "params": [ - "url" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "views", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "duration" - ], - "type": "field" - }, - { - "params": [], - "type": "count" - } - ] - ], - "tags": [ - { - "key": "hostname", - "operator": "=~", - "value": "/^$Hostname$/" - }, - { - "condition": "AND", - "key": "pathname", - "operator": "=~", - "value": "/\\/$/" - }, - { - "condition": "AND", - "key": "country", - "operator": "=~", - "value": "/^$Country$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Page views", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "circleMaxSize": 30, - "circleMinSize": 2, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_REQUESTS}", - "decimals": 0, - "esMetric": "Count", - "gridPos": { - "h": 11, - "w": 6, - "x": 12, - "y": 0 - }, - "hideEmpty": false, - "hideZero": false, - "id": 10, - "initialZoom": 1, - "locationData": "countries", - "mapCenter": "(0°, 0°)", - "mapCenterLatitude": 0, - "mapCenterLongitude": 0, - "maxDataPoints": 1, - "mouseWheelZoom": false, - "options": {}, - "showLegend": true, - "stickyLabels": false, - "tableQueryOptions": { - "geohashField": "geohash", - "latitudeField": "latitude", - "longitudeField": "longitude", - "metricField": "metric", - "queryType": "geohash" - }, - "targets": [ - { - "alias": "$tag_country", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "country" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "views", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "duration" - ], - "type": "field" - }, - { - "params": [], - "type": "count" - } - ] - ], - "tags": [ - { - "key": "hostname", - "operator": "=~", - "value": "/^$Hostname$/" - }, - { - "condition": "AND", - "key": "pathname", - "operator": "=~", - "value": "/\\/$/" - } - ] - } - ], - "thresholds": "0,10", - "timeFrom": null, - "timeShift": null, - "title": "World views", - "type": "grafana-worldmap-panel", - "unitPlural": "", - "unitSingle": "", - "valueName": "total" - }, - { - "columns": [ - { - "text": "Total", - "value": "total" - } - ], - "datasource": "${DS_REQUESTS}", - "fontSize": "100%", - "gridPos": { - "h": 11, - "w": 6, - "x": 18, - "y": 0 - }, - "id": 6, - "options": {}, - "pageSize": null, - "showHeader": true, - "sort": { - "col": 1, - "desc": true - }, - "styles": [ - { - "alias": "Time", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "link": false, - "pattern": "Time", - "type": "date" - }, - { - "alias": "", - "align": "right", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 0, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "alias": "$tag_pathname", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "pathname" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "views", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "duration" - ], - "type": "field" - }, - { - "params": [], - "type": "count" - } - ] - ], - "tags": [ - { - "key": "hostname", - "operator": "=~", - "value": "/^$Hostname$/" - }, - { - "condition": "AND", - "key": "pathname", - "operator": "=~", - "value": "/\\/$/" - } - ] - } - ], - "timeFrom": "30d", - "timeShift": null, - "title": "Popular pages", - "transform": "timeseries_aggregations", - "type": "table" - }, - { - "columns": [ - { - "text": "Total", - "value": "total" - } - ], - "datasource": "${DS_REQUESTS}", - "fontSize": "100%", - "gridPos": { - "h": 12, - "w": 4, - "x": 0, - "y": 11 - }, - "id": 5, - "options": {}, - "pageSize": null, - "showHeader": true, - "sort": { - "col": 1, - "desc": true - }, - "styles": [ - { - "alias": "Time", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "link": false, - "pattern": "Time", - "type": "date" - }, - { - "alias": "", - "align": "right", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 0, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "alias": "$tag_pathname", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "pathname" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "views", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "duration" - ], - "type": "field" - }, - { - "params": [], - "type": "count" - } - ] - ], - "tags": [ - { - "key": "hostname", - "operator": "=~", - "value": "/^$Hostname$/" - }, - { - "condition": "AND", - "key": "pathname", - "operator": "=~", - "value": "/\\/$/" - } - ] - } - ], - "timeFrom": "7d", - "timeShift": null, - "title": "Popular pages", - "transform": "timeseries_aggregations", - "type": "table" - }, - { - "columns": [ - { - "text": "Total", - "value": "total" - } - ], - "datasource": "${DS_REQUESTS}", - "fontSize": "100%", - "gridPos": { - "h": 12, - "w": 4, - "x": 4, - "y": 11 - }, - "id": 7, - "options": {}, - "pageSize": null, - "showHeader": true, - "sort": { - "col": 1, - "desc": true - }, - "styles": [ - { - "alias": "Time", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "link": false, - "pattern": "Time", - "type": "date" - }, - { - "alias": "", - "align": "right", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 0, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "alias": "$tag_referer", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "referer" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "views", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "duration" - ], - "type": "field" - }, - { - "params": [], - "type": "count" - } - ] - ], - "tags": [ - { - "key": "hostname", - "operator": "=~", - "value": "/^$Hostname$/" - }, - { - "condition": "AND", - "key": "pathname", - "operator": "=~", - "value": "/\\/$/" - }, - { - "condition": "AND", - "key": "referer", - "operator": "!=", - "value": "vanadium23.me" - }, - { - "condition": "AND", - "key": "referer", - "operator": "!=", - "value": "empty" - } - ] - } - ], - "timeFrom": "7d", - "timeShift": null, - "title": "Referer tops", - "transform": "timeseries_aggregations", - "type": "table" - }, - { - "columns": [ - { - "text": "Total", - "value": "total" - } - ], - "datasource": "${DS_REQUESTS}", - "fontSize": "100%", - "gridPos": { - "h": 12, - "w": 4, - "x": 8, - "y": 11 - }, - "id": 8, - "options": {}, - "pageSize": null, - "showHeader": true, - "sort": { - "col": 1, - "desc": true - }, - "styles": [ - { - "alias": "Time", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "link": false, - "pattern": "Time", - "type": "date" - }, - { - "alias": "", - "align": "right", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 0, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "alias": "$tag_utm_source", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "utm_source" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "views", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "duration" - ], - "type": "field" - }, - { - "params": [], - "type": "count" - } - ] - ], - "tags": [ - { - "key": "hostname", - "operator": "=~", - "value": "/^$Hostname$/" - }, - { - "condition": "AND", - "key": "pathname", - "operator": "=~", - "value": "/\\/$/" - }, - { - "condition": "AND", - "key": "utm_source", - "operator": "!=", - "value": "empty" - } - ] - } - ], - "timeFrom": "7d", - "timeShift": null, - "title": "utm_sources tops", - "transform": "timeseries_aggregations", - "type": "table" - }, - { - "aliasColors": {}, - "breakPoint": "50%", - "cacheTimeout": null, - "combine": { - "label": "Others", - "threshold": 0 - }, - "datasource": "${DS_REQUESTS}", - "fontSize": "80%", - "format": "short", - "gridPos": { - "h": 12, - "w": 4, - "x": 12, - "y": 11 - }, - "id": 12, - "interval": null, - "legend": { - "show": true, - "values": true - }, - "legendType": "On graph", - "links": [], - "maxDataPoints": 3, - "nullPointMode": "connected", - "options": {}, - "pieType": "pie", - "strokeWidth": "2", - "targets": [ - { - "alias": "$tag_cf_cache", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "cf_cache" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "views", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "duration" - ], - "type": "field" - }, - { - "params": [], - "type": "count" - } - ] - ], - "tags": [ - { - "key": "hostname", - "operator": "=~", - "value": "/^$Hostname$/" - }, - { - "condition": "AND", - "key": "country", - "operator": "=~", - "value": "/^$Country$/" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Cloudflare caches", - "type": "grafana-piechart-panel", - "valueName": "current" - }, - { - "bgColor": null, - "cacheTimeout": null, - "clockType": "24 hour", - "countdownSettings": { - "customFormat": null, - "endCountdownTime": "2020-02-10T11:31:00.000Z", - "endText": "00:00:00" - }, - "datasource": "${DS_REQUESTS}", - "dateSettings": { - "dateFormat": "YYYY-MM-DD", - "fontSize": "20px", - "fontWeight": "normal", - "showDate": true - }, - "gridPos": { - "h": 12, - "w": 8, - "x": 16, - "y": 11 - }, - "id": 14, - "links": [], - "mode": "time", - "options": {}, - "refreshSettings": { - "syncWithDashboard": false - }, - "targets": [ - { - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [] - } - ], - "timeFrom": null, - "timeSettings": { - "customFormat": "HH:mm:ss", - "fontSize": "60px", - "fontWeight": "normal" - }, - "timeShift": null, - "timezone": null, - "timezoneSettings": { - "fontSize": "12px", - "fontWeight": "normal", - "showTimezone": false, - "zoneFormat": "offsetAbbv" - }, - "title": "Time passes", - "type": "grafana-clock-panel" - } - ], - "refresh": "5m", - "schemaVersion": 22, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "allValue": null, - "current": {}, - "datasource": "${DS_REQUESTS}", - "definition": "SHOW TAG VALUES ON \"requests\" WITH KEY = \"hostname\"", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "Hostname", - "options": [], - "query": "SHOW TAG VALUES ON \"requests\" WITH KEY = \"hostname\"", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": "${DS_REQUESTS}", - "definition": "SHOW TAG VALUES ON \"requests\" WITH KEY = \"country\"", - "hide": 0, - "includeAll": true, - "label": null, - "multi": false, - "name": "Country", - "options": [], - "query": "SHOW TAG VALUES ON \"requests\" WITH KEY = \"country\"", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "selected": true, - "text": "6h", - "value": "6h" - }, - "hide": 0, - "label": null, - "name": "Interval", - "options": [ - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": true, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - }, - { - "selected": false, - "text": "7d", - "value": "7d" - }, - { - "selected": false, - "text": "14d", - "value": "14d" - }, - { - "selected": false, - "text": "30d", - "value": "30d" - } - ], - "query": "1h,6h,12h,1d,7d,14d,30d", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" - } - ] - }, - "time": { - "from": "now-2d", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "Blog", - "uid": "CEu7c7UWk", - "version": 16 -} \ No newline at end of file diff --git a/static/dashboard.png b/static/dashboard.png deleted file mode 100644 index 3845302..0000000 Binary files a/static/dashboard.png and /dev/null differ diff --git a/test/influx.test.js b/test/influx.test.js new file mode 100644 index 0000000..877ee58 --- /dev/null +++ b/test/influx.test.js @@ -0,0 +1,128 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' +import { createMetricsFromRequest, writeMetrics, reportRequestMetrics } from '../lib/influx.js' + +describe('reportRequestMetrics', () => { + const date = new Date() + + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(date) + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('reports request metrics to InfluxDB over HTTP', async () => { + const env = givenTestEnvironment() + const request = { + url: 'https://example.com/path', + method: 'GET', + headers: new Map([['api-key', 'test-key']]), + } + global.fetch = vi.fn().mockResolvedValue(new Response(null, { status: 204 })); + + await reportRequestMetrics(request, env); + + expect(global.fetch).toHaveBeenCalledWith( + 'https://influx.example.com/api/v2/write?&bucket=test_db&precision=ms', + expect.objectContaining({ + method: 'POST', + headers: expect.objectContaining({ + Authorization: 'Token test_token', + 'Content-Type': 'application/octet-stream', + }), + body: `test_metric api_key="test-key" ${date.getTime()}`, + }), + ) + }); +}); + +describe('createMetricsFromRequest', () => { + const date = new Date() + const env = givenTestEnvironment() + + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(date) + }) + + afterEach(() => { + vi.useRealTimers() + }) + + const testCases = [ + { + description: 'when request has an API key in the Authorization header', + request: { + headers: new Map([['Authorization', 'Bearer test-key']]), + }, + expectedApiKey: 'test-key', + }, + { + description: 'when request has an API key in the api-key header', + request: { + headers: new Map([['api-key', 'test-key']]), + }, + expectedApiKey: 'test-key', + }, + { + description: 'when request has an API key in the query string', + request: { + url: 'https://example.com/path?api-key=test-key', + }, + expectedApiKey: 'test-key', + }, + { + description: 'when request has no API key', + request: {}, + expectedApiKey: 'unknown', + }, + ] + + testCases.forEach(({ description, request, expectedApiKey }) => { + it(description, () => { + const fullRequest = { + url: 'https://example.com/path', + method: 'GET', + headers: new Map(), + ...request + } + + const result = createMetricsFromRequest(fullRequest, env) + expect(result).toContain(expectedApiKey) + }) + }) +}) + +describe('writeMetrics', () => { + it('send request metrics to InfluxDB over HTTP', async () => { + const env = givenTestEnvironment() + const lineProtocolData = 'test_metric api_key="test-key"' + global.fetch = vi.fn().mockResolvedValue(new Response(null, { status: 204 })) + + const response = await writeMetrics(lineProtocolData, env) + + expect(global.fetch).toHaveBeenCalledWith( + 'https://influx.example.com/api/v2/write?&bucket=test_db&precision=ms', + expect.objectContaining({ + method: 'POST', + headers: expect.objectContaining({ + Authorization: 'Token test_token', + 'Content-Type': 'application/octet-stream', + }), + body: lineProtocolData, + }), + ) + expect(response.status).toBe(204) + }) +}) + +const givenTestEnvironment = () => { + return { + INFLUX_METRIC_NAME: 'test_metric', + INFLUX_URL: 'https://influx.example.com', + INFLUX_DATABASE: 'test_db', + INFLUX_TOKEN: 'test_token', + } +} diff --git a/test/worker.test.js b/test/worker.test.js new file mode 100644 index 0000000..938aec6 --- /dev/null +++ b/test/worker.test.js @@ -0,0 +1,22 @@ +import worker from '../bin/worker.js'; +import { describe, it, expect, vi } from 'vitest'; + +describe('worker.fetch', () => { + it('should fetch and report metrics', async () => { + const request = { + url: 'https://example.com/path', + method: 'GET' + }; + const env = {}; + const ctx = { + waitUntil: vi.fn() + }; + global.fetch = vi.fn().mockResolvedValue(new Response(null, { status: 200 })); + const reportRequestMetrics = vi.fn() + const response = await worker.fetch(request, env, ctx, { reportRequestMetrics }); + expect(global.fetch).toHaveBeenCalledWith(request); + expect(reportRequestMetrics).toHaveBeenCalledWith(request, env); + expect(ctx.waitUntil).toHaveBeenCalledWith(reportRequestMetrics(request, env)); + expect(response.status).toBe(200); + }); +}); diff --git a/wrangler.toml.example b/wrangler.toml.example index 5ca3df3..e7a549c 100644 --- a/wrangler.toml.example +++ b/wrangler.toml.example @@ -1,11 +1,16 @@ -name = "cf-metrics" -main = "index.js" +name = "spark-stats-request-metrics" +main = "bin/worker.js" compatibility_date = "2024-12-05" compatibility_flags = ["nodejs_compat"] -[vars] -REQUEST_URL = "https://example.com" -INFLUX_METRIC = "example" +[env.dev.vars] +ENVIRONMENT = "dev" +INFLUX_METRIC_NAME = "example" INFLUX_URL = "http://localhost:8086" -INFLUX_TOKEN = "example" -INFLUX_DATABASE = "example" \ No newline at end of file +INFLUX_DATABASE = "example" + +[env.dev.production] +ENVIRONMENT = "production" +INFLUX_METRIC_NAME = "$INFLUX_METRIC_NAME" +INFLUX_URL = "$INFLUX_URL" +INFLUX_DATABASE = "$INFLUX_DATABASE"