feat: support environment variable interpolation (#180)

* fix: add missing typescript type packages for validation handler * chore: refactor validation API to allow transmitting sample env * chore: wire env from validation payload in validation handler Note: Michele today found out that globally redeclaring ProcessEnv has a few caveats to being absolutely awesome. * fix: JSON stringify validation payload in Next.js * fix: adjust empty config validation to new shape of gateway event * fix: update content-type of validation requests to application/json * fix: stringify request body in tests before sending * fix: handle base64 encoding of event payload * feat: adapt validation in OTelBin next.js app to changed lambda contract --------- Co-authored-by: Marcel Birkner <[email protected]> Co-authored-by: Ben Blackmore <[email protected]>
dash0hq · Jan 11, 2024 · b71a90f · b71a90f
1 parent e902eb9
commit b71a90f
Show file tree

Hide file tree

Showing 10 changed files with 187 additions and 46 deletions.
diff --git a/packages/otelbin-validation-image/package-lock.json b/packages/otelbin-validation-image/package-lock.json
diff --git a/packages/otelbin-validation-image/package.json b/packages/otelbin-validation-image/package.json
@@ -18,12 +18,14 @@
   },
   "homepage": "https://github.com/dash0hq/otelbin#readme",
   "devDependencies": {
-    "@types/aws-lambda": "^8.10.129",
-    "esbuild": "^0.19.8"
+		"@types/aws-lambda": "^8.10.129",
+    "@types/js-yaml": "^4.0.8",
+    "@types/node": "^20.8.10",
+		"esbuild": "^0.19.8"
   },
   "dependencies": {
-    "aws-lambda": "^1.0.7",
     "@expo/spawn-async": "^1.7.2",
+    "aws-lambda": "^1.0.7",
     "js-yaml": "^4.1.0"
   }
 }
diff --git a/packages/otelbin-validation-image/src/index.ts b/packages/otelbin-validation-image/src/index.ts
@@ -1,5 +1,5 @@
 import { realpath, writeFile } from 'fs/promises';
-import { spawn } from 'node:child_process';
+import { spawn } from 'child_process';
 import spawnAsync from '@expo/spawn-async';
 import { APIGatewayProxyResult, APIGatewayEvent } from 'aws-lambda';
 import * as yaml from 'js-yaml';
@@ -12,20 +12,21 @@ interface SpawnError extends Error {
   signal: string;
 }
 
-declare global {
-  namespace NodeJS {
-    interface ProcessEnv {
-      DISTRO_NAME: string;
-    }
-  }
+interface ValidationPayload {
+  config: string;
+  env: Env;
+}
+
+interface Env {
+  [key: string]: string;
 }
 
 const distroName = process.env.DISTRO_NAME;
 
 const defaultErrorPrefix = 'Error: ';
 const adotInvalidConfigPrefix = 'Error: invalid configuration: ';
 
-export const validateAdot = async (otelcolRealPath: string, configPath: string): Promise<void> => {
+export const validateAdot = async (otelcolRealPath: string, configPath: string, env: Env): Promise<void> => {
   /*
 	 * ADOT does not support the `validate` subcommand
 	 * (see https://github.com/aws-observability/aws-otel-collector/issues/2391),
@@ -45,7 +46,12 @@ export const validateAdot = async (otelcolRealPath: string, configPath: string):
    * (see https://github.com/nodejs/node/issues/19218). Getting a shell around the otelcol binary
    * increases the reliability.
    */
-  const otelcol = spawn('/bin/sh', ['-c', `${otelcolRealPath} --config=${configPath}`]);
+  const otelcol = spawn('/bin/sh', ['-c', `${otelcolRealPath} --config=${configPath}`], {
+    env: {
+      ...process.env, // Ensure $PATH, terminal env vars and other basic niceties are set
+      ...env
+    },
+  });
 
   let stdout = '';
   let stderr = '';
@@ -91,7 +97,7 @@ export const validateAdot = async (otelcolRealPath: string, configPath: string):
     });
 };
 
-export const validateOtelCol = async (otelcolRealPath: string, configPath: string): Promise<void> => {
+export const validateOtelCol = async (otelcolRealPath: string, configPath: string, env: Env): Promise<void> => {
   /*
    * Node.js spawn is unreliable in terms of collecting stdout and stderr through the spawn call
    * (see https://github.com/nodejs/node/issues/19218). Getting a shell around the otelcol binary
@@ -114,16 +120,25 @@ const extractErrorPath = (errorMessage: string) => {
 };
 
 export const handler = async (event: APIGatewayEvent): Promise<APIGatewayProxyResult> => {
-  const config = event.body;
+  let body = event.body!;
+
+  if (event.isBase64Encoded) {
+    let buff = Buffer.from(body, 'base64');
+    body = buff.toString('ascii');
+  }
+
+  const validationPayload = JSON.parse(body) as ValidationPayload;
+  const config = validationPayload.config;
+  const env = validationPayload.env;
 
   if (
-    !config || // Empty string
-    !config?.trim().length || // Blank string (only whitespaces)
+    !validationPayload || // Empty event
+    !config || // Empty configuration string
+    !config?.trim().length || // Blank configuration string (only whitespaces)
     !Object.keys(yaml.load(config) as Object).length // Empty YAML
   ) {
     return {
       statusCode: 200,
-      // Unfortunately the collector returns one validation error at the time
       body: JSON.stringify({
         message: 'The provided configuration is invalid',
         error: 'the provided configuration is empty',
@@ -143,10 +158,10 @@ export const handler = async (event: APIGatewayEvent): Promise<APIGatewayProxyRe
 
     switch (distroName) {
       case 'adot':
-        await validateAdot(otelcolRealPath, configPath);
+        await validateAdot(otelcolRealPath, configPath, env);
         break;
       default:
-        await validateOtelCol(otelcolRealPath, configPath);
+        await validateOtelCol(otelcolRealPath, configPath, env);
     }
 
     return {

diff --git a/packages/otelbin-validation/src/main.ts b/packages/otelbin-validation/src/main.ts
@@ -10,14 +10,6 @@ import { Bucket, BlockPublicAccess } from 'aws-cdk-lib/aws-s3';
 import { BucketDeployment, Source } from 'aws-cdk-lib/aws-s3-deployment';
 import { Construct } from 'constructs';
 
-declare global {
-  namespace NodeJS {
-    interface ProcessEnv {
-      TEST_ENVIRONMENT_NAME: string;
-    }
-  }
-}
-
 export interface Distributions {
   [key: string]: Distribution;
 }

diff --git a/packages/otelbin-validation/test/assets/config-env-interpolation.yaml b/packages/otelbin-validation/test/assets/config-env-interpolation.yaml
@@ -0,0 +1,36 @@
+# Learn more about the OpenTelemetry Collector via
+# https://opentelemetry.io/docs/collector/
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+      http:
+
+processors:
+  batch:
+
+exporters:
+  otlp:
+    endpoint: ${OTLP_ENDPOINT}
+
+extensions:
+  health_check:
+  pprof:
+  zpages:
+
+service:
+  extensions: [health_check, pprof, zpages]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp]
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp]
+    logs:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp]
diff --git a/packages/otelbin-validation/test/main.test.ts b/packages/otelbin-validation/test/main.test.ts
@@ -8,6 +8,10 @@ import { describe, expect, jest, test } from '@jest/globals';
 import axios from 'axios';
 import { Distributions } from '../src/main';
 
+interface Env {
+  [key: string]: string;
+}
+
 const apiUrl = process.env.API_GATEWAY_URL?.replace(/[\/\\]+$/, '');
 const apiKey = process.env.VALIDATION_API_KEY;
 
@@ -45,15 +49,21 @@ const enumerateTestCases = () => {
 
 const assetFolderPath = join(__dirname, 'assets');
 
-const readConfig = (testConfigFilename: string) => readFileSync(join(assetFolderPath, testConfigFilename)).toString();
+const prepareValidationPayload = (testConfigFilename: string, env?: Env) => JSON.stringify({
+  config: readFileSync(join(assetFolderPath, testConfigFilename)).toString(),
+  env,
+});
 
 const defaultTimeout = 10_000; // 10 seconds
 
-const otelcolConfigValid = readConfig('config-default.yaml');
-const otelcolConfigInvalidNoReceivers = readConfig('config-no-receivers.yaml');
-const otelcolConfigInvalidUndeclaredExtension = readConfig('config-undeclared-extension.yaml');
-const otelcolConfigInvalidUndeclaredReceiver = readConfig('config-undeclared-receiver.yaml');
-const otelcolConfigInvalidUndeclaredReceiverNamedPipeline = readConfig('config-undeclared-receiver-named-pipelines.yaml');
+const otelcolConfigValid = prepareValidationPayload('config-default.yaml');
+const otelcolConfigValidEnvInterpolation = prepareValidationPayload('config-default.yaml', {
+  'OTLP_ENDPOINT': 'otelcol:4317',
+});
+const otelcolConfigInvalidNoReceivers = prepareValidationPayload('config-no-receivers.yaml');
+const otelcolConfigInvalidUndeclaredExtension = prepareValidationPayload('config-undeclared-extension.yaml');
+const otelcolConfigInvalidUndeclaredReceiver = prepareValidationPayload('config-undeclared-receiver.yaml');
+const otelcolConfigInvalidUndeclaredReceiverNamedPipeline = prepareValidationPayload('config-undeclared-receiver-named-pipelines.yaml');
 
 describe.each(enumerateTestCases())('Validation API', (distributionName, release) => {
 
@@ -72,7 +82,7 @@ describe.each(enumerateTestCases())('Validation API', (distributionName, release
       test(`has data about ${distributionName}/${release}`, async () => {
         const res = await axios.get(supportedDistributionsUrl, {
           headers: {
-            'Content-Type': 'application/yaml',
+            'Content-Type': 'application/json',
             'X-Api-Key': apiKey,
           },
         });
@@ -104,7 +114,21 @@ describe.each(enumerateTestCases())('Validation API', (distributionName, release
       test('accepts valid configuration', async () => {
         await expect(axios.post(validationUrl, otelcolConfigValid, {
           headers: {
-            'Content-Type': 'application/yaml',
+            'Content-Type': 'application/json',
+            'X-Api-Key': apiKey,
+          },
+        })).resolves.toMatchObject({
+          status: 200,
+          data: {
+            message: 'Configuration is valid',
+          },
+        });
+      }, defaultTimeout);
+
+      test('accepts valid configuration with env var interpolation', async () => {
+        await expect(axios.post(validationUrl, otelcolConfigValidEnvInterpolation, {
+          headers: {
+            'Content-Type': 'application/json',
             'X-Api-Key': apiKey,
           },
         })).resolves.toMatchObject({
@@ -115,10 +139,25 @@ describe.each(enumerateTestCases())('Validation API', (distributionName, release
         });
       }, defaultTimeout);
 
+      test('rejects empty validation payload', async () => {
+        await expect(axios.post(validationUrl, '{}', {
+          headers: {
+            'Content-Type': 'application/json',
+            'X-Api-Key': apiKey,
+          },
+        })).resolves.toMatchObject({
+          status: 200,
+          data: {
+            message: 'The provided configuration is invalid',
+            error: 'the provided configuration is empty',
+          },
+        });
+      }, defaultTimeout);
+
       test('rejects empty configuration', async () => {
-        await expect(axios.post(validationUrl, '', {
+        await expect(axios.post(validationUrl, '{"config":"", env: {"foo":"bar"}}', {
           headers: {
-            'Content-Type': 'application/yaml',
+            'Content-Type': 'application/json',
             'X-Api-Key': apiKey,
           },
         })).resolves.toMatchObject({
@@ -133,7 +172,7 @@ describe.each(enumerateTestCases())('Validation API', (distributionName, release
       test('rejects configuration without declared receivers', async () => {
         await expect(axios.post(validationUrl, otelcolConfigInvalidNoReceivers, {
           headers: {
-            'Content-Type': 'application/yaml',
+            'Content-Type': 'application/json',
             'X-Api-Key': apiKey,
           },
         })).resolves.toMatchObject({
@@ -148,7 +187,7 @@ describe.each(enumerateTestCases())('Validation API', (distributionName, release
       test('rejects configuration with undeclared receiver', async () => {
         await expect(axios.post(validationUrl, otelcolConfigInvalidUndeclaredReceiver, {
           headers: {
-            'Content-Type': 'application/yaml',
+            'Content-Type': 'application/json',
             'X-Api-Key': apiKey,
           },
         })).resolves.toMatchObject({
@@ -180,7 +219,7 @@ describe.each(enumerateTestCases())('Validation API', (distributionName, release
       test('rejects configuration with undeclared extension', async () => {
         await expect(axios.post(validationUrl, otelcolConfigInvalidUndeclaredExtension, {
           headers: {
-            'Content-Type': 'application/yaml',
+            'Content-Type': 'application/json',
             'X-Api-Key': apiKey,
           },
         })).resolves.toMatchObject({

diff --git a/packages/otelbin/package-lock.json b/packages/otelbin/package-lock.json
diff --git a/packages/otelbin/package.json b/packages/otelbin/package.json
@@ -55,7 +55,8 @@
 		"tailwindcss-animate": "^1.0.6",
 		"ts-jest": "^29.1.1",
 		"yaml": "^2.3.4",
-		"yaml-language-server": "^1.14.0"
+		"yaml-language-server": "^1.14.0",
+		"zod": "^3.22.4"
 	},
 	"devDependencies": {
 		"@svgr/webpack": "^8.1.0",