Add support for using shell env variables in config.json (#59)

During startup, envsubst(1) is used to replace env vars
in config.json and it is copied to /etc.

Version 2.2.0

cloudrun-malware-scanner/Dockerfile

@@ -48,6 +48,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         gnupg \
         jq \
         gawk \
+        gettext-base \
         clamav-daemon \
         clamav-freshclam \
         python3-crcmod && \

cloudrun-malware-scanner/bootstrap.sh

@@ -64,13 +64,20 @@ done &
 service clamav-daemon stop &
 service clamav-freshclam stop &
 
-# Get name of CVD Mirror bucket from config file.
+# Check and perform shell-varable substitution on config file, copying it to /etc
 #
+Log INFO main "Perfoming env var substitution on config file"
 CONFIG_FILE=./config.json
 if [[ ! -e "${CONFIG_FILE}" ]] ; then
     Log ERROR main "${CONFIG_FILE} does not exist"
     exit 1
 fi
+envsubst < "${CONFIG_FILE}" > /etc/malware-scanner-config.json
+CONFIG_FILE=/etc/malware-scanner-config.json
+
+# Get name of CVD Mirror bucket from config file
+#
+Log INFO main "Checking ClamCvdMirrorBucket from config file"
 CVD_MIRROR_BUCKET=$(/usr/bin/jq -r '.ClamCvdMirrorBucket' "${CONFIG_FILE}")
 if [[ -z "${CVD_MIRROR_BUCKET}" || "${CVD_MIRROR_BUCKET}" = "null" ]] ; then
     Log ERROR main "ClamCvdMirrorBucket is not defined in ${CONFIG_FILE}"
@@ -107,6 +114,8 @@ function updateClamConfigFile {
     Log INFO updateClamConfigFile "Updated ${CLAM_CONFIG_FILE} with parameters: ${MODIFIED_PARAMS}"
 }
 
+Log INFO main "Updating ClamAV config files"
+
 # Set Clam config file values
 # see clamd.conf documentation:
 # https://manpages.debian.org/bullseye/clamav-daemon/clamd.conf.5.en.html
@@ -164,4 +173,4 @@ service clamav-freshclam force-reload &
 
 # Run node server process
 Log INFO main "Starting malware-scanner service"
-npm start
+npm start "${CONFIG_FILE}"

cloudrun-malware-scanner/config.json.tmpl

@@ -7,7 +7,10 @@
     "Each object must have the 3 properties 'unscanned', 'clean' and 'quarantined', specifying the bucket names to use.",
     "",
     "'ClamCvdMirrorBucket' is a GCS bucket used to mirror the clamav database definition files to prevent overloading the Clam servers",
-    "and being rate limited/blacklisted. Its contents are maintained by the updateCvdMirror.sh script"
+    "and being rate limited/blacklisted. Its contents are maintained by the updateCvdMirror.sh script",
+    "",
+    "Shell environmental variable substitution is supported in this file.",
+    "At runtime, it will be copied to /etc"
   ],
   "buckets": [
     {

cloudrun-malware-scanner/logger.js

@@ -24,8 +24,8 @@ const pkgJson = require('./package.json');
 const loggingBunyan = new LoggingBunyan({
   redirectToStdout: true,
   projectId: process.env.PROJECT_ID,
-  logName: "malware-scanner",
-  useMessageField: false
+  logName: 'malware-scanner',
+  useMessageField: false,
 });
 
 exports.logger = bunyan.createLogger({

cloudrun-malware-scanner/metrics.js

@@ -98,21 +98,21 @@ function writeScanCompletedMetric_(measure, sourceBucket, destinationBucket,
 
 /**
  * Writes metrics when a CVD Mirror Update occurs.
- * 
+ *
  * @param {boolean} success
  * @param {boolean} isUpdated
  */
- function writeCvdMirrorUpdatedMetric(success, isUpdated) {
+function writeCvdMirrorUpdatedMetric(success, isUpdated) {
   const tags = new TagMap();
   tags.set(TAGS.cloudRunRevision, {value: process.env.K_REVISION});
-  tags.set(TAGS.cvdUpdateStatus, 
-    {value: (
+  tags.set(TAGS.cvdUpdateStatus,
+      {value: (
        success ? (
-        isUpdated ? "SUCCESS_UPDATED" : "SUCCESS_NO_UPDATES" )
-      : "FAILURE" )});
+        isUpdated ? 'SUCCESS_UPDATED' : 'SUCCESS_NO_UPDATES' ) :
+      'FAILURE' )});
   globalStats.record(
-    [{ measure: METRICS.cvdUpdates, value: 1}],
-    tags);
+      [{measure: METRICS.cvdUpdates, value: 1}],
+      tags);
 }
 
 /**
@@ -134,8 +134,8 @@ async function initMetrics(projectId) {
     TAGS.clamVersion,
     TAGS.cloudRunRevision,
     TAGS.sourceBucket,
-    TAGS.destinationBucket
-  ]
+    TAGS.destinationBucket,
+  ];
 
   METRICS.cleanFiles = globalStats.createMeasureInt64(
       METRIC_TYPE_ROOT + 'clean-files', MeasureUnit.UNIT,
@@ -183,7 +183,8 @@ async function initMetrics(projectId) {
       'The scan duration in milliseconds');
   const scanDurationView = globalStats.createView(
       METRICS.scanDuration.name, METRICS.scanDuration,
-      AggregationType.DISTRIBUTION, fileScanTags, 'Duration spent scanning files',
+      AggregationType.DISTRIBUTION, fileScanTags,
+      'Duration spent scanning files',
       // Bucket Boundaries in ms
       [
         0,
@@ -203,12 +204,12 @@ async function initMetrics(projectId) {
   globalStats.registerView(scanDurationView);
 
   METRICS.cvdUpdates = globalStats.createMeasureInt64(
-    METRIC_TYPE_ROOT + 'cvd-mirror-updates', MeasureUnit.UNIT,
-    'Number of CVD mirror Update Checks performed');
+      METRIC_TYPE_ROOT + 'cvd-mirror-updates', MeasureUnit.UNIT,
+      'Number of CVD mirror Update Checks performed');
 
   const cvdUpdatesView = globalStats.createView(
       METRICS.cvdUpdates.name, METRICS.cvdUpdates,
-      AggregationType.COUNT, 
+      AggregationType.COUNT,
       [TAGS.cloudRunRevision, TAGS.cvdUpdateStatus],
       'Number of CVD mirror update checks performed with their status');
   globalStats.registerView(cvdUpdatesView);

cloudrun-malware-scanner/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gcs-malware-scanner",
-  "version": "2.1.0",
+  "version": "2.2.0",
   "description": "Service to scan GCS documents for the malware and move the analyzed documents to appropriate buckets",
   "main": "index.js",
   "scripts": {

cloudrun-malware-scanner/server.js

@@ -17,7 +17,6 @@
 const clamd = require('clamdjs');
 const express = require('express');
 const {Storage} = require('@google-cloud/storage');
-const {ApiError} = require('@google-cloud/common');
 const {GoogleAuth} = require('google-auth-library');
 const {logger} = require('./logger.js');
 const pkgJson = require('./package.json');
@@ -44,12 +43,10 @@ const CLAMD_TIMEOUT = 600000;
 // large enough.
 const MAX_FILE_SIZE = 500000000; // 500MiB
 
-const CONFIG_FILE = './config.json';
-
 /**
  * Configuration object.
  *
- * Values are read from the CONFIG_FILE
+ * Values are read from the JSON configuration file.
  * See {@link readAndVerifyConfig}.
  *
  * @type {{
@@ -321,25 +318,28 @@ async function moveProcessedFile(filename, isClean, config) {
 }
 
 /**
- * Read configuration from CONFIG_FILE
+ * Read configuration from JSON configuration file.
  * and store in BUCKET_CONFIG global
  *
  * @async
+ * @param {string} configFile
  */
-async function readAndVerifyConfig() {
+async function readAndVerifyConfig(configFile) {
+  logger.info(`Using configuration file: ${configFile}`);
+
   try {
-    const config = require(CONFIG_FILE);
+    const config = require(configFile);
     delete config.comments;
     Object.assign(BUCKET_CONFIG, config);
   } catch (e) {
     logger.fatal(
         {err: e},
-        `Unable to read JSON file from ${CONFIG_FILE}`);
-    throw new Error(`Invalid configuration ${CONFIG_FILE}`);
+        `Unable to read JSON file from ${configFile}`);
+    throw new Error(`Invalid configuration ${configFile}`);
   }
 
   if (BUCKET_CONFIG.buckets.length === 0) {
-    logger.fatal(`No buckets configured for scanning in ${CONFIG_FILE}`);
+    logger.fatal(`No buckets configured for scanning in ${configFile}`);
     throw new Error('No buckets configured');
   }
 
@@ -360,7 +360,7 @@ async function readAndVerifyConfig() {
         config.unscanned === config.quarantined ||
         config.clean === config.quarantined) {
       logger.fatal(
-          `Error in ${CONFIG_FILE} buckets[${x}]: bucket names are not unique`);
+          `Error in ${configFile} buckets[${x}]: bucket names are not unique`);
       success = false;
     }
   }
@@ -440,7 +440,15 @@ async function run() {
     projectId = await (new GoogleAuth().getProjectId());
   }
   await metrics.init(projectId);
-  await readAndVerifyConfig();
+
+  let configFile;
+  if (process.argv.length >= 3) {
+    configFile = process.argv[2];
+  } else {
+    configFile = './config.json';
+  }
+  await readAndVerifyConfig(configFile);
+
   await waitForClamD();
 
   app.listen(PORT, () => {