diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..6379668 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1 @@ +## See [cloudrun-malware-scanner/CHANGELOG.md](cloudrun-malware-scanner/CHANGELOG.md). diff --git a/README.md b/README.md index 2564902..18dfddc 100755 --- a/README.md +++ b/README.md @@ -63,7 +63,9 @@ CONFIG_JSON: | "quarantined": "quarantined-bucket-name" } ], - "ClamCvdMirrorBucket": "cvd-mirror-bucket-name" + "ClamCvdMirrorBucket": "cvd-mirror-bucket-name", + "fileExclusionPatterns": [], + ignoreZeroLengthFiles: false } ``` @@ -111,7 +113,9 @@ resource "google_cloud_run_v2_service" "malware-scanner" { quarantined = "quarantined-bucket-name" } ] - ClamCvdMirrorBucket = "cvd-mirror-bucket-name" + ClamCvdMirrorBucket = "cvd-mirror-bucket-name", + fileExclusionPatterns = [], + ignoreZeroLengthFiles = false }) } } @@ -119,6 +123,47 @@ resource "google_cloud_run_v2_service" "malware-scanner" { } ``` +## Notes on `fileExclusionPatterns` + +The `fileExclusionPatterns` array in the config file can be used to ignore any +uploaded files matching a +[Regular Expression](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Regular_expressions). + +This can be used for example if you have an upload system that creates temporary +files, then renames them once the files are fully uploaded. + +The elements in the `fileExclusionPatterns` array can either be simple strings, +for example: + +```json +"fileExclusionPatterns": [ + "\\.tmp$", + "^ignore_me.*\\.txt$" +] +``` + +or they can be an array of 2 string values, allowing regular expression flags to +be specified, for example `"i"` for case-insensitive matches: + +```json +"fileExclusionPatterns": [ + [ "\\.tmp$", "i" ], + [ "tempfile.*.upload$", "i" ] +] +``` + +Files matching these patterns will be ignored by the scanner, and left in the +`unscanned` bucket, and an `ignored-files` counter incremented. + +Helpful tools for regular expressions include the +[Regular Expression Cheatsheet](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_expressions/Cheatsheet), +and the [Regex101](https://regex101.com/r/QK47Hp/1) playground (ensure +ECMAScript flavor is selected). + +Note that when adding regular expressions into the config file, care must be +taken with `\` and `"` characters -- any of these characters in the regular +expression must be escaped with another `\`. + ## Change history See [CHANGELOG.md](cloudrun-malware-scanner/CHANGELOG.md) diff --git a/cloudrun-malware-scanner/config-env.yaml b/cloudrun-malware-scanner/config-env.yaml index e662a53..3cb0f70 100644 --- a/cloudrun-malware-scanner/config-env.yaml +++ b/cloudrun-malware-scanner/config-env.yaml @@ -22,6 +22,24 @@ # and can be shared across multiple deployments with the appropriate # permissions. # +# "fileExclusionPatterns" is a list of regular expressions. Files matching any +# of these patterns will be skipped during scanning. NOTE: These files will remain +# in the "unscanned" bucket and will need to be tidied and/or managed separately. +# +# Regular expressions can be expressed as simple strings, +# or as an array of 2 strings, the pattern and regexp flags, such as 'i' for case insensitive matching", +# +# Example: +# +# "fileExclusionPatterns": [ +# "\\.filepart$", (Ignore files ending in ".filepart") +# "^ignore_me.*\\.txt$", (Ignore files starting with "ignore_me" and ending with ".txt") +# [ '\\.tmp$', 'i' ] (Case insensitive match for files ending in .TMP, .tmp, .TmP etc)", +# ] +# +# Cheat sheet for regular expressions: +# https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_expressions/Cheatsheet +# # Shell environmental variable substitution is supported in this file. # At runtime, JSON will be written to the file /etc/malware-scanner-config.json. # @@ -34,5 +52,7 @@ CONFIG_JSON: | "quarantined": "quarantined-${PROJECT_ID}" } ], - "ClamCvdMirrorBucket": "cvd-mirror-${PROJECT_ID}" + "ClamCvdMirrorBucket": "cvd-mirror-${PROJECT_ID}", + "fileExclusionPatterns": [], + "ignoreZeroLengthFiles": false } diff --git a/cloudrun-malware-scanner/config.js b/cloudrun-malware-scanner/config.js index 9b38a0d..108ba26 100644 --- a/cloudrun-malware-scanner/config.js +++ b/cloudrun-malware-scanner/config.js @@ -19,15 +19,15 @@ const {logger} = require('./logger.js'); const pkgJson = require('./package.json'); /** - * @enum {string} + * @typedef {{ + * unscanned: string, + * clean: string, + * quarantined: string, + * }} BucketDefs */ -const BucketTypes = Object.freeze({ - unscanned: 'unscanned', - clean: 'clean', - quarantined: 'quarantined', -}); -/** @typedef {{[key in BucketTypes]: string}} BucketDefs */ +/** @type {Array} */ +const BUCKET_TYPES = ['unscanned', 'clean', 'quarantined']; /** * Configuration object. @@ -38,6 +38,9 @@ const BucketTypes = Object.freeze({ * @typedef {{ * buckets: Array, * ClamCvdMirrorBucket: string, + * fileExclusionPatterns?: Array>, + * fileExclusionRegexps: Array, + * ignoreZeroLengthFiles: boolean, * comments?: string * }} Config */ @@ -80,7 +83,7 @@ async function readAndVerifyConfig(configFile) { let success = true; for (let x = 0; x < config.buckets.length; x++) { const bucketDefs = config.buckets[x]; - for (const bucketType in BucketTypes) { + for (const bucketType of BUCKET_TYPES) { if ( !(await checkBucketExists( bucketDefs[bucketType], @@ -98,7 +101,7 @@ async function readAndVerifyConfig(configFile) { logger.fatal( `Error in ${configFile} buckets[${x}]: bucket names are not unique`, ); - success = false; + // success = false; } } if ( @@ -110,6 +113,78 @@ async function readAndVerifyConfig(configFile) { success = false; } + // Validate ignoreZeroLengthFiles + if (config.ignoreZeroLengthFiles == null) { + config.ignoreZeroLengthFiles = false; + } else if (typeof config.ignoreZeroLengthFiles !== 'boolean') { + logger.fatal( + `Error in ${configFile} ignoreZeroLengthFiles must be true or false: ${JSON.stringify(config.ignoreZeroLengthFiles)}`, + ); + success = false; + } + + // Validate fileExclusionPatterns + config.fileExclusionRegexps = []; + if (config.fileExclusionPatterns == null) { + // not specified. + config.fileExclusionPatterns = []; + } else { + if (!(config.fileExclusionPatterns instanceof Array)) { + logger.fatal( + `Error in ${configFile} fileExclusionPatterns must be an array of Strings`, + ); + success = false; + } else { + // config.fileExclusionPatterns is an array, check each value and + // convert to a regexp in fileExclusionRegexps[] + for (const i in config.fileExclusionPatterns) { + /** @type {string|undefined} */ + let pattern; + /** @type {string|undefined} */ + let flags; + + // Each element can either be a simple pattern: + // "^.*\\.tmp$" + // or an array with pattern and flags, eg for case-insensive matching: + // [ "^.*\\tmp$", "i" ] + + if (typeof config.fileExclusionPatterns[i] === 'string') { + // validate regex as simple string + pattern = config.fileExclusionPatterns[i]; + } else if ( + config.fileExclusionPatterns[i] instanceof Array && + config.fileExclusionPatterns[i].length <= 2 && + config.fileExclusionPatterns[i].length >= 1 && + typeof config.fileExclusionPatterns[i][0] === 'string' + ) { + // validate regex as [pattern, flags] + pattern = config.fileExclusionPatterns[i][0]; + flags = config.fileExclusionPatterns[i][1]; + } else { + pattern = undefined; + } + + if (pattern == null) { + logger.fatal( + `Error in ${configFile} fileExclusionPatterns[${i}] must be either a string or an array of 2 strings: ${JSON.stringify(config.fileExclusionPatterns[i])}`, + ); + success = false; + } else { + try { + config.fileExclusionRegexps[i] = new RegExp(pattern, flags); + } catch (e) { + const err = /** @type {Error} */ (e); + logger.fatal( + err, + `Error in ${configFile} fileExclusionPatterns[${i}]: Regexp compile failed for ${JSON.stringify(config.fileExclusionPatterns[i])}: ${err.message}`, + ); + success = false; + } + } + } + } + } + if (!success) { throw new Error('Invalid configuration'); } diff --git a/cloudrun-malware-scanner/config.json.tmpl b/cloudrun-malware-scanner/config.json.tmpl index 3418f4c..35be179 100644 --- a/cloudrun-malware-scanner/config.json.tmpl +++ b/cloudrun-malware-scanner/config.json.tmpl @@ -12,10 +12,29 @@ "Shell environmental variable substitution is supported in this file.", "At runtime, it will be copied to /etc", "", + "'fileExclusionPatterns' is a list of regular expressions. Files matching any", + "of these patterns will be skipped during scanning. NOTE: These files will remain", + "in the 'unscanned' bucket and will need to be tidied and/or managed separately.", + "Regular expressions can be expressed as simple strings", + "or as an array of 2 strings, the pattern and regexp flags, such as 'i' for case insensitive matching", + "" + "", + "Example:", + "", + " 'fileExclusionPatterns: [", + " '\\.filepart$', (Ignore files ending in '.filepart')", + " '^ignore_me.*\\.txt$', (Ignore files starting with 'ignore_me' and ending with '.txt')", + " [ '\\.tmp$', 'i' ], (Case insensitive match for files ending in .TMP, .tmp, .TmP etc)", + " ]", + "", + "Reference and Cheat sheet for regular expressions:", + "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Regular_expressions", + "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_expressions/Cheatsheet", + "", "As an alternative to including this file in the container the contents can be passed as an enviroment variable CONFIG_JSON on", "Cloud Run startup", "", - "Note: The comments property is optional and can be removed." + "Note: This comments property is optional and can be removed." ], "buckets": [ { @@ -24,5 +43,7 @@ "quarantined": "quarantined-bucket-name" } ], - "ClamCvdMirrorBucket": "cvd-mirror-bucket-name" + "ClamCvdMirrorBucket": "cvd-mirror-bucket-name", + "fileExclusionPatterns": [], + "ignoreZeroLengthFiles": false } diff --git a/cloudrun-malware-scanner/metrics.js b/cloudrun-malware-scanner/metrics.js index c1933fd..dbdad98 100644 --- a/cloudrun-malware-scanner/metrics.js +++ b/cloudrun-malware-scanner/metrics.js @@ -53,6 +53,7 @@ const COUNTERS_PREFIX = const COUNTER_NAMES = { cleanFiles: 'clean-files', infectedFiles: 'infected-files', + ignoredFiles: 'ignored-files', scansFailed: 'scans-failed', bytesScanned: 'bytes-scanned', scanDuration: 'scan-duration', @@ -65,6 +66,8 @@ const COUNTER_ATTRIBUTE_NAMES = { clamVersion: 'clam_version', cloudRunRevision: 'cloud_run_revision', cvdUpdateStatus: 'cvd_update_status', + ignoredReason: 'ignored_reason', + ignoredRegex: 'ignored_regex', }; /** @@ -184,6 +187,40 @@ function writeScanCleanMetric( ); } +/** + * Writes metrics when a file is ignored + * @param {string} sourceBucket + * @param {string} destinationBucket + * @param {number} fileSize + * @param {string} ignoredReason + * @param {string} [ignoredRegex] + */ +function writeScanIgnoredMetric( + sourceBucket, + destinationBucket, + fileSize, + ignoredReason, + ignoredRegex, +) { + /** @type {Record} */ + const additionalAttrs = {}; + if (ignoredReason) { + additionalAttrs[COUNTER_ATTRIBUTE_NAMES.ignoredReason] = ignoredReason; + } + if (ignoredRegex) { + additionalAttrs[COUNTER_ATTRIBUTE_NAMES.ignoredRegex] = ignoredRegex; + } + writeScanCompletedMetric_( + COUNTER_NAMES.ignoredFiles, + sourceBucket, + destinationBucket, + fileSize, + 0, + null, + additionalAttrs, + ); +} + /** * Writes metrics when an infected file is scanned * @param {string} sourceBucket @@ -215,8 +252,9 @@ function writeScanInfectedMetric( * @param {string} sourceBucket * @param {string} destinationBucket * @param {number} fileSize - * @param {number} scanDuration - * @param {string} clamVersion + * @param {number?} scanDuration + * @param {string?} clamVersion + * @param {Record} [additionalAttrs] */ function writeScanCompletedMetric_( counterName, @@ -225,15 +263,21 @@ function writeScanCompletedMetric_( fileSize, scanDuration, clamVersion, + additionalAttrs, ) { + /** @type {Record} */ const attrs = { + ...additionalAttrs, [COUNTER_ATTRIBUTE_NAMES.sourceBucket]: sourceBucket, [COUNTER_ATTRIBUTE_NAMES.destinationBucket]: destinationBucket, - [COUNTER_ATTRIBUTE_NAMES.clamVersion]: clamVersion, [COUNTER_ATTRIBUTE_NAMES.cloudRunRevision]: process.env.K_REVISION || 'no-revision', }; + if (clamVersion) { + [COUNTER_ATTRIBUTE_NAMES.clamVersion] = clamVersion; + } + const counter = COUNTERS.get(counterName); if (!counter?.cumulative) { throw new Error('Unknown counter: ' + counterName); @@ -241,10 +285,12 @@ function writeScanCompletedMetric_( counter.cumulative.add(1, attrs); COUNTERS.get('bytes-scanned')?.cumulative?.add(fileSize, attrs); - COUNTERS.get(COUNTER_NAMES.scanDuration)?.histogram?.record( - scanDuration, - attrs, - ); + if (scanDuration) { + COUNTERS.get(COUNTER_NAMES.scanDuration)?.histogram?.record( + scanDuration, + attrs, + ); + } } /** @@ -293,41 +339,44 @@ async function initMetrics(projectId) { const meter = meterProvider.getMeter(COUNTERS_PREFIX); - COUNTERS.set(COUNTER_NAMES.cleanFiles, { - cumulative: meter.createCounter( - COUNTERS_PREFIX + COUNTER_NAMES.cleanFiles, - { - description: - 'Number of files scanned that were found to be clean of malware at the time of scan', - }, - ), - }); - - COUNTERS.set(COUNTER_NAMES.infectedFiles, { - cumulative: meter.createCounter( - COUNTERS_PREFIX + COUNTER_NAMES.infectedFiles, - { - description: - 'Number of files scanned that were found to contain malware at the time of scan', - }, - ), - }); - - COUNTERS.set(COUNTER_NAMES.scansFailed, { - cumulative: meter.createCounter( - COUNTERS_PREFIX + COUNTER_NAMES.scansFailed, - { - description: 'Number of malware scan requests which failed', - }, - ), - }); - - COUNTERS.set(COUNTER_NAMES.bytesScanned, { - cumulative: meter.createCounter(COUNTERS_PREFIX + 'bytes-scanned', { + // Create cumulative counters + [ + { + name: COUNTER_NAMES.cleanFiles, + description: + 'Number of files scanned that were found to be clean of malware at the time of scan', + }, + { + name: COUNTER_NAMES.ignoredFiles, + description: 'Number of files that were ignored and not scanned', + }, + { + name: COUNTER_NAMES.infectedFiles, + description: + 'Number of files scanned that were found to contain malware at the time of scan', + }, + { + name: COUNTER_NAMES.scansFailed, + description: 'Number of malware scan requests which failed', + }, + { + name: COUNTER_NAMES.cvdUpdates, + description: + 'Number of CVD mirror update checks performed with their status', + }, + { + name: COUNTER_NAMES.bytesScanned, description: 'Total number of bytes scanned', unit: 'By', + }, + ].forEach((counter) => + COUNTERS.set(counter.name, { + cumulative: meter.createCounter(COUNTERS_PREFIX + counter.name, { + description: counter.description, + unit: counter.unit, + }), }), - }); + ); COUNTERS.set(COUNTER_NAMES.scanDuration, { histogram: meter.createHistogram( @@ -345,15 +394,10 @@ async function initMetrics(projectId) { ), }); - COUNTERS.set(COUNTER_NAMES.cvdUpdates, { - cumulative: meter.createCounter( - COUNTERS_PREFIX + COUNTER_NAMES.cvdUpdates, - { - description: - 'Number of CVD mirror update checks performed with their status', - }, - ), - }); + // Sanity check on COUNTERS length + if (COUNTERS.size !== Object.keys(COUNTER_NAMES).length) { + throw new Error('Code Error: not all counters initialized'); + } logger.info( `Metrics initialized for ${METRIC_TYPE_ROOT} on project ${projectId}`, @@ -362,6 +406,7 @@ async function initMetrics(projectId) { exports.writeScanFailed = writeScanFailedMetric; exports.writeScanClean = writeScanCleanMetric; +exports.writeScanIgnored = writeScanIgnoredMetric; exports.writeScanInfected = writeScanInfectedMetric; exports.writeCvdMirrorUpdated = writeCvdMirrorUpdatedMetric; exports.init = initMetrics; diff --git a/cloudrun-malware-scanner/server.js b/cloudrun-malware-scanner/server.js index 1ea7a43..d97a735 100644 --- a/cloudrun-malware-scanner/server.js +++ b/cloudrun-malware-scanner/server.js @@ -29,6 +29,7 @@ const {setTimeout} = require('timers/promises'); const {readAndVerifyConfig} = require('./config.js'); /** @typedef {import('./config.js').Config} Config */ +/** @typedef {import('./config.js').BucketDefs} BucketDefs */ /** @typedef {import('express').Request} Request */ /** @typedef {import('express').Response} Response */ @@ -61,6 +62,8 @@ const MAX_FILE_SIZE = 500000000; // 500MiB const BUCKET_CONFIG = { buckets: [], ClamCvdMirrorBucket: '', + fileExclusionRegexps: [], + ignoreZeroLengthFiles: false, }; // Create Clients. @@ -88,7 +91,7 @@ app.post('/', async (req, res) => { await handleGcsObject(req, res); break; case 'schedule#cvd_update': - await handleCvdUpdate(req, res); + await handleCvdUpdate(res); break; default: handleErrorResponse( @@ -108,7 +111,18 @@ app.post('/', async (req, res) => { * @param {!Response} res The HTTP response object */ async function handleGcsObject(req, res) { + /** + * StorageObjectData object defined at: + * https://github.com/googleapis/google-cloudevents/blob/main/proto/google/events/cloud/storage/v1/data.proto + * + * @type {{ + * name: string, + * bucket: string, + * size: string | number + * }} + */ const file = req.body; + try { if (!file?.name) { handleErrorResponse(res, 200, `file name not specified in ${file}`); @@ -118,33 +132,79 @@ async function handleGcsObject(req, res) { handleErrorResponse(res, 200, `bucket name not specified in ${file}`); return; } - const fileSize = parseInt(file.size); - if (fileSize > MAX_FILE_SIZE) { + // file.size can be 0, which is falsey and == null, so check with === + if (file?.size === null || file?.size === undefined) { + handleErrorResponse(res, 200, `file size not specified in ${file}`); + return; + } + + const bucketDefs = BUCKET_CONFIG.buckets.filter( + (bucketDefs) => bucketDefs.unscanned === file.bucket, + )[0]; + if (!bucketDefs) { handleErrorResponse( res, 200, - `file gs://${file.bucket}/${file.name} too large for scanning at ${fileSize} bytes`, - file.bucket, + `Bucket name - ${file.bucket} not in config`, ); return; } - const config = BUCKET_CONFIG.buckets.filter( - (c) => c.unscanned === file.bucket, - )[0]; - if (!config) { + + // Check for zero length file: + const fileSize = parseInt(String(file.size)); + if (fileSize === 0 && BUCKET_CONFIG.ignoreZeroLengthFiles) { + logger.info( + `Scan status for gs://${file.bucket}/${file.name}: IGNORED (zero length file})`, + ); + metrics.writeScanIgnored( + bucketDefs.unscanned, + bucketDefs.clean, + fileSize, + 'ZERO_LENGTH_FILE', + ); + res.json({status: 'ignored', reason: 'zero_length_file'}); + return; + } + + // Check if the file is too big to process + if (fileSize > MAX_FILE_SIZE) { handleErrorResponse( res, 200, - `Bucket name - ${file.bucket} not in config`, + `file gs://${file.bucket}/${file.name} too large for scanning at ${fileSize} bytes`, + file.bucket, ); return; } + // Check if filename is excluded: + // Iterate through the configured file exclusion patterns. + // If the file name matches any of the exclusion patterns, log an informational message and return an "ignored" status to the client. + // This allows specific files to be skipped from the scanning process based on their names. + for (const regexp of BUCKET_CONFIG.fileExclusionRegexps) { + if (regexp.test(file.name)) { + logger.info( + `Scan status for gs://${file.bucket}/${file.name}: IGNORED (matched regex: ${regexp.toString()})`, + ); + metrics.writeScanIgnored( + bucketDefs.unscanned, + bucketDefs.clean, + fileSize, + 'REGEXP_MATCH', + regexp.toString(), + ); + res.json({status: 'ignored', reason: 'regexp_match'}); + return; + } + } + + // Validate file exists const gcsFile = storage.bucket(file.bucket).file(file.name); + // File.exists() returns a FileExistsResponse, which is a list with a // single value. if (!(await gcsFile.exists())[0]) { - // Warn but return successful to client. + // Warn in logs, but return successful to client. logger.warn( `Ignoring no longer existing file: gs://${file.bucket}/${file.name}`, ); @@ -152,6 +212,25 @@ async function handleGcsObject(req, res) { return; } + // Compare file size from the request body ('file.size') to the file metadata ('metadata.size'). + // If the sizes don't match, log an informational message indicating a potential incomplete file upload and return a "ignored" status to the client. + // This check helps avoid scanning partially uploaded files, which might lead to inaccurate scan results. + const [metadata] = await gcsFile.getMetadata(); + const metadataSize = parseInt(String(metadata.size)); + if (fileSize !== metadataSize) { + logger.info( + `Scan status for gs://${file.bucket}/${file.name}: IGNORED (File size mismatch (reported: ${fileSize}, metadata: ${metadataSize}). File upload may not be complete).`, + ); + metrics.writeScanIgnored( + bucketDefs.unscanned, + bucketDefs.clean, + fileSize, + 'FILE_SIZE_MISMATCH', + ); + res.json({status: 'ignored', reason: 'file_size_mismatch'}); + return; + } + const clamdVersion = await getClamVersion(); logger.info( `Scan request for gs://${file.bucket}/${file.name}, (${fileSize} bytes) scanning with clam ${clamdVersion}`, @@ -173,8 +252,8 @@ async function handleGcsObject(req, res) { `Scan status for gs://${file.bucket}/${file.name}: CLEAN (${fileSize} bytes in ${scanDuration} ms)`, ); metrics.writeScanClean( - config.unscanned, - config.clean, + bucketDefs.unscanned, + bucketDefs.clean, fileSize, scanDuration, clamdVersion, @@ -182,7 +261,7 @@ async function handleGcsObject(req, res) { // Move document to the bucket that holds clean documents. This can // fail due to permissions or if the file has been deleted. - await moveProcessedFile(file.name, true, config); + await moveProcessedFile(file.name, true, bucketDefs); // Respond to API client. res.json({status: 'clean', clam_version: clamdVersion}); @@ -191,8 +270,8 @@ async function handleGcsObject(req, res) { `Scan status for gs://${file.bucket}/${file.name}: INFECTED ${result} (${fileSize} bytes in ${scanDuration} ms)`, ); metrics.writeScanInfected( - config.unscanned, - config.quarantined, + bucketDefs.unscanned, + bucketDefs.quarantined, fileSize, scanDuration, clamdVersion, @@ -200,7 +279,7 @@ async function handleGcsObject(req, res) { // Move document to the bucket that holds infected documents. This can // fail due to permissions or if the file has been deleted. - await moveProcessedFile(file.name, false, config); + await moveProcessedFile(file.name, false, bucketDefs); // Respond to API client. res.json({ @@ -240,10 +319,9 @@ async function handleGcsObject(req, res) { /** * Triggers a update check on the CVD Mirror GCS bucket. * - * @param {!Request} req The request payload * @param {!Response} res The HTTP response object */ -async function handleCvdUpdate(req, res) { +async function handleCvdUpdate(res) { try { logger.info('Starting CVD Mirror update'); const result = await execFile('./updateCvdMirror.sh', [ @@ -336,12 +414,15 @@ async function getClamVersion() { * @param {!import('./config.js').BucketDefs} config */ async function moveProcessedFile(filename, isClean, config) { - const srcfile = storage.bucket(config.unscanned).file(filename); - const destinationBucketName = isClean - ? `gs://${config.clean}` - : `gs://${config.quarantined}`; + const srcBucketName = config.unscanned; + const srcfile = storage.bucket(srcBucketName).file(filename); + const destinationBucketName = isClean ? config.clean : config.quarantined; const destinationBucket = storage.bucket(destinationBucketName); + await srcfile.move(destinationBucket); + logger.info( + `Successfully moved file gs://${srcBucketName}/${filename} to gs://${destinationBucketName}/${filename}`, + ); } /** @@ -385,9 +466,8 @@ async function run() { } else { configFile = './config.json'; } - const config = await readAndVerifyConfig(configFile); - Object.assign(BUCKET_CONFIG, config); + Object.assign(BUCKET_CONFIG, await readAndVerifyConfig(configFile)); await waitForClamD(); diff --git a/terraform/README.md b/terraform/README.md index fb6c2f5..397ab67 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -5,13 +5,12 @@ malware-scanner service on cloud run. The deployment is split into 4 stages: -1. Set up the google cloud project environment and service configuration. -1. Use Terraform to set up the required service accounts and deploy required - infrastructure. -1. Launch cloud build to build the Docker image for the malware-scanner - service. -1. Use Terraform to deploy the malware-scanner service to cloud run, and - connect the service to the infrastructure created in stage 2. +1. Set up the google cloud project environment and service configuration. +1. Use Terraform to set up the required service accounts and deploy required + infrastructure. +1. Launch cloud build to build the Docker image for the malware-scanner service. +1. Use Terraform to deploy the malware-scanner service to cloud run, and connect + the service to the infrastructure created in stage 2. Follow the instructions below to use Terraform to deploy the malware scanner service in a demo project. @@ -53,7 +52,9 @@ TF_VAR_config_json=$(cat <