Enhance testing capabilities

test/fixtures.ts

@@ -188,7 +188,9 @@ abstract class LanguageFixture extends Fixture {
     abstract test(
         filename: string,
         additionalRendererOptions: RendererOptions,
-        additionalFiles: string[]
+        additionalFiles: string[],
+        testComparisonArgs?: Partial<ComparisonArgs>,
+        expectedFilename?: string
     ): Promise<number>;
 
     additionalFiles(_sample: Sample): string[] {
@@ -198,6 +200,7 @@ abstract class LanguageFixture extends Fixture {
     async runWithSample(sample: Sample, index: number, total: number) {
         const cwd = this.getRunDirectory();
         const sampleFile = path.resolve(sample.path);
+        const sampleOutFile = sample.outPath ? path.resolve(sample.outPath) : undefined;
         const shouldSkip = this.shouldSkipTest(sample);
         const additionalFiles = this.additionalFiles(sample).map(p => path.resolve(p));
 
@@ -223,7 +226,13 @@ abstract class LanguageFixture extends Fixture {
 
             try {
                 numFiles = await timeout(
-                    this.test(sampleFile, sample.additionalRendererOptions, additionalFiles),
+                    this.test(
+                        sampleFile,
+                        sample.additionalRendererOptions,
+                        additionalFiles,
+                        sample.comparisonArgs,
+                        sampleOutFile
+                    ),
                     MAX_TEST_RUNTIME_MS
                 );
             } catch (e) {
@@ -254,7 +263,10 @@ abstract class LanguageFixture extends Fixture {
 }
 
 class JSONFixture extends LanguageFixture {
-    constructor(language: languages.Language, public name: string = language.name) {
+    constructor(
+        language: languages.Language,
+        public name: string = language.name
+    ) {
         super(language);
     }
 
@@ -270,14 +282,28 @@ class JSONFixture extends LanguageFixture {
     async test(
         filename: string,
         additionalRendererOptions: RendererOptions,
-        _additionalFiles: string[]
+        _additionalFiles: string[],
+        testComparisonArgs?: Partial<ComparisonArgs>,
+        expectedFilename?: string
     ): Promise<number> {
         if (this.language.compileCommand) {
             await execAsync(this.language.compileCommand);
         }
         if (this.language.runCommand === undefined) return 0;
 
-        compareJsonFileToJson(comparisonArgs(this.language, filename, filename, additionalRendererOptions));
+        let sampleComparisonArgs = comparisonArgs(
+            this.language,
+            filename,
+            expectedFilename ? expectedFilename : filename,
+            additionalRendererOptions
+        );
+
+        // Add any additional comparison args specified in optionMap for this test
+        if (testComparisonArgs) {
+            sampleComparisonArgs = { ...sampleComparisonArgs, ...testComparisonArgs };
+        }
+
+        compareJsonFileToJson(sampleComparisonArgs);
 
         if (this.language.diffViaSchema && !_.includes(this.language.skipDiffViaSchema, path.basename(filename))) {
             debug("* Diffing with code generated via JSON Schema");
@@ -304,6 +330,9 @@ class JSONFixture extends LanguageFixture {
         if (fs.statSync(sample.path).size > 32 * 1024 * 1024) {
             return true;
         }
+        if (sample.language && this.language.name !== sample.language.name) {
+            return true;
+        }
         if (this.language.includeJSON !== undefined) {
             return !_.includes(this.language.includeJSON, path.basename(sample.path));
         }
@@ -568,7 +597,10 @@ class JSONTypeScriptFixture extends JSONToXToYFixture {
 
 // This fixture tests generating code from JSON Schema.
 class JSONSchemaFixture extends LanguageFixture {
-    constructor(language: languages.Language, readonly name: string = `schema-${language.name}`) {
+    constructor(
+        language: languages.Language,
+        readonly name: string = `schema-${language.name}`
+    ) {
         super(language);
     }
 
@@ -699,7 +731,10 @@ class GraphQLFixture extends LanguageFixture {
 }
 
 class CommandSuccessfulLanguageFixture extends LanguageFixture {
-    constructor(language: languages.Language, public name: string = language.name) {
+    constructor(
+        language: languages.Language,
+        public name: string = language.name
+    ) {
         super(language);
     }
 

test/inputs/json/misc/227e2.in.json

@@ -0,0 +1,11 @@
+{
+    "results": [
+        {
+            "age": 52,
+            "name": null
+        },
+        {
+            "age": 27
+        }
+    ]
+}

test/inputs/json/misc/227e2.out.json

@@ -0,0 +1,10 @@
+{
+    "results": [
+        {
+            "age": 52
+        },
+        {
+            "age": 27
+        }
+    ]
+}

test/lib/optionMap.ts

@@ -0,0 +1,23 @@
+import { RendererOptions } from "quicktype-core";
+import { ComparisonArgs } from "../utils";
+import { GoLanguage, Language } from "../languages";
+
+type TestOptions = {
+    cliOptions: RendererOptions;
+    language: Language;
+    comparisonArgs?: Partial<ComparisonArgs>;
+};
+
+const optionMap: Record<string, TestOptions> = {
+    "test/inputs/json/misc/227e2.in.json": {
+        cliOptions: {
+            "omit-empty": true
+        },
+        language: GoLanguage,
+        comparisonArgs: {
+            strict: true
+        }
+    }
+};
+
+export default optionMap;

test/utils.ts

@@ -1,20 +1,26 @@
 import * as fs from "fs";
 import * as path from "path";
 
-import * as _ from "lodash";
+import _ from "lodash";
 import * as shell from "shelljs";
 
 import { main as quicktype_, CLIOptions } from "../src";
 import { RendererOptions } from "quicktype-core";
 import * as languages from "./languages";
 import deepEquals from "./lib/deepEquals";
 
+import optionMap from "./lib/optionMap";
+
 import chalk from "chalk";
+import { Language } from "./languages";
 const strictDeepEquals: (x: any, y: any) => boolean = require("deep-equal");
 
 const DEBUG = process.env.DEBUG !== undefined;
 const ASSUME_STRINGS_EQUAL = process.env.ASSUME_STRINGS_EQUAL !== undefined;
 
+const inputFilePattern = /^(.*)\.in\.(.*)$/;
+const outputFilePattern = /^.*\.out\..*$/;
+
 export function debug<T>(x: T): T {
     if (DEBUG) {
         console.log(x);
@@ -180,10 +186,49 @@ export interface Sample {
     path: string;
     additionalRendererOptions: RendererOptions;
     saveOutput: boolean;
+    outPath?: string;
+    comparisonArgs?: Partial<ComparisonArgs>;
+    language?: Language;
+}
+
+function sampleFromPath(path: string): Sample {
+    const currentSample: Sample = {
+        path: path,
+        additionalRendererOptions: {},
+        saveOutput: true
+    };
+
+    // Check optionMap for any CLI options and comparison options the test in this path should run with
+    if (optionMap[path]) {
+        const { cliOptions, language, comparisonArgs } = optionMap[path];
+        currentSample.additionalRendererOptions = cliOptions;
+        currentSample.language = language;
+        currentSample.comparisonArgs = comparisonArgs;
+    }
+
+    // If this is an input file, we should expect a corresponding output file to compare against
+    const inputFileMatch = path.match(inputFilePattern);
+    if (inputFileMatch) {
+        // Search for expected output file. Add to sample if found, throw error if one does not exist.
+        const outFilePath = inputFileMatch[1] + ".out." + inputFileMatch[2];
+        if (!fs.existsSync(outFilePath)) {
+            throw new Error(`Input file with name ${path} does not have a matching output file named ${outFilePath}`);
+        }
+        currentSample.outPath = outFilePath;
+    }
+    return currentSample;
 }
 
 export function samplesFromPaths(paths: string[]): Sample[] {
-    return paths.map(p => ({ path: p, additionalRendererOptions: {}, saveOutput: true }));
+    const samples: Sample[] = [];
+    for (const path of paths) {
+        // Output files will be processed with their corresponding input file and added to the same sample.
+        const outputFileMatch = path.match(outputFilePattern);
+        if (outputFileMatch) continue;
+
+        samples.push(sampleFromPath(path));
+    }
+    return samples;
 }
 
 export function samplesFromSources(