More control over in/out formats

generate.go

@@ -37,12 +37,14 @@ var generateOptions struct {
 	vmOptions
 
 	stdout bool
+	format string
 }
 
 func init() {
 	initAllVMFlags(generateCmd, &generateOptions.vmOptions)
 
 	generateCmd.PersistentFlags().BoolVar(&generateOptions.stdout, "stdout", false, "print values on stdout")
+	generateCmd.PersistentFlags().StringVar(&generateOptions.format, "format", "", "force all values to this format")
 
 	jk.AddCommand(generateCmd)
 }
@@ -51,6 +53,19 @@ func skipException(err error) bool {
 	return strings.Contains(err.Error(), "jk-internal-skip: ")
 }
 
+var errUnsupportedFormat = errors.New("--format accepts 'json' or 'yaml'")
+
+func setGenerateFormat(format string, vm *vm) {
+	switch format {
+	case "":
+		return
+	case "json", "yaml":
+		vm.parameters.SetString("jk.generate.format", format)
+	default:
+		log.Fatal(errUnsupportedFormat)
+	}
+}
+
 func generateArgs(cmd *cobra.Command, args []string) error {
 	if len(args) != 1 {
 		return errors.New("generate requires an input script")
@@ -70,6 +85,7 @@ func generate(cmd *cobra.Command, args []string) {
 
 	vm := newVM(&generateOptions.vmOptions, ".")
 	vm.parameters.SetBool("jk.generate.stdout", generateOptions.stdout)
+	setGenerateFormat(generateOptions.format, vm)
 
 	if err := vm.Run("@jkcfg/std/cmd/<generate>", fmt.Sprintf(string(std.Module("cmd/generate-module.js")), args[0])); err != nil {
 		if !skipException(err) {

std/cmd/generate-module.js

@@ -1,9 +1,10 @@
 import * as param from '@jkcfg/std/param';
-import { generate } from '@jkcfg/std/cmd/generate';
+import { generate, OutputFormat, maybeSetFormat } from '@jkcfg/std/cmd/generate';
 import generateDefinition from '%s';
 
-const inputParams = {
+let inputParams = {
   stdout: param.Boolean('jk.generate.stdout', false),
 };
+maybeSetFormat(inputParams, param.String('jk.generate.format', undefined));
 
 generate(generateDefinition, inputParams);

std/cmd/generate.ts

@@ -1,5 +1,6 @@
 import * as std from '../index';
 import { WriteOptions } from '../write';
+import { splitPath, formatFromPath } from '../read';
 import { ValidateFn } from './validate';
 import { normaliseResult, formatError } from '../validation';
 
@@ -16,15 +17,43 @@ export interface File {
   validate?: ValidateFn;
 }
 
+/*
+ * OutputFormat enumerates the values that a "forced format" argument
+ * can take.
+ */
+export enum OutputFormat {
+  JSON = "json",
+  YAML = "yaml",
+}
+
+const outputFormatToFormat = {
+  [OutputFormat.JSON]: std.Format.JSON,
+  [OutputFormat.YAML]: std.Format.YAML,
+};
+
 /*
  * GenerateParams types the optional arguments to generate.
  */
 export interface GenerateParams {
   stdout?: boolean;
+  format?: OutputFormat;
   overwrite?: std.Overwrite;
   writeFile?: (v: any, p: string, o?: WriteOptions) => void;
 }
 
+export function maybeSetFormat(inputParams: GenerateParams, format?: string) {
+  switch (format) {
+  case "json":
+    inputParams.format = OutputFormat.JSON;
+    break;
+  case "yaml":
+    inputParams.format = OutputFormat.YAML;
+    break;
+  default:
+    break;
+  }
+}
+
 const helpMsg = `
 To use generate, export a default value with the list of files to generate:
 
@@ -77,25 +106,6 @@ const nth = (n: number): string => {
   return n + (s[mod(v - 20, 10)] || s[v] || s[0]);
 };
 
-function extension(path: string): string {
-  return path.split('.').pop();
-}
-
-function formatFromPath(path: string): std.Format {
-  switch (extension(path)) {
-  case 'yaml':
-  case 'yml':
-    return std.Format.YAML;
-  case 'json':
-    return std.Format.JSON;
-  case 'hcl':
-  case 'tf':
-    return std.Format.HCL;
-  default:
-    return std.Format.JSON;
-  }
-}
-
 const isString = (s: any): boolean => typeof s === 'string' || s instanceof String;
 
 // represents a file spec that has its promise resolved, if necessary
@@ -159,6 +169,33 @@ function validateFormat(files: RealisedFile[], params: GenerateParams) {
   return { valid, showHelp: !valid };
 }
 
+function forceFormat(forced: OutputFormat, files: RealisedFile[]) {
+  for (const file of files) {
+    const { path, value, format } = file;
+    // this makes sure the forced file format is a stream if the
+    // original file is a stream.
+    switch (fileFormat(file)) {
+    case std.Format.YAMLStream:
+      if (forced === OutputFormat.JSON) {
+        file.format = std.Format.JSONStream;
+      }
+      break;
+    case std.Format.JSONStream:
+      if (forced == OutputFormat.YAML) {
+        file.format = std.Format.YAMLStream;
+      }
+      break;
+    default:
+      file.format = outputFormatToFormat[forced];
+      break;
+    }
+    const [p, ext] = splitPath(path);
+    if (ext !== '') {
+      file.path = [p, forced].join('.');
+    }
+  }
+}
+
 function assembleForStdout(values: RealisedFile[]) {
   // When writing to stdout, we need to
   //  1. make sure everything is a mutually compatible format (e.g.,
@@ -267,6 +304,10 @@ export function generate(definition: GenerateArg, params: GenerateParams) {
       throw new Error('jk-internal-skip: values failed validation');
     }
 
+    if (params.format !== undefined) {
+      forceFormat(params.format, files)
+    }
+
     if (stdout) {
       const { valid, stdoutFormat, stream } = assembleForStdout(files);
       if (!valid) {

std/cmd/transform.ts

@@ -1,20 +1,21 @@
-import { Format, Overwrite } from '../index';
+import { Format, Overwrite, read, stdin, print } from '../index';
 import * as host from '@jkcfg/std/internal/host'; // magic module
 import * as param from '../param';
-import { generate, File, GenerateParams } from './generate';
-import { valuesFormatFromPath } from '../read';
+import { generate, File, GenerateParams, maybeSetFormat } from './generate';
+import { valuesFormatFromPath, valuesFormatFromExtension } from '../read';
 
 type TransformFn = (value: any) => any | void;
 
-const inputParams: GenerateParams = {
+const generateParams: GenerateParams = {
   stdout: param.Boolean('jk.transform.stdout', false),
   overwrite: param.Boolean('jk.transform.overwrite', false) ? Overwrite.Write : Overwrite.Err,
 };
+maybeSetFormat(generateParams, param.String('jk.generate.format', undefined)); // NB jk.generate. param
 
 // If we're told to overwrite, we need to be able to write to the
 // files mentioned on the command-line; but not otherwise.
-if (inputParams.overwrite == Overwrite.Write) {
-  inputParams.writeFile = host.write;
+if (generateParams.overwrite == Overwrite.Write) {
+  generateParams.writeFile = host.write;
 }
 
 function transform(fn: TransformFn): void {
@@ -27,7 +28,17 @@ function transform(fn: TransformFn): void {
 
   const inputFiles = param.Object('jk.transform.input', {});
   const outputs = [];
+
   for (const path of Object.keys(inputFiles)) {
+    if (path === '') { // read from stdin
+      const stdinFormat = param.String('jk.transform.stdin.format', 'yaml');
+      const format = valuesFormatFromExtension(stdinFormat);
+      const path = `stdin.${stdinFormat}`;  // path is a stand-in
+      const value = read(stdin, { format }).then(v => v.map(transformOne));
+      outputs.push({ path, value, format });
+      continue;
+    }
+
     const format = valuesFormatFromPath(path);
     outputs.push(host.read(path, { format }).then((obj): File => {
       switch (format) {
@@ -43,7 +54,7 @@ function transform(fn: TransformFn): void {
       }
     }));
   }
-  generate(Promise.all(outputs), inputParams);
+  generate(Promise.all(outputs), generateParams);
 }
 
 export default transform;

std/read.ts

@@ -29,13 +29,47 @@ export interface ReadOptions {
   module?: string;
 }
 
-// valuesFormatFromPath guesses, for a path, the format that will
-// return all values in a file. In other words, it prefers YAML
-// streams and concatenated JSON. You may need to treat the read value
-// differently depending on the format you got here, since YAMLStream
-// and JSONStream will both result in an array of values.
-export function valuesFormatFromPath(path: string): Format {
-  const ext = path.split('.').pop();
+// splitPath returns [all-but-extension, extension] for a path. If a
+// path does not end with an extension, it will be an empty string.
+export function splitPath(path: string): [string, string] {
+  const parts = path.split('.');
+  const ext = parts.pop();
+  // When there's no extension, either there will be a single part (no
+  // dots anywhere), or a path separator in the last part (a dot
+  // somewhere before the last path segment)
+  if (parts.length === 0 || ext.includes('/')) {
+    return [ext, ''];
+  }
+  return [parts.join(''), ext];
+}
+
+function extension(path: string): string {
+  return splitPath(path)[1];
+}
+
+// formatFromPath guesses, for a file path, the format in which to
+// read the file. It will assume one value per file, so if you have
+// files that may have multiple values (e.g., YAML streams), it's
+// better to use `valuesFormatFromPath` and be prepared to get
+// multiple values.
+export function formatFromPath(path: string): Format {
+  switch (extension(path)) {
+  case 'yaml':
+  case 'yml':
+    return Format.YAML;
+  case 'json':
+    return Format.JSON;
+  case 'hcl':
+  case 'tf':
+    return Format.HCL;
+  default:
+    return Format.JSON;
+  }
+}
+
+// valuesFormatFromExtension returns the format implied by a
+// particular file extension.
+export function valuesFormatFromExtension(ext: string): Format {
   switch (ext) {
   case 'yaml':
   case 'yml':
@@ -47,6 +81,16 @@ export function valuesFormatFromPath(path: string): Format {
   }
 }
 
+// valuesFormatFromPath guesses, for a path, the format that will
+// return all values in a file. In other words, it prefers YAML
+// streams and concatenated JSON. You may need to treat the read value
+// differently depending on the format you got here, since YAMLStream
+// and JSONStream will both result in an array of values.
+export function valuesFormatFromPath(path: string): Format {
+  const ext = extension(path);
+  return valuesFormatFromExtension(ext);
+}
+
 type ReadPath = string | typeof stdin;
 
 // read requests the path and returns a promise that will be resolved

tests/generate-force-format.js

@@ -0,0 +1,19 @@
+import { Format } from '@jkcfg/std';
+
+function valueAndFormat(f) {
+  return {
+    format: f,
+    value: [
+      { item1: Format[f] },
+      { item2: Format[f] },
+      { item3: Format[f] },
+    ],
+  };
+}
+
+export default [
+  { path: 'jsonarray.json', ...valueAndFormat(Format.JSON) },
+  { path: 'jsonstream.json', ...valueAndFormat(Format.JSONStream) },
+  { path: 'yamlarray.yaml', ...valueAndFormat(Format.YAML) },
+  { path: 'yamlstream.yaml', ...valueAndFormat(Format.YAMLStream) },
+];

tests/test-generate-force-format-stdout.js.cmd

@@ -0,0 +1 @@
+jk generate --format=yaml --stdout ./generate-force-format.js

tests/test-generate-force-format-stdout.js.expected

@@ -0,0 +1,19 @@
+- item1: JSON
+- item2: JSON
+- item3: JSON
+---
+item1: JSONStream
+---
+item2: JSONStream
+---
+item3: JSONStream
+---
+- item1: YAML
+- item2: YAML
+- item3: YAML
+---
+item1: YAMLStream
+---
+item2: YAMLStream
+---
+item3: YAMLStream

tests/test-generate-force-format.expected/jsonarray.json

@@ -0,0 +1,11 @@
+[
+  {
+    "item1": "JSON"
+  },
+  {
+    "item2": "JSON"
+  },
+  {
+    "item3": "JSON"
+  }
+]

tests/test-generate-force-format.expected/jsonstream.json

@@ -0,0 +1,3 @@
+{"item1":"JSONStream"}
+{"item2":"JSONStream"}
+{"item3":"JSONStream"}

tests/test-generate-force-format.expected/yamlarray.json

@@ -0,0 +1,11 @@
+[
+  {
+    "item1": "YAML"
+  },
+  {
+    "item2": "YAML"
+  },
+  {
+    "item3": "YAML"
+  }
+]

tests/test-generate-force-format.expected/yamlstream.json

@@ -0,0 +1,3 @@
+{"item1":"YAMLStream"}
+{"item2":"YAMLStream"}
+{"item3":"YAMLStream"}

tests/test-generate-force-format.js.cmd

@@ -0,0 +1 @@
+jk generate --format=json -o %d ./generate-force-format.js

tests/test-transform-force-format.expected/transform-force-format/items.json

@@ -0,0 +1,2 @@
+{"item":1,"seen":true}
+[{"entry":"one"},{"entry":"two"},{"seen":true}]

tests/test-transform-force-format.js.cmd

@@ -0,0 +1 @@
+jk transform --format=json --overwrite -o %d ./transform-force-format.js transform-force-format/*.yaml

tests/test-transform-stdin.js.cmd

@@ -0,0 +1 @@
+jk transform -c "v => v + 1" --stdin-format=json - ./test-transform-files/*.json

tests/test-transform-stdin.js.expected

@@ -0,0 +1,3 @@
+7
+2
+3

tests/test-transform-stdin.js.in

@@ -0,0 +1 @@
+6