Merge pull request #81 from JarvusInnovations/develop

Release: v0.21.3

.github/workflows/release-validate.yml

@@ -9,15 +9,13 @@ env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
-  release-validate:
 
+  release-validate:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Grab PR Title
+    - name: Validate PR title
       run: |
-        set -e
-
         PR_TITLE=$(jq -r ".pull_request.title" $GITHUB_EVENT_PATH)
 
         # check title format and extract tag
@@ -30,7 +28,7 @@ jobs:
         fi
 
         # check that tag doesn't exist
-        if git ls-remote --exit-code origin "refs/tags/${RELEASE_TAG}"; then
+        if git ls-remote --exit-code "https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}" "refs/tags/${RELEASE_TAG}"; then
             echo "The PR title's version exists already"
             exit 1
         fi

backend/bin/cli.js

@@ -2,18 +2,22 @@
 
 
 // setup logger
-const logger = require('winston')
-module.exports = { logger }
+const logger = require('winston');
+const loggerConsole = new logger.transports.Console({
+  level: process.env.DEBUG ? 'debug' : 'info',
+  format: logger.format.combine(
+    logger.format.colorize(),
+    logger.format.prettyPrint(),
+    logger.format.splat(),
+    logger.format.simple(),
+  ),
 
-if (process.env.DEBUG) {
-  logger.level = 'debug'
-}
+  // all logger output to STDERR
+  stderrLevels: Object.keys(require('winston/lib/winston/config').cli.levels),
+});
+logger.add(loggerConsole);
 
-
-// all logger output to STDERR
-for (const level in logger.levels) {
-  logger.default.transports.console.stderrLevels[level] = true;
-}
+module.exports = { logger };
 
 
 // route command line
@@ -25,15 +29,33 @@ require('yargs')
     default: false,
     global: true,
   })
+  .option('q', {
+    alias: 'quiet',
+    type: 'boolean',
+    default: false,
+    global: true,
+  })
   .check(function (argv) {
     if (argv.debug) {
-      logger.level = 'debug'
+      loggerConsole.level = 'debug';
     } else if (argv.quiet) {
-      logger.level = 'error'
+      loggerConsole.level = 'error';
     }
 
     return true;
   })
-  .commandDir('../commands')
+  .commandDir('../commands', { exclude: /\.test\.js$/ })
   .demandCommand()
-  .argv
+  .showHelpOnFail(false, 'Specify --help for available options')
+  .fail((msg, err) => {
+    logger.error(msg || err.message);
+
+    if (err) {
+      logger.debug(err.stack);
+    }
+
+    process.exit(1);
+  })
+  .strict()
+  .help()
+  .argv;

backend/commands/edit.js

@@ -19,24 +19,30 @@ exports.builder = {
 exports.handler = async function edit({ recordPath, resumePath, encoding }) {
   const fs = require('fs');
   const { spawn } = require('child_process');
+  const path = require('path');
+  const tmp = require('tmp');
   const TOML = require('@iarna/toml');
   const Repository = require('../lib/Repository.js');
   const Sheet = require('../lib/Sheet.js')
   const repo = await Repository.getFromEnvironment({ working: true });
   const git = await repo.getGit();
 
   // open record
-  const recordToml = fs.readFileSync(resumePath || recordPath, encoding);
+  let recordToml = fs.readFileSync(resumePath || recordPath, encoding);
 
-  // get temp path
-  const tempFilePath = await new Promise((resolve, reject) => {
-    const mktemp = spawn('mktemp', ['-t', 'gitsheet.XXXXXX.toml']);
-
-    let stdout = '', stderr = '';
-    mktemp.stdout.on('data', chunk => stdout += chunk);
-    mktemp.stderr.on('data', chunk => stderr += chunk);
+  // try to parse and format
+  try {
+    const record = TOML.parse(recordToml);
+    recordToml = Sheet.stringifyRecord(record);
+  } catch (err) {
+    console.warn(`Failed to parse opened record:\n${err}`);
+  }
 
-    mktemp.on('close', code => code === 0 ? resolve(stdout.trim()) : reject(stderr.trim()));
+  // get temp path
+  const { name: tempFilePath } = tmp.fileSync({
+    prefix: path.basename(recordPath, '.toml'),
+    postfix: '.toml',
+    discardDescriptor: true,
   });
 
   // populate temp path
@@ -67,7 +73,7 @@ exports.handler = async function edit({ recordPath, resumePath, encoding }) {
   try {
     editedRecord = TOML.parse(editedToml);
   } catch (err) {
-    console.error(`Failed to parse record:\n${err}`);
+    console.error(`Failed to parse edited record:\n${err}`);
     console.error(`To resume editing, run: git sheet edit ${recordPath} ${tempFilePath}`);
     process.exit(1);
   }

backend/commands/normalize.js

@@ -54,9 +54,23 @@ exports.handler = async function query({
     }
 
     // loop through all records and re-upsert
-    for await (const record of sheet.query()) {
-      logger.info(`rewriting ${sheetName}/${record[Symbol.for('gitsheets-path')]}`);
-      await sheet.upsert(record);
+    try {
+      for await (const record of sheet.query()) {
+        const originalPath = record[Symbol.for('gitsheets-path')];
+        logger.info(`rewriting ${path.join(root, prefix, sheetName, originalPath)}.toml`);
+        const { path: normalizedPath } = await sheet.upsert(record);
+
+        if (normalizedPath !== originalPath) {
+          logger.warn(`^- moved to ${path.join(root, prefix, sheetName, normalizedPath)}.toml`);
+        }
+      }
+    } catch (err) {
+      if (err.constructor.name == 'TomlError') {
+        logger.error(`failed to parse ${path.join(root, prefix, err.file)}\n${err.message}`);
+        process.exit(1);
+      }
+
+      throw err;
     }
   }
 

backend/commands/read.js

@@ -3,7 +3,7 @@ exports.desc = 'Read a record, converting to desired format';
 exports.builder = {
   'record-path': {
     type: 'string',
-    describe: 'The path to a record file to edit',
+    describe: 'The path to a record file to read',
     demandOption: true,
   },
   encoding: {
@@ -24,12 +24,7 @@ exports.builder = {
 
 exports.handler = async function edit({ recordPath, encoding, format, headers }) {
   const fs = require('fs');
-  const { spawn } = require('child_process');
   const TOML = require('@iarna/toml');
-  const Repository = require('../lib/Repository.js');
-  const Sheet = require('../lib/Sheet.js')
-  const repo = await Repository.getFromEnvironment({ working: true });
-  const git = await repo.getGit();
 
   // open record
   const recordToml = fs.readFileSync(recordPath, encoding);
@@ -64,6 +59,6 @@ async function outputCsv(record, { headers = true, delimiter = ',' } = {}) {
 }
 
 async function outputToml(record) {
-  const TOML = require('@iarna/toml');
-  console.log(`${TOML.stringify(record)}`);
+  const Sheet = require('../lib/Sheet.js')
+  console.log(`${Sheet.stringifyRecord(record)}`);
 }

backend/lib/Sheet.js

@@ -118,9 +118,20 @@ class Sheet extends Configurable
   async readRecord (blob, path = null) {
     const cache = this.#recordCache.get(blob.hash);
 
-    const record = cache
-      ? v8.deserialize(cache)
-      : await blob.read().then(TOML.parse);
+    let record;
+
+    if (cache) {
+      record = v8.deserialize(cache);
+    } else {
+      const toml = await blob.read();
+
+      try {
+        record = TOML.parse(toml);
+      } catch (err) {
+        err.file = path;
+        throw err;
+      }
+    }
 
     // annotate with gitsheets keys
     record[RECORD_SHEET_KEY] = this.name;
@@ -153,14 +164,14 @@ class Sheet extends Configurable
     const pathTemplate = PathTemplate.fromString(pathTemplateString);
     const sheetDataTree = await this.dataTree.getSubtree(sheetRoot);
 
-    BLOBS: for await (const blob of pathTemplate.queryTree(sheetDataTree, query)) {
-      const record = await this.readRecord(blob);
+    BLOBS: for await (const { blob, path: blobPath } of pathTemplate.queryTree(sheetDataTree, query)) {
+      const record = await this.readRecord(blob, blobPath);
 
       if (!queryMatches(query, record)) {
         continue BLOBS;
       }
 
-      record[RECORD_PATH_KEY] = pathTemplate.render(record);
+      record[RECORD_PATH_KEY] = blobPath || pathTemplate.render(record);
 
       yield record;
     }

backend/lib/path/Template.js

@@ -50,7 +50,7 @@ class Template
     return recordPath.join('/');
   }
 
-  async* queryTree (tree, query, depth = 0) {
+  async* queryTree (tree, query, pathPrefix = '', depth = 0) {
     const numComponents = this.#components.length;
 
     if (!tree) {
@@ -68,7 +68,7 @@ class Template
           const child = await currentTree.getChild(`${nextName}.toml`);
 
           if (child) {
-            yield child;
+            yield { path: path.join(pathPrefix, nextName), blob: child };
           }
 
           // absolute match on a leaf, we're done with this query
@@ -97,8 +97,9 @@ class Template
               continue;
             }
 
-            attachmentsPrefix = `${childPath.substr(0, childPath.length - 5)}/`;
-            yield child;
+            const childName = childPath.substr(0, childPath.length - 5);
+            attachmentsPrefix = `${childName}/`;
+            yield { path: path.join(pathPrefix, childName), blob: child };
           }
 
           return;
@@ -117,14 +118,14 @@ class Template
       } else {
         // each tree in current tree could contain matching records
         const children = await currentTree.getChildren();
-        for (const childName in children) {
-          const child = children[childName];
+        for (const childPath in children) {
+          const child = children[childPath];
 
           if (!child.isTree) {
             continue;
           }
 
-          yield* this.queryTree(child, query, i+1);
+          yield* this.queryTree(child, query, path.join(pathPrefix, childPath), i+1);
         }
 
         return;