From 419fe0930e40803e1efe1b87ab4145ca31807ab8 Mon Sep 17 00:00:00 2001 From: Brian Burns Date: Tue, 2 Jul 2024 15:04:02 -0500 Subject: [PATCH] temp --- .github/workflows/tests.yaml | 5 +- packages/cli/lib/cli.ts | 19 +++-- packages/cli/lib/services/compile.service.ts | 79 +++++++++++++++---- .../lib/services/compile.service.unit.test.ts | 14 +++- packages/cli/lib/services/config.service.ts | 4 +- packages/cli/lib/sync.unit.cli-test.ts | 10 ++- packages/cli/lib/tests/helpers.ts | 3 + packages/nango-yaml/lib/helpers.ts | 8 +- packages/nango-yaml/lib/load.ts | 3 +- packages/node-client/lib/utils.unit.test.ts | 2 +- 10 files changed, 113 insertions(+), 34 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 8ed18ebe879..ceea62e9a0a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -13,11 +13,14 @@ concurrency: jobs: tests: - runs-on: ubuntu-latest + # runs-on: ubuntu-latest strategy: matrix: + os: [ubuntu-latest, windows-latest] node-version: [18.x, 20.x] + runs-on: ${{ matrix.os }} + steps: - uses: actions/checkout@v4 with: diff --git a/packages/cli/lib/cli.ts b/packages/cli/lib/cli.ts index 66d99a4be0c..159c200b960 100644 --- a/packages/cli/lib/cli.ts +++ b/packages/cli/lib/cli.ts @@ -54,9 +54,11 @@ export function generate({ fullPath, debug = false }: { fullPath: string; debug? }); const stripped = rendered.replace(/^\s+/, ''); - if (!fs.existsSync(`${fullPath}/${providerConfigKey}/${type}s/${name}.ts`)) { - fs.mkdirSync(`${fullPath}/${providerConfigKey}/${type}s`, { recursive: true }); - fs.writeFileSync(`${fullPath}/${providerConfigKey}/${type}s/${name}.ts`, stripped); + //.............. + // if (!fs.existsSync(path.join(fullPath, `${providerConfigKey}/${type}s/${name}.ts`)) { + if (!fs.existsSync(path.join(fullPath, `{providerConfigKey}/${type}s/${name}.ts`))) { + fs.mkdirSync(path.join(fullPath, `${providerConfigKey}/${type}s`), { recursive: true }); + fs.writeFileSync(path.join(fullPath, `${providerConfigKey}/${type}s/${name}.ts`), stripped); if (debug) { printDebug(`Created ${name}.ts file`); } @@ -81,7 +83,7 @@ export function generate({ fullPath, debug = false }: { fullPath: string; debug? process.exit(1); } - if (fs.existsSync(`${fullPath}/${name}.ts`) || fs.existsSync(`${fullPath}/${providerConfigKey}/${type}s/${name}.ts`)) { + if (fs.existsSync(path.join(fullPath, `${name}.ts`)) || fs.existsSync(path.join(fullPath, `${providerConfigKey}/${type}s/${name}.ts`))) { if (debug) { printDebug(`${name}.ts file already exists, so will not overwrite it.`); } @@ -119,10 +121,10 @@ export function generate({ fullPath, debug = false }: { fullPath: string; debug? const stripped = rendered.replace(/^\s+/, ''); if (layoutMode === 'root') { - fs.writeFileSync(`${fullPath}/${name}.ts`, stripped); + fs.writeFileSync(path.join(fullPath, `${name}.ts`), stripped); } else { - fs.mkdirSync(`${fullPath}/${providerConfigKey}/${type}s`, { recursive: true }); - fs.writeFileSync(`${fullPath}/${providerConfigKey}/${type}s/${name}.ts`, stripped); + fs.mkdirSync(path.join(fullPath, `${providerConfigKey}/${type}s`), { recursive: true }); + fs.writeFileSync(path.join(fullPath, `${providerConfigKey}/${type}s/${name}.ts`), stripped); } if (debug) { console.log(chalk.green(`Created ${name}.ts file`)); @@ -203,7 +205,8 @@ NANGO_DEPLOY_AUTO_CONFIRM=false # Default value` } export function tscWatch({ fullPath, debug = false }: { fullPath: string; debug?: boolean }) { - const tsconfig = fs.readFileSync(`${getNangoRootPath()}/tsconfig.dev.json`, 'utf8'); + // const tsconfig = fs.readFileSync(`${getNangoRootPath()}/tsconfig.dev.json`, 'utf8'); + const tsconfig = fs.readFileSync(path.join(getNangoRootPath()!, 'tsconfig.dev.json'), 'utf8'); const res = loadYamlAndGenerate({ fullPath, debug }); if (!res.success) { console.log(chalk.red(res.error?.message)); diff --git a/packages/cli/lib/services/compile.service.ts b/packages/cli/lib/services/compile.service.ts index ea24efc8824..2c749dd9811 100644 --- a/packages/cli/lib/services/compile.service.ts +++ b/packages/cli/lib/services/compile.service.ts @@ -26,7 +26,11 @@ export async function compileAllFiles({ providerConfigKey?: string; type?: ScriptFileType; }): Promise { - const tsconfig = fs.readFileSync(`${getNangoRootPath()}/tsconfig.dev.json`, 'utf8'); + console.log('compileAllFiles', fullPath); + + // const tsconfig = fs.readFileSync(`${getNangoRootPath()}/tsconfig.dev.json`, 'utf8'); + const tsconfig = fs.readFileSync(path.join(getNangoRootPath() || '', 'tsconfig.dev.json'), 'utf8'); + console.log('tsconfig', tsconfig); const distDir = path.join(fullPath, 'dist'); if (!fs.existsSync(distDir)) { @@ -43,15 +47,15 @@ export async function compileAllFiles({ const parsed = res.response!; const compilerOptions = (JSON.parse(tsconfig) as { compilerOptions: Record }).compilerOptions; + if (debug) { + printDebug(`Compiler options: ${JSON.stringify(compilerOptions, null, 2)}`); + } + const compiler = tsNode.create({ skipProject: true, // when installed locally we don't want ts-node to pick up the package tsconfig.json file compilerOptions }); - if (debug) { - printDebug(`Compiler options: ${JSON.stringify(compilerOptions, null, 2)}`); - } - let scriptDirectory: string | undefined; if (scriptName && providerConfigKey && type) { scriptDirectory = resolveTsFileLocation({ fullPath, scriptName, providerConfigKey, type }).replace(fullPath, ''); @@ -59,10 +63,12 @@ export async function compileAllFiles({ } const integrationFiles = listFilesToCompile({ scriptName, fullPath, scriptDirectory, parsed, debug }); - let success = true; + console.log('integrationFiles', integrationFiles); + let success = true; for (const file of integrationFiles) { try { + // console.log('calling compile', { fullPath, file, parsed }); const completed = await compile({ fullPath, file, parsed, compiler, debug }); if (!completed) { if (scriptName && file.inputPath.includes(scriptName)) { @@ -191,42 +197,50 @@ async function compile({ compiler: tsNode.Service; debug: boolean; }): Promise { - const providerConfiguration = getProviderConfigurationFromPath({ filePath: file.inputPath, parsed }); + console.log('compile', { fullPath, file }); + //. get _, eg foo.yaml? + const providerConfiguration = getProviderConfigurationFromPath({ filePath: file.inputPath, parsed }); if (!providerConfiguration) { return false; } + // get _, eg 'sync', ... const syncConfig = [...providerConfiguration.syncs, ...providerConfiguration.actions].find((sync) => sync.name === file.baseName); const type = syncConfig?.type || 'sync'; + //. compile any imported files? const success = compileImportedFile({ fullPath, filePath: file.inputPath, compiler, type, parsed }); - if (!success) { return false; } + //. compile ts file with tsNode ? + console.log('compile with tsNode...'); compiler.compile(fs.readFileSync(file.inputPath, 'utf8'), file.inputPath); + // get output path, eg 'email-gmail-... .ts' ? const dirname = path.dirname(file.outputPath); const extname = path.extname(file.outputPath); const basename = path.basename(file.outputPath, extname); - const fileNameWithExtension = `${basename}-${providerConfiguration.providerConfigKey}${extname}`; const outputPath = path.join(dirname, fileNameWithExtension); - if (debug) { printDebug(`Compiling ${file.inputPath} -> ${outputPath}`); } + //. compile with tsup ? but used tsNode above? + console.log('calling build...'); + // build may throw an error await build({ entryPoints: [file.inputPath], - tsconfig: path.join(getNangoRootPath()!, 'tsconfig.dev.json'), + tsconfig: path.join(getNangoRootPath() || '', 'tsconfig.dev.json'), skipNodeModulesBundle: true, silent: !debug, outDir: path.join(fullPath, 'dist'), outExtension: () => ({ js: '.js' }), onSuccess: async () => { + console.log('onSuccess - file:', file); if (fs.existsSync(file.outputPath)) { await fs.promises.rename(file.outputPath, outputPath); console.log(chalk.green(`Compiled "${file.inputPath}" successfully`)); @@ -250,7 +264,8 @@ export function getFileToCompile({ fullPath, filePath }: { fullPath: string; fil const baseName = path.basename(filePath, '.ts'); return { inputPath: filePath, - outputPath: path.join(fullPath, 'dist', `${baseName}.js`), + // outputPath: path.join(fullPath, 'dist', `${baseName}.js`), + outputPath: path.join(fullPath, `dist/${baseName}.js`), baseName }; } @@ -266,7 +281,8 @@ export function resolveTsFileLocation({ providerConfigKey: string; type: ScriptFileType; }) { - const nestedPath = path.resolve(fullPath, providerConfigKey, type, `${scriptName}.ts`); + // const nestedPath = path.resolve(fullPath, providerConfigKey, type, `${scriptName}.ts`); + const nestedPath = path.resolve(fullPath, `${providerConfigKey}/${type}/${scriptName}.ts`); if (fs.existsSync(nestedPath)) { return fs.realpathSync(path.resolve(nestedPath, '../')); } @@ -313,6 +329,7 @@ export function listFilesToCompile({ ...getMatchingFiles(fullPath, actionPath, 'ts'), ...getMatchingFiles(fullPath, postConnectionPath, 'ts') ]; + console.log('files', files); if (debug) { if (getMatchingFiles(fullPath, syncPath, 'ts').length > 0) { @@ -333,11 +350,41 @@ export function listFilesToCompile({ }); } -// get array of posix file paths that match the given path parts. +// get array of file paths that match the given path parts. // last part is treated as a file extension. // eg getMatchingFiles('foo', 'ts') -> glob.sync('foo/*.ts') function getMatchingFiles(...args: string[]): string[] { args.splice(-1, 1, `*.${args.slice(-1)[0]}`); - const pattern = args.join('/'); - return glob.sync(pattern, { posix: true }); + console.log('args', args); + // const pattern = args.join('/'); + // console.log('pattern', pattern); + // return glob.sync(pattern, { posix: true }); + const pattern = path.join(...args); + console.log('pattern', pattern); + + // windowsPathsNoEscape + // Use \\ as a path separator only, and never as an escape + // character. + // If set, all \\ characters are replaced with / in the + // pattern. Note that this makes it impossible to match + // against paths containing literal glob pattern characters, + // but allows matching with patterns constructed using + // path.join() and path.resolve() on Windows platforms, + // mimicking the (buggy!) behavior of Glob v7 and before on + // Windows. + + // absolute + // Set to false to always return relative paths. When + // this option is not set, absolute paths are returned for + // patterns that are absolute, and otherwise paths are + // returned that are relative to the cwd setting. + + // posix + // Return / delimited paths, even on Windows. + // On posix systems, this has no effect. But, on Windows, it + // means that paths will be / delimited, and absolute paths + // will be their full resolved UNC forms, eg instead of + // 'C:\\foo\\bar', it would return '//?/C:/foo/bar' + + return glob.sync(pattern, { windowsPathsNoEscape: true, absolute: false, posix: true }); } diff --git a/packages/cli/lib/services/compile.service.unit.test.ts b/packages/cli/lib/services/compile.service.unit.test.ts index 1b5f9242217..038aa9164e2 100644 --- a/packages/cli/lib/services/compile.service.unit.test.ts +++ b/packages/cli/lib/services/compile.service.unit.test.ts @@ -4,16 +4,26 @@ import { getFileToCompile, listFilesToCompile } from './compile.service'; import { fileURLToPath } from 'node:url'; import type { NangoYamlParsed } from '@nangohq/types'; +// eg "C:\Users\bburns\Workspace\test\various\src" const thisFolder = path.dirname(fileURLToPath(import.meta.url)); +// function join(...args) { +// args = args.map(arg => ) +// return path.join(...args) +// } + describe('listFiles', () => { it('should list files with glob', () => { const files = listFilesToCompile({ fullPath: thisFolder, parsed: { integrations: [], models: new Map(), yamlVersion: 'v2' } }); expect(files.length).toBeGreaterThan(1); expect(files[0]).toStrictEqual({ baseName: 'verification.service', - inputPath: `${thisFolder}/verification.service.ts`, - outputPath: `${thisFolder}/dist/verification.service.js` + // inputPath: `${thisFolder}/verification.service.ts`, + // outputPath: `${thisFolder}/dist/verification.service.js` + // inputPath: path.join(thisFolder, 'verification.service.ts'), + // outputPath: path.join(thisFolder, 'dist', 'verification.service.js') + inputPath: path.join(thisFolder, 'verification.service.ts'), + outputPath: path.join(thisFolder, 'dist/verification.service.js') }); }); diff --git a/packages/cli/lib/services/config.service.ts b/packages/cli/lib/services/config.service.ts index 833cc3be401..53978f2f95e 100644 --- a/packages/cli/lib/services/config.service.ts +++ b/packages/cli/lib/services/config.service.ts @@ -1,4 +1,5 @@ import fs from 'fs'; +import path from 'path'; import Ajv from 'ajv'; import addErrors from 'ajv-errors'; import chalk from 'chalk'; @@ -100,7 +101,8 @@ export function validateYaml(yaml: any): ValidationMessage[] { const version = determineVersion(yaml); const validationFile = version === 'v1' ? 'nango.yaml.schema.v1.json' : 'nango.yaml.schema.v2.json'; - const schema = fs.readFileSync(`${getNangoRootPath()}/lib/${validationFile}`, 'utf8'); + // const schema = fs.readFileSync(`${getNangoRootPath()}/lib/${validationFile}`, 'utf8'); + const schema = fs.readFileSync(path.join(getNangoRootPath() || '', `lib/${validationFile}`), 'utf8'); const validate = ajv.compile(JSON.parse(schema)); if (validate(yaml)) { diff --git a/packages/cli/lib/sync.unit.cli-test.ts b/packages/cli/lib/sync.unit.cli-test.ts index 387ecc434d6..962ee341a95 100644 --- a/packages/cli/lib/sync.unit.cli-test.ts +++ b/packages/cli/lib/sync.unit.cli-test.ts @@ -445,12 +445,16 @@ describe('generate function tests', () => { const dir = getTestDirectory('nested'); init({ absolutePath: dir }); - await copyDirectoryAndContents(`${fixturesPath}/nango-yaml/v2/nested-integrations/hubspot`, `${dir}/hubspot`); - await copyDirectoryAndContents(`${fixturesPath}/nango-yaml/v2/nested-integrations/github`, `${dir}/github`); - await fs.promises.copyFile(`${fixturesPath}/nango-yaml/v2/nested-integrations/nango.yaml`, `${dir}/nango.yaml`); + // await copyDirectoryAndContents(`${fixturesPath}/nango-yaml/v2/nested-integrations/hubspot`, `${dir}/hubspot`); + await copyDirectoryAndContents(path.join(fixturesPath, 'nango-yaml/v2/nested-integrations/hubspot'), path.join(dir, 'hubspot')); + // await copyDirectoryAndContents(`${fixturesPath}/nango-yaml/v2/nested-integrations/github`, `${dir}/github`); + await copyDirectoryAndContents(path.join(fixturesPath, 'nango-yaml/v2/nested-integrations/github'), path.join(dir, 'github')); + // await fs.promises.copyFile(`${fixturesPath}/nango-yaml/v2/nested-integrations/nango.yaml`, `${dir}/nango.yaml`); + await fs.promises.copyFile(path.join(fixturesPath, 'nango-yaml/v2/nested-integrations/nango.yaml'), path.join(dir, 'nango.yaml')); const success = await compileAllFiles({ fullPath: dir, debug: true }); + //. these should report any failed paths somehow, not just true!=false expect(fs.existsSync(path.join(dir, 'models.ts'))).toBe(true); expect(fs.existsSync(path.join(dir, 'hubspot/syncs/contacts.ts'))).toBe(true); expect(fs.existsSync(path.join(dir, 'dist/contacts-hubspot.js'))).toBe(true); diff --git a/packages/cli/lib/tests/helpers.ts b/packages/cli/lib/tests/helpers.ts index 6f216a47cdc..a712d8d61a3 100644 --- a/packages/cli/lib/tests/helpers.ts +++ b/packages/cli/lib/tests/helpers.ts @@ -2,6 +2,9 @@ import path from 'path'; import fs from 'fs/promises'; export const copyDirectoryAndContents = async (source: string, destination: string) => { + // source = path.join(...source.split('/')); + // destination = path.join(...destination.split('/')); + await fs.mkdir(destination, { recursive: true }); const files = await fs.readdir(source, { withFileTypes: true }); diff --git a/packages/nango-yaml/lib/helpers.ts b/packages/nango-yaml/lib/helpers.ts index 6bf6f2a3e2f..ac019a9147e 100644 --- a/packages/nango-yaml/lib/helpers.ts +++ b/packages/nango-yaml/lib/helpers.ts @@ -178,8 +178,14 @@ export function shouldQuote(name: string) { return !regQuote.test(name); } +//. add to nodejs +// function split(filepath: string) { +// return filepath.split(path.sep); +// } + export function getProviderConfigurationFromPath({ filePath, parsed }: { filePath: string; parsed: NangoYamlParsed }): NangoYamlParsedIntegration | null { - const pathSegments = filePath.split('/'); + // const pathSegments = filePath.split('/'); + const pathSegments = filePath.split(path.sep); const scriptType = pathSegments.length > 1 ? pathSegments[pathSegments.length - 2] : null; const isNested = scriptType === 'syncs' || scriptType === 'actions' || scriptType === 'post-connection-scripts'; diff --git a/packages/nango-yaml/lib/load.ts b/packages/nango-yaml/lib/load.ts index db27157bb08..df62f16f9ac 100644 --- a/packages/nango-yaml/lib/load.ts +++ b/packages/nango-yaml/lib/load.ts @@ -14,7 +14,8 @@ import { nangoConfigFile } from './constant.js'; * Load nango.yaml */ export function loadNangoYaml({ fullPath }: { fullPath: string }): NangoYamlParser { - const location = path.resolve(`${fullPath}/${nangoConfigFile}`); + // const location = path.resolve(`${fullPath}/${nangoConfigFile}`); + const location = path.resolve(fullPath, nangoConfigFile); try { const content = fs.readFileSync(location, 'utf8'); const raw = yaml.load(content) as NangoYaml; diff --git a/packages/node-client/lib/utils.unit.test.ts b/packages/node-client/lib/utils.unit.test.ts index 41f614f5ddf..0499c22ffae 100644 --- a/packages/node-client/lib/utils.unit.test.ts +++ b/packages/node-client/lib/utils.unit.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; import { getUserAgent } from './utils.js'; -const regex = 'nango-node-client/[0-9.]+ .[a-z]+/[0-9a-z.-]+; node.js/[0-9.]+.'; +const regex = 'nango-node-client/[0-9.]+ .[a-z0-9]+/[0-9a-z.-]+; node.js/[0-9.]+.'; describe('getUserAgent', () => { it('should output default user agent', () => { expect(getUserAgent()).toMatch(new RegExp(regex));