Earthly Local CI (#314)

* Earthly Local CI * change test timeout to jest config
konfig-dev · Oct 30, 2023 · 6e163cc · 6e163cc
1 parent f8942eb
commit 6e163cc
Show file tree

Hide file tree

Showing 24 changed files with 239 additions and 69 deletions.
diff --git a/generator/konfig-dash/.changeset/grumpy-cougars-jam.md b/generator/konfig-dash/.changeset/grumpy-cougars-jam.md
@@ -0,0 +1,7 @@
+---
+'konfig-kill-port': patch
+'konfig-cli': patch
+'konfig-lib': patch
+---
+
+Fix kill port in containers
diff --git a/generator/konfig-dash/.earthlyignore b/generator/konfig-dash/.earthlyignore
@@ -0,0 +1,2 @@
+/node_modules/
+/dist/
diff --git a/generator/konfig-dash/Earthfile b/generator/konfig-dash/Earthfile
@@ -0,0 +1,35 @@
+VERSION 0.7
+FROM node:16-slim
+WORKDIR /konfig-dash
+
+build-konfig-dash:
+  RUN apt-get update
+  RUN apt-get install -y openssl # not present in node:16-slim but required by prisma
+  # Copy everything we need to run `yarn` without copying the source code so that dependencies are cached
+  COPY package.json yarn.lock .yarnrc.yml redwood.toml .
+  COPY .yarn .yarn
+  COPY api/package.json api/package.json
+  COPY web/package.json web/package.json
+  COPY packages/konfig-cli/package.json packages/konfig-cli/package.json
+  COPY packages/konfig-kill-port/package.json packages/konfig-kill-port/package.json
+  COPY packages/konfig-lib/package.json packages/konfig-lib/package.json
+  COPY packages/konfig-openapi-spec/package.json packages/konfig-openapi-spec/package.json
+  COPY packages/konfig-postman-to-openapi/package.json packages/konfig-postman-to-openapi/package.json
+  COPY packages/konfig-release-it/package.json packages/konfig-release-it/package.json
+  COPY packages/konfig-spectral-ruleset/package.json packages/konfig-spectral-ruleset/package.json
+  COPY packages/konfig-swagger2openapi/package.json packages/konfig-swagger2openapi/package.json
+  COPY packages/konfig-typescript-sdk/package.json packages/konfig-typescript-sdk/package.json
+  COPY packages/konfig-zod-to-openapi/package.json packages/konfig-zod-to-openapi/package.json
+  RUN --secret NPM_TOKEN yarn
+  # Now that dependencies are installed, copy the source code and build
+  COPY api api
+  COPY web web
+  COPY packages packages
+  COPY bash-scripts/build.sh bash-scripts/build.sh
+  RUN --secret NPM_TOKEN yarn build
+
+run-konfig-api:
+  FROM +build-konfig-dash
+  ENTRYPOINT ["yarn", "rw", "deploy", "render", "api"]
+  EXPOSE 8911
+  SAVE IMAGE konfig-api:latest
diff --git a/...ator/konfig-dash/api/db/dataMigrations/20221121050642-create-dummy-generate-config-row.ts b/...ator/konfig-dash/api/db/dataMigrations/20221121050642-create-dummy-generate-config-row.ts
diff --git a/generator/konfig-dash/packages/konfig-cli/src/util/execute-test-command.ts b/generator/konfig-dash/packages/konfig-cli/src/util/execute-test-command.ts
@@ -74,9 +74,8 @@ export async function executeTestCommand({
 
     // if this process exits in any way, kill the mock server
     const handleTermination = async () => {
-      CliUx.ux.log('🛑 Killing mock server')
+      CliUx.ux.log('Exit hook - Ensuring mock server is not running.')
       await kill(mockServerPort)
-      process.exit()
     }
 
     process.on('exit', handleTermination)
@@ -197,7 +196,8 @@ export async function executeTestCommand({
   // If we made it here then we successfully ran all tests
   CliUx.ux.info('Successfully ran all tests!')
 
-  process.exit()
+  CliUx.ux.log('🛑 Killing mock server')
+  await kill(mockServerPort)
 }
 
 const defaultTestScripts: Record<

diff --git a/generator/konfig-dash/packages/konfig-cli/src/util/get-default-branch.ts b/generator/konfig-dash/packages/konfig-cli/src/util/get-default-branch.ts
@@ -3,9 +3,16 @@ import * as shell from 'shelljs'
  * Get usually "master" or "main"
  */
 export function getDefaultBranch({ cwd }: { cwd: string }) {
-  const result = shell.exec(
-    "git symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@'",
-    { silent: true, cwd }
-  )
-  return result.stdout.trim()
+  try {
+    const result = shell.exec(
+      "git symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@'",
+      { silent: true, cwd }
+    )
+    return result.stdout.trim()
+  } catch (e) {
+    console.error(
+      'Warning! Encountered error when trying to use git. If running in CI, this can be safely ignored.'
+    )
+  }
+  return 'main'
 }
diff --git a/generator/konfig-dash/packages/konfig-cli/src/util/is-submodule.ts b/generator/konfig-dash/packages/konfig-cli/src/util/is-submodule.ts
@@ -29,7 +29,9 @@ export async function isSubmodule({
     const isSameRemoteUrl = url.includes(gitConfigUrl)
     return !isSameRemoteUrl
   } catch (e) {
-    console.error(e)
+    console.error(
+      'Warning! Encountered error when trying to use git. If running in CI, this can be safely ignored.'
+    )
   }
   return false
 }
diff --git a/generator/konfig-dash/packages/konfig-kill-port/index.js b/generator/konfig-dash/packages/konfig-kill-port/index.js
@@ -47,9 +47,9 @@ module.exports = function (port, method = 'tcp') {
 			return Promise.reject(new Error('No process running on port'))
 
 		return sh(
-			`lsof -i ${method === 'udp' ? 'udp' : 'tcp'}:${port} | grep ${
+			`pid=$(lsof -i ${method === 'udp' ? 'udp' : 'tcp'}:${port} | grep ${
 				method === 'udp' ? 'UDP' : 'LISTEN'
-			} | awk '{print $2}' | xargs kill -9`
+			} | awk '{print $2}') && kill -9 $pid`
 		)
 	})
 }
diff --git a/generator/konfig-dash/packages/konfig-lib/generate-json-schema.ts b/generator/konfig-dash/packages/konfig-lib/generate-json-schema.ts
@@ -14,5 +14,7 @@ const docOutputPath = path.join(
   'static',
   'konfig-yaml.schema.json'
 )
-fs.writeFileSync(outputPath, JSON.stringify(jsonSchema, undefined, 2))
-fs.writeFileSync(docOutputPath, JSON.stringify(jsonSchema, undefined, 2))
+if (fs.existsSync(outputPath))
+  fs.writeFileSync(outputPath, JSON.stringify(jsonSchema, undefined, 2))
+if (fs.existsSync(docOutputPath))
+  fs.writeFileSync(docOutputPath, JSON.stringify(jsonSchema, undefined, 2))
diff --git a/generator/konfig-generator-api/Earthfile b/generator/konfig-generator-api/Earthfile
@@ -0,0 +1,16 @@
+VERSION 0.7
+FROM maven:3.8.6-jdk-11-slim
+WORKDIR /konfig-generator-api
+
+build-generator:
+  COPY src src
+  COPY pom.xml .
+  RUN mvn -f pom.xml clean package
+  SAVE ARTIFACT target/openapi-generator-api-1.0.0.jar
+
+run-generator:
+  FROM openjdk:11-jre-slim
+  COPY +build-generator/openapi-generator-api-1.0.0.jar /usr/local/lib/openapi-generator-api.jar
+  EXPOSE 8080
+  ENTRYPOINT ["java","-jar","/usr/local/lib/openapi-generator-api.jar", "--add-opens=java.base/java.lang=ALL-UNNAMED", "--add-opens=java.base/java.util=ALL-UNNAMED"]
+  SAVE IMAGE konfig-generator:latest
diff --git a/generator/konfig-integration-tests/.earthlyignore b/generator/konfig-integration-tests/.earthlyignore
@@ -0,0 +1 @@
+generate-id.txt
diff --git a/generator/konfig-integration-tests/.gitignore b/generator/konfig-integration-tests/.gitignore
@@ -191,4 +191,6 @@ dist
 # and uncomment the following lines
 # .pnp.*
 
-# End of https://www.toptal.com/developers/gitignore/api/node,yarn,macos
+# End of https://www.toptal.com/developers/gitignore/api/node,yarn,macos
+
+*.pyc
diff --git a/generator/konfig-integration-tests/Earthfile b/generator/konfig-integration-tests/Earthfile
@@ -0,0 +1,50 @@
+VERSION 0.7
+FROM ../konfig-dash+build-konfig-dash
+WORKDIR /konfig-integration-tests
+
+konfig-test-dependencies:
+  ### NODE comes from base image
+  ### PYTHON
+  ENV PYTHONDONTWRITEBYTECODE=1 # Don't write .pyc files, which are not needed in a container
+  ENV PIP_DEFAULT_TIMEOUT=100
+  ENV POETRY_VERSION=1.5.1
+
+  RUN apt-get update
+  # lsof required for killing mock server after test
+  RUN apt-get install -y lsof python3 python3-pip
+  RUN pip3 install poetry==$POETRY_VERSION
+  # Clean up python installations
+  RUN apt-get clean && rm -rf /var/lib/apt/lists/*
+
+integration-tests:
+  FROM +konfig-test-dependencies
+  COPY package.json yarn.lock .
+  RUN yarn
+  COPY tsconfig.json jest.config.ts util.ts .
+  COPY sdks sdks
+  COPY tests tests
+  ENTRYPOINT ["yarn", "test", "--no-cache"] # TODO: see if we can speed this up more
+  SAVE IMAGE konfig-integration-tests:latest
+
+test:
+  FROM earthly/dind:alpine
+  COPY compose/.env .
+  COPY compose/compose.yaml .
+  WITH DOCKER \
+        --compose compose.yaml \
+        --load konfig-python-formatter:latest=../konfig-python-formatter-server-blackd+run-python-formatter \
+        --load konfig-generator:latest=../konfig-generator-api+run-generator \
+        --load konfig-api:latest=../konfig-dash+run-konfig-api \
+        --load konfig-integration-tests:latest=+integration-tests
+    RUN --secret NPM_TOKEN docker run \
+        --network="konfig-network" \
+        -e "KONFIG_API_URL=http://konfig-api:8911" \
+        -e "NPM_TOKEN=${NPM_TOKEN}" \
+        konfig-integration-tests:latest
+  END
+
+build-all:
+  BUILD ../konfig-python-formatter-server-blackd+run-python-formatter
+  BUILD ../konfig-generator-api+run-generator
+  BUILD ../konfig-dash+run-konfig-api
+  BUILD +integration-tests
diff --git a/generator/konfig-integration-tests/README.md b/generator/konfig-integration-tests/README.md
@@ -1,11 +1,25 @@
 # konfig-integration-tests
 
- - test SDK directories under `sdks/`
- - each map to test under `tests/`
- - see `util.ts` for implementation of tests
+- test SDK directories under `sdks/`
+- each map to test under `tests/`
+- see `util.ts` for implementation of tests
 
-## To run rests
+## To run tests
 
 1. `cd` into `konfig-integration-tests`
 2. Run `yarn`
 3. Run `yarn test`
+
+## To run tests using earthly (local)
+
+1. `cd` into `konfig-integration-tests`
+2. Ensure `EARTHLY_SECRETS` env var is set
+   - EARTHLY_SECRETS: `NPM_TOKEN=xxx`
+   - This is used during the container building phase for konfig-dash
+3. Ensure `compose/.env` file is present and contains the following:
+   - `NPM_TOKEN`, `SESSION_SECRET`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`
+   - Each line should be formatted as key=value
+   - These are passed to docker compose and are used by while running containerized services
+4. Run `earthly -P +test`
+   - `-P` runs earthly in privileged mode (required for earthly image)
+   - `+test` is the name of the earthly target we are running
diff --git a/generator/konfig-integration-tests/compose/compose.yaml b/generator/konfig-integration-tests/compose/compose.yaml
@@ -0,0 +1,48 @@
+version: "3.8"
+services:
+  konfig-api:
+    image: konfig-api:latest
+    environment:
+      # prisma has trouble connecting to postgres from node:16 base image. Adding connect_timeout to url fixes it
+      # https://stackoverflow.com/questions/68476229/m1-related-prisma-cant-reach-database-server-at-database5432
+      DATABASE_URL: postgresql://konfig:konfig@konfig-db:5432/konfig?connect_timeout=300
+      SKIP_INSTALL_DEPS: "true"
+      NODE_VERSION: 16
+      GENERATOR_API_HOST_PORT: konfig-generator:8080
+      BLACKD_API_HOST_PORT: konfig-python-formatter:10000
+      NPM_TOKEN: ${NPM_TOKEN}
+      SESSION_SECRET: ${SESSION_SECRET}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+    ports:
+      - "8911:8911"
+    depends_on:
+      - konfig-db
+
+  konfig-python-formatter:
+    image: konfig-python-formatter:latest
+    ports:
+      - "10000:10000"
+
+  konfig-generator:
+    image: konfig-generator:latest
+    environment:
+      PORT: 8080
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+    ports:
+      - "8080:8080"
+
+  konfig-db:
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_USER: konfig
+      POSTGRES_PASSWORD: konfig
+    ports:
+      - "5432:5432"
+
+networks:
+  default:
+    name: konfig-network
+    attachable: true
diff --git a/generator/konfig-integration-tests/jest.config.ts b/generator/konfig-integration-tests/jest.config.ts
@@ -6,7 +6,7 @@
 import type { Config } from "jest";
 
 const config: Config = {
-  testTimeout: 120_000,
+  testTimeout: 900_000,
   // All imported modules in your tests should be mocked automatically
   // automock: false,
 

diff --git a/generator/konfig-integration-tests/tests/__snapshots__/leap-workflows-sdks.test.ts.snap b/generator/konfig-integration-tests/tests/__snapshots__/leap-workflows-sdks.test.ts.snap
@@ -7,7 +7,6 @@ The Leap Workflows API allows developers to run workflows, fetch workflow runs,
 
 
 [![PyPI](https://img.shields.io/badge/PyPI-v1.0.0-blue)](https://pypi.org/project/leap-workflows-python-sdk/1.0.0)
-[![GitHub last commit](https://img.shields.io/github/last-commit/leap-ai/workflows-sdks.svg)](https://github.com/leap-ai/workflows-sdks/commits)
 [![README.md](https://img.shields.io/badge/README-Click%20Here-green)](https://github.com/leap-ai/workflows-sdks/tree/main/sdks/python#readme)
 [![More Info](https://img.shields.io/badge/More%20Info-Click%20Here-orange)](https://tryleap.ai/)
 
@@ -207,7 +206,6 @@ exports[`leap-workflows-sdks 2`] = `
 The Leap Workflows API allows developers to run workflows, fetch workflow runs, and provide other utility functions related to workflow runs. Please use the X-Api-Key for authenticated requests.
 
 [![npm](https://img.shields.io/badge/npm-v1.0.0-blue)](https://www.npmjs.com/package/@leap-ai/workflows/v/1.0.0)
-[![GitHub last commit](https://img.shields.io/github/last-commit/leap-ai/workflows-sdks/tree/main/sdks/typescript.svg)](https://github.com/leap-ai/workflows-sdks/tree/main/sdks/typescript/commits)
 [![More Info](https://img.shields.io/badge/More%20Info-Click%20Here-orange)](https://tryleap.ai/)
 
 ## Table of Contents

diff --git a/...ator/konfig-integration-tests/tests/__snapshots__/python-dataclass-responses.test.ts.snap b/...ator/konfig-integration-tests/tests/__snapshots__/python-dataclass-responses.test.ts.snap
@@ -7,7 +7,6 @@ A simple API based on python dataclass responses.
 
 
 [![PyPI](https://img.shields.io/badge/PyPI-v1.0.0-blue)](https://pypi.org/project/python-dataclass-responses-python-sdk/1.0.0)
-[![GitHub last commit](https://img.shields.io/github/last-commit/konfig-dev/konfig.svg)](https://github.com/konfig-dev/konfig/commits)
 [![README.md](https://img.shields.io/badge/README-Click%20Here-green)](https://github.com/konfig-dev/konfig/tree/main/python#readme)
 [![More Info](https://img.shields.io/badge/More%20Info-Click%20Here-orange)](http://example.com/support)
 

diff --git a/...ator/konfig-integration-tests/tests/__snapshots__/python-typeddict-responses.test.ts.snap b/...ator/konfig-integration-tests/tests/__snapshots__/python-typeddict-responses.test.ts.snap
@@ -7,7 +7,6 @@ A simple API based on python typeddict responses.
 
 
 [![PyPI](https://img.shields.io/badge/PyPI-v1.0.0-blue)](https://pypi.org/project/python-typeddict-responses-python-sdk/1.0.0)
-[![GitHub last commit](https://img.shields.io/github/last-commit/konfig-dev/konfig.svg)](https://github.com/konfig-dev/konfig/commits)
 [![README.md](https://img.shields.io/badge/README-Click%20Here-green)](https://github.com/konfig-dev/konfig/tree/main/python#readme)
 [![More Info](https://img.shields.io/badge/More%20Info-Click%20Here-orange)](http://example.com/support)
 

diff --git a/generator/konfig-integration-tests/util.ts b/generator/konfig-integration-tests/util.ts
@@ -59,10 +59,19 @@ export async function e2e(mockServerPort: number) {
   ];
   for (const generator of generators) {
     expect(
-      fs.readFileSync(
-        path.join(sdkDir, generator.outputDirectory, "README.md"),
-        "utf-8"
+      normalizeDocumentation(
+        fs.readFileSync(
+          path.join(sdkDir, generator.outputDirectory, "README.md"),
+          "utf-8"
+        )
       )
     ).toMatchSnapshot();
   }
 }
+
+// Removes the [GitHub last commit] line from the README
+function normalizeDocumentation(readme: string) {
+  // matches [GitHub last commit] to the end of the line
+  const pattern = /\[!\[GitHub last commit\].*$\n?/gm;
+  return readme.replace(pattern, "");
+}