From c7333a3a65d297ef36eb9f08acb09716e3ef32c8 Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Thu, 16 Jan 2025 16:19:16 +0100 Subject: [PATCH 01/16] Add google watch disabling --- .github/workflows/pull-request.yml | 3 ++- package-lock.json | 4 ++-- src/cli/wikigdrive-server.ts | 13 ++++++++----- src/containers/server/SocketManager.ts | 16 +++++++++------- src/model/CliParams.ts | 1 + 5 files changed, 22 insertions(+), 15 deletions(-) diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 12de2678..5daea8ff 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -95,7 +95,8 @@ jobs: --service_account /service_account.json \ --share_email mie-docs-wikigdrive@wikigdrive.iam.gserviceaccount.com \ --workdir /data \ - server 3000 + server 3000 \ + --disable_google_watch remove: if: github.event.pull_request.head.ref != 'develop' && !contains( github.event.pull_request.labels.*.name, 'deploy-pr') diff --git a/package-lock.json b/package-lock.json index f2796a0b..3a78d6be 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@mieweb/wikigdrive", - "version": "2.12.1", + "version": "2.14.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@mieweb/wikigdrive", - "version": "2.12.1", + "version": "2.14.1", "license": "ISC", "workspaces": [ "apps/ui", diff --git a/src/cli/wikigdrive-server.ts b/src/cli/wikigdrive-server.ts index 2289f629..196df3de 100644 --- a/src/cli/wikigdrive-server.ts +++ b/src/cli/wikigdrive-server.ts @@ -91,10 +91,12 @@ export class MainService { process.exit(1); } - const changesContainer = new WatchChangesContainer({ name: 'watch_changes', share_email: this.params.share_email }); - await changesContainer.mount(await this.mainFileService); - await this.containerEngine.registerContainer(changesContainer); - await changesContainer.run(); + if (!this.params.disable_google_watch) { + const changesContainer = new WatchChangesContainer({ name: 'watch_changes', share_email: this.params.share_email }); + await changesContainer.mount(await this.mainFileService); + await this.containerEngine.registerContainer(changesContainer); + await changesContainer.run(); + } const port = parseInt(this.params.args[1]) || 3000; const serverContainer = new ServerContainer({ name: 'server', share_email: this.params.share_email }, port); @@ -159,7 +161,8 @@ async function main() { service_account: argv['service_account'] || null, share_email: argv['share_email'] || process.env.SHARE_EMAIL || null, - server_port: +argv['server_port'] + server_port: +argv['server_port'], + disable_google_watch: !!argv['disable_google_watch'] }; const mainService = new MainService(params); diff --git a/src/containers/server/SocketManager.ts b/src/containers/server/SocketManager.ts index db91afcf..c490878c 100644 --- a/src/containers/server/SocketManager.ts +++ b/src/containers/server/SocketManager.ts @@ -47,13 +47,15 @@ export class SocketManager { })); - const watchChangesContainer = this.engine.getContainer('watch_changes'); - const changes = await watchChangesContainer.getChanges(driveId); - const filteredChanges = await this.getFilteredChanges(driveId, changes); - ws.send(JSON.stringify({ - cmd: 'changes:changed', - payload: filteredChanges - })); + if (this.engine.hasContainer('watch_changes')) { + const watchChangesContainer = this.engine.getContainer('watch_changes'); + const changes = await watchChangesContainer.getChanges(driveId); + const filteredChanges = await this.getFilteredChanges(driveId, changes); + ws.send(JSON.stringify({ + cmd: 'changes:changed', + payload: filteredChanges + })); + } ws.on('close', () => { this.socketsMap[driveId].delete(ws); diff --git a/src/model/CliParams.ts b/src/model/CliParams.ts index 483d3230..7aa7375d 100644 --- a/src/model/CliParams.ts +++ b/src/model/CliParams.ts @@ -8,4 +8,5 @@ export interface CliParams { service_account?: string; server_port?: number; share_email?: string; + disable_google_watch: boolean; } From 2026b8f81f2e83f055007a64cd3fbd37e695f142 Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Fri, 18 Oct 2024 14:16:27 +0200 Subject: [PATCH 02/16] Add support for github actions --- apps/ui/src/components/JobsViewer.vue | 141 ++++---- apps/wgd-action-runner/Dockerfile | 9 +- apps/wgd-action-runner/steps/step_gh_action | 10 + package.json | 2 + .../action/ActionRunnerContainer.ts | 310 ++++++++---------- src/containers/action/DockerContainer.ts | 147 +++++++++ 6 files changed, 378 insertions(+), 241 deletions(-) create mode 100644 apps/wgd-action-runner/steps/step_gh_action create mode 100644 src/containers/action/DockerContainer.ts diff --git a/apps/ui/src/components/JobsViewer.vue b/apps/ui/src/components/JobsViewer.vue index e9f08214..dda9a828 100644 --- a/apps/ui/src/components/JobsViewer.vue +++ b/apps/ui/src/components/JobsViewer.vue @@ -2,70 +2,79 @@
-
-
-
- Sync Single - Sync All - Transform Single Markdown - Transform All Markdown - Render Preview -
-
+
-
-
- Jobs +
+
+
+ Sync Single + Sync All + Transform Single Markdown + Transform All Markdown +
-
-
-
- Last full sync - Last synced - {{ last_job.dateStr }} -  {{ last_job.durationStr }} -
-
- Last transform took -  {{ last_transform.durationStr }} -
+
+
Workflows
+
+ Render Preview +
+ {{actions}} +
+
+
+
+
+ Jobs
+
+
+
+ Last full sync + Last synced + {{ last_job.dateStr }} +  {{ last_job.durationStr }} +
+
+ Last transform took +  {{ last_transform.durationStr }} +
+
- - - - - - - - +
JobStartedFinished
+ + + + + + + - - - - - - - + + + + + + + - - - - - - - -
JobStartedFinished
{{ job.title }}{{ job.startedStr || job.state }} -  {{ job.progress.completed }} / {{ job.progress.total }} - Logs -
{{ job.title }}{{ job.startedStr || job.state }} +  {{ job.progress.completed }} / {{ job.progress.total }} + Logs +
{{ job.title }}{{ job.startedStr }} - {{ job.finishedStr }} - ({{ job.durationStr }}) - Logs -
+ + + {{ job.title }} + {{ job.startedStr }} + + {{ job.finishedStr }} + ({{ job.durationStr }}) + Logs + + + + +
-
@@ -73,6 +82,7 @@ import {UtilsMixin} from './UtilsMixin.ts'; import {UiMixin} from './UiMixin.ts'; import StatusToolBar from './StatusToolBar.vue'; +import yaml from 'js-yaml'; export default { name: 'JobsViewer', @@ -83,7 +93,15 @@ export default { type: String } }, + data() { + return { + actions: [] + }; + }, components: {StatusToolBar}, + created() { + this.fetchConfig(); + }, computed: { active_jobs_reverse() { return [].concat(this.active_jobs) @@ -114,6 +132,17 @@ export default { methods: { showLogs(job) { this.$router.push(`/drive/${this.driveId}#drive_logs:job-${job.id}`); + }, + async fetchConfig() { + const response = await this.authenticatedClient.fetchApi(`/api/config/${this.driveId}`); + const json = await response.json(); + if (json.config?.actions_yaml) { + const actions_yaml = json.config?.actions_yaml; + this.actions = []; + yaml.loadAll(actions_yaml, (actions) => { + this.actions = actions.map(action => action.on); + }); + } } } }; diff --git a/apps/wgd-action-runner/Dockerfile b/apps/wgd-action-runner/Dockerfile index 84737841..f4a52fa2 100644 --- a/apps/wgd-action-runner/Dockerfile +++ b/apps/wgd-action-runner/Dockerfile @@ -1,4 +1,4 @@ -FROM node:lts-buster +FROM node:22-bookworm-slim MAINTAINER Grzegorz Godlewski ADD site/ /site @@ -12,10 +12,11 @@ VOLUME ["/site/content"] VOLUME ["/site/public"] RUN apt-get update -RUN apt-get install -y git nodejs curl python3 python3-pip -RUN pip3 install s3cmd +RUN apt-get install -y git nodejs curl python3 python3-pip s3cmd yq RUN npm i -g postcss postcss-cli hugo-extended +RUN mkdir -p /gh_actions + RUN git clone https://github.com/budparr/gohugo-theme-ananke.git /themes/ananke RUN git init /site @@ -23,3 +24,5 @@ RUN git config --global --add safe.directory /site ADD steps/ /steps RUN chmod a+x /steps/step_* + +WORKDIR /site diff --git a/apps/wgd-action-runner/steps/step_gh_action b/apps/wgd-action-runner/steps/step_gh_action new file mode 100644 index 00000000..bf2b3708 --- /dev/null +++ b/apps/wgd-action-runner/steps/step_gh_action @@ -0,0 +1,10 @@ +#!/bin/sh + +ACTION_REPO=$1 +ACTION_VERSION=$2 + +git clone --depth 1 --branch $ACTION_VERSION https://github.com/$ACTION_REPO /gh_actions/"$ACTION_REPO"@"$ACTION_VERSION" + +RUNS_MAIN=$(yq -r '.runs.main' /gh_actions/"$ACTION_REPO"@"$ACTION_VERSION"/action.yml) + +node /gh_actions/"$ACTION_REPO"@"$ACTION_VERSION"/"$RUNS_MAIN" || cat /root/.npm/_logs/* diff --git a/package.json b/package.json index 435a708a..1a18efbe 100644 --- a/package.json +++ b/package.json @@ -114,6 +114,8 @@ "sharp-phash": "2.2.0", "slugify": "1.6.6", "stream": "^0.0.2", + "tar-fs": "3.0.6", + "tar-stream": "3.1.7", "ts-node": "10.9.2", "typescript": "5.3.3", "winston": "3.8.2", diff --git a/src/containers/action/ActionRunnerContainer.ts b/src/containers/action/ActionRunnerContainer.ts index c13ac975..fa9b1a09 100644 --- a/src/containers/action/ActionRunnerContainer.ts +++ b/src/containers/action/ActionRunnerContainer.ts @@ -2,7 +2,6 @@ import * as path from 'node:path'; import process from 'node:process'; import winston from 'winston'; -import Docker from 'dockerode'; import yaml from 'js-yaml'; import {Container, ContainerEngine} from '../../ContainerEngine.ts'; @@ -17,6 +16,7 @@ const __filename = import.meta.filename; export interface ActionStep { name?: string; uses: string; + with?: {[key: string]: string}; env?: {[key: string]: string}; } @@ -80,6 +80,15 @@ export async function convertActionYaml(actionYaml: string): Promise { + async run(driveId: FileId) { if (!process.env.ACTION_IMAGE) { this.logger.error('No env.ACTION_IMAGE'); - return -1; + this.isErr = true; + return; } if (!process.env.VOLUME_DATA) { this.logger.error('No env.VOLUME_DATA'); - return -1; + this.isErr = true; + return; } if (!process.env.VOLUME_PREVIEW) { this.logger.error('No env.VOLUME_PREVIEW'); - return -1; + this.isErr = true; + return; } if (!process.env.DOMAIN) { this.logger.error('No env.DOMAIN'); - return -1; - } - - let code = 0; - const themeUrl = config?.hugo_theme?.url || ''; - const themeSubPath = config?.hugo_theme?.path || ''; - - const driveIdTransform: string = path.basename(generatedFileService.getRealPath()); - - const contentDir = config.transform_subdir ? - `/${driveIdTransform}${ !config.transform_subdir.startsWith('/') ? '/' : '' }${config.transform_subdir}` : - `/${driveIdTransform}`; - - // const docker = new Docker({socketPath: '/var/run/docker.sock'}); - // HACK: https://github.com/apocas/dockerode/issues/747 - // https://github.com/denoland/deno/issues/17910 - const docker = new Docker({protocol: 'http', host: 'localhost', port: 5000}); - - const themeId = config?.hugo_theme?.id || ''; - const configToml = config?.config_toml || '#relativeURLs = true\n' + - 'languageCode = "en-us"\n' + - 'title = "My New Hugo Site"\n'; - - await this.filesService.mkdir('tmp_dir'); - - if (themeId) { - const configTomlPrefix = `theme="${themeId}"\n`; - await this.filesService.writeFile('tmp_dir/config.toml', configTomlPrefix + configToml); - } else { - await this.filesService.writeFile('tmp_dir/config.toml', configToml); - } - - const committer = { - name: this.params.user_name || 'WikiGDrive', - email: this.params.user_email || 'wikigdrive@wikigdrive.com' - }; - - const additionalEnv = this.payloadToEnv(); - - try { - const writable = new BufferWritable(); - - let result; - - await this.generatedFileService.remove('resources'); - - if (themeId) { - const env = ['render_hugo', 'exec', 'commit_branch'].includes(step.uses) ? Object.assign({ - CONFIG_TOML: '/site/tmp_dir/config.toml', - BASE_URL: `${process.env.DOMAIN}/preview/${driveId}/${themeId}/`, - THEME_ID: themeId, - THEME_SUBPATH: themeSubPath, - THEME_URL: themeUrl, - GIT_AUTHOR_NAME: committer.name, - GIT_AUTHOR_EMAIL: committer.email, - GIT_COMMITTER_NAME: committer.name, - GIT_COMMITTER_EMAIL: committer.email - }, step.env, additionalEnv) : Object.assign({}, step.env, additionalEnv); - - this.logger.info(`DockerAPI:\ndocker run \\ - --user=${process.getuid()} \\ - -v "${process.env.VOLUME_DATA}/${driveId}_transform:/repo" \\ - -v "${process.env.VOLUME_DATA}${contentDir}:/site/content" \\ - -v "${process.env.VOLUME_PREVIEW}/${driveId}/${themeId}:/site/public" \\ - -v "${process.env.VOLUME_DATA}/${driveId}/tmp_dir:/site/tmp_dir" \\ - --mount type=tmpfs,destination=/site/resources" \ - ${Object.keys(env).map(key => `--env ${key}="${env[key]}"`).join(' ')} \\ - ${process.env.ACTION_IMAGE} /steps/step_${step.uses} - `); - - result = await docker.run(process.env.ACTION_IMAGE, [`/steps/step_${step.uses}`], writable, { - HostConfig: { - Binds: [ // Unlike Mounts those are created if not existing in the host - `${process.env.VOLUME_PREVIEW}/${driveId}/${themeId}:/site/public:rw`, - `${process.env.VOLUME_DATA}/${driveId}_transform:/repo:ro`, - `${process.env.VOLUME_DATA}${contentDir}:/site/content:ro`, - `${process.env.VOLUME_DATA}/${driveId}/tmp_dir:/site/tmp_dir:rw`, - ], - Mounts: [ - { - Source: '', - Target: '/site/resources', - Type: 'tmpfs', - ReadOnly: false, - TmpfsOptions: { - SizeBytes: undefined, - Mode: 0o777 - } - } - ] - }, - Env: Object.keys(env).map(key => `${key}=${env[key]}`), - User: String(process.getuid()) - }); - } else { - const env = ['render_hugo', 'exec', 'commit_branch'].includes(step.uses) ? Object.assign({ - CONFIG_TOML: '/site/tmp_dir/config.toml', - BASE_URL: `${process.env.DOMAIN}/preview/${driveId}/_manual/`, - GIT_AUTHOR_NAME: committer.name, - GIT_AUTHOR_EMAIL: committer.email, - GIT_COMMITTER_NAME: committer.name, - GIT_COMMITTER_EMAIL: committer.email - }, step.env, additionalEnv) : Object.assign({}, step.env, additionalEnv); - - this.logger.info(`DockerAPI:\ndocker run \\ - --user=${process.getuid()} \\ - -v "${process.env.VOLUME_DATA}/${driveId}_transform:/repo" \\ - -v "${process.env.VOLUME_DATA}/${driveIdTransform}:/site" \\ - -v "${process.env.VOLUME_DATA}${contentDir}:/site/content" \\ - -v "${process.env.VOLUME_PREVIEW}/${driveId}/_manual:/site/public" \\ - -v "${process.env.VOLUME_DATA}/${driveId}/tmp_dir:/site/tmp_dir" \\ - --mount "type=tmpfs,destination=/site/resources" \\ - ${Object.keys(env).map(key => `--env ${key}="${env[key]}"`).join(' ')} \\ - ${process.env.ACTION_IMAGE} /steps/step_${step.uses} - `); - - result = await docker.run(process.env.ACTION_IMAGE, [`/steps/step_${step.uses}`], writable, { - HostConfig: { - Binds: [ // Unlike Mounts those are created if not existing in the host - `${process.env.VOLUME_PREVIEW}/${driveId}/_manual:/site/public:rw`, - `${process.env.VOLUME_DATA}/${driveId}_transform:/repo:ro`, - `${process.env.VOLUME_DATA}/${driveIdTransform}:/site:rw`, - `${process.env.VOLUME_DATA}${contentDir}:/site/content:rw`, - `${process.env.VOLUME_DATA}/${driveId}/tmp_dir:/site/tmp_dir:rw`, - ], - Mounts: [ - { - Source: '', - Target: '/site/resources', - Type: 'tmpfs', - ReadOnly: false, - TmpfsOptions: { - SizeBytes: undefined, - Mode: 0o777 - } - } - ] - }, - Env: Object.keys(env).map(key => `${key}=${env[key]}`), - User: String(process.getuid()) - }); - } - - if (result?.length > 0 && result[0].StatusCode > 0) { - this.logger.error(writable.getBuffer().toString()); - code = result[0].StatusCode; - } else { - this.logger.info(writable.getBuffer().toString()); - } - } catch (err) { - console.error('eeee', err); - code = err.statusCode || 1; - this.logger.error(err.stack ? err.stack : err.message); + this.isErr = true; + return; } - return code; - } - async run(driveId: FileId) { const config = this.userConfigService.config; const gitScanner = new GitScanner(this.logger, this.generatedFileService.getRealPath(), 'wikigdrive@wikigdrive.com'); @@ -298,42 +156,130 @@ export class ActionRunnerContainer extends Container { throw new Error('No action steps'); } - for (const step of actionDef.steps) { - this.logger.info('Step: ' + (step.name || step.uses)); + const driveIdTransform: string = path.basename(generatedFileService.getRealPath()); - if (!step.env) { - step.env = {}; - } - step.env['OWNER_REPO'] = ownerRepo; - step.env['PAYLOAD'] = this.params.payload; + const committer = { + name: this.params.user_name || 'WikiGDrive', + email: this.params.user_email || 'wikigdrive@wikigdrive.com' + }; + + const additionalEnv = this.payloadToEnv(); - let lastCode = 0; - switch (step.uses) { - case 'push_branch': + const writable = new BufferWritable(); + + // await this.generatedFileService.remove('resources'); + +/* const env = ['render_hugo', 'exec', 'commit_branch'].includes(step.uses) ? Object.assign({ + CONFIG_TOML: '/site/tmp_dir/config.toml', + BASE_URL: `${process.env.DOMAIN}/preview/${driveId}/_manual/`, + GIT_AUTHOR_NAME: committer.name, + GIT_AUTHOR_EMAIL: committer.email, + GIT_COMMITTER_NAME: committer.name, + GIT_COMMITTER_EMAIL: committer.email + }, step.env, additionalEnv) : Object.assign({}, step.env, additionalEnv);*/ + + const env = Object.assign({ + CONFIG_TOML: '/site/tmp_dir/config.toml', + BASE_URL: `${process.env.DOMAIN}/preview/${driveId}/_manual/`, + GIT_AUTHOR_NAME: committer.name, + GIT_AUTHOR_EMAIL: committer.email, + GIT_COMMITTER_NAME: committer.name, + GIT_COMMITTER_EMAIL: committer.email + }, additionalEnv); + + //--user=$(id -u):$(getent group docker | cut -d: -f3) + this.logger.info(`DockerAPI:\ndocker start \\ + --user=${process.getuid()}:${process.getegid()} \\ + // -v "${process.env.VOLUME_DATA}/${driveId}_transform:/repo:ro" \\ + // -v "${process.env.VOLUME_DATA}/${driveIdTransform}:/site:rw" \\ + // --mount "type=tmpfs,destination=/site/resources" \\ + ${Object.keys(env).map(key => `--env ${key}="${env[key]}"`).join(' ')} \\ + ${process.env.ACTION_IMAGE} + `); + + const container = new DockerContainer(process.env.ACTION_IMAGE); + + try { + await container.create(env, writable); + await container.start(); + this.logger.info('container created: ' + container.id); + + this.logger.info('docker cp . /site'); + await container.copy(generatedFileService.getRealPath(), '/site'); + + // Convert to step: + const configToml = config?.config_toml || '#relativeURLs = true\n' + + 'languageCode = "en-us"\n' + + 'title = "My New Hugo Site"\n'; + this.logger.info('docker write /site/tmp_dir/config.toml'); + await container.putFile(new TextEncoder().encode(configToml), '/site/tmp_dir/config.toml'); + // + + for (const step of actionDef.steps) { + this.logger.info('Step: ' + (step.name || step.uses)); + + if (!step.env) { + step.env = {}; + } + step.env['OWNER_REPO'] = ownerRepo; + step.env['PAYLOAD'] = this.params.payload; + + let lastCode = 0; + switch (step.uses) { + case 'push_branch': { const additionalEnv = this.payloadToEnv(); await gitScanner.pushBranch(`wgd/${additionalEnv['BRANCH']}`, { privateKeyFile: await this.userConfigService.getDeployPrivateKeyPath() }, `wgd/${additionalEnv['BRANCH']}`); } - break; - case 'auto_commit': + break; + case 'auto_commit': { await gitScanner.autoCommit(); } + break; + default: + this.logger.info(`docker exec ${container.id} /steps/step_${step.uses}`); + try { + if (step.uses.indexOf('/') > -1 && step.uses.indexOf('@') > -1) { + const [action_repo, action_version] = step.uses.split('@'); + lastCode = await container.exec(`/steps/step_gh_action ${action_repo} ${action_version}`, Object.assign(step.env, withToEnv(step.with))); + } else { + lastCode = await container.exec(`/steps/step_${step.uses}`, Object.assign(step.env, withToEnv(step.with))); + } + if (lastCode > 0) { + this.logger.error(writable.getBuffer().toString()); + } else { + this.logger.info(writable.getBuffer().toString()); + } + } catch (err) { + this.logger.error(err.stack ? err.stack : err.message); + lastCode = 1; + } + break; + } + if (0 !== lastCode) { + this.isErr = true; break; - default: - lastCode = await this.runDocker(driveId, generatedFileService, step, config); - break; - } - if (0 !== lastCode) { - this.isErr = true; - break; + } } + + // Convert to step + const previewOutput = `${process.env.VOLUME_PREVIEW}/${driveId}/_manual`; + this.logger.info('docker export /site/public ' + previewOutput); + await container.export('/site/public', previewOutput); + // + + this.logger.info('Action completed'); + + } catch (err) { + this.logger.error(err.stack ? err.stack : err.message); + this.isErr = true; + } finally { + await container.stop(); } } - - // fs.unlinkSync(`${tempDir}/config.toml`); } // eslint-disable-next-line @typescript-eslint/no-empty-function diff --git a/src/containers/action/DockerContainer.ts b/src/containers/action/DockerContainer.ts new file mode 100644 index 00000000..486e7f23 --- /dev/null +++ b/src/containers/action/DockerContainer.ts @@ -0,0 +1,147 @@ +import Docker from 'dockerode'; +import path from 'path'; +import tarFs from 'tar-fs'; +import tarStream from 'tar-stream'; +import {PassThrough, Writable} from 'stream'; + +export class DockerContainer { + public id: string; + private docker: Docker; + private container: Docker.Container; + private writable: Writable; + constructor(private image: string) { + this.docker = new Docker({socketPath: '/var/run/docker.sock'}); + } + + async create(env: { [p: string]: string }, writable: Writable) { + this.writable = writable; + this.container = await this.docker.createContainer({ + Image: process.env.ACTION_IMAGE, + AttachStdin: false, + AttachStdout: true, + AttachStderr: true, + Tty: true, + OpenStdin: false, + StdinOnce: false, + HostConfig: { + // Binds: [ // Unlike Mounts those are created if not existing in the host + // `${process.env.VOLUME_DATA}/${driveId}_transform:/repo:ro`, + // `${process.env.VOLUME_DATA}/${driveIdTransform}:/site:rw`, + // `${process.env.VOLUME_DATA}${contentDir}:/site/content:rw`, + // ], + Mounts: [ + { + Source: '', + Target: '/site/resources', + Type: 'tmpfs', + ReadOnly: false, + TmpfsOptions: { + SizeBytes: undefined, + Mode: 0o777 + } + } + ] + }, + Env: Object.keys(env).map(key => `${key}=${env[key]}`), + User: String(process.getuid())+ ':' + String(process.getegid()) + }); + this.id = this.container.id; + } + + async start() { + return this.container.start(); + } + + async stop() { + return this.container.stop(); + } + + async copy(realPath: string, remotePath: string, ignoreGit = false) { + const archive = tarFs.pack(realPath, { + ignore (name) { + if (ignoreGit && name.startsWith(path.join(realPath, '.git'))) { + return true; + } + if (name.startsWith(path.join(realPath, '.private'))) { + return true; + } + return false; + }, + }); + + await this.container.putArchive(archive, { + path: remotePath + }); + } + + async putFile(configToml: Uint8Array, remotePath: string) { + const archive = tarStream.pack(); + archive.entry({ name: remotePath }, configToml); + archive.finalize(); + await this.container.putArchive(archive, { + path: '/' + }); + } + + async export(remotePath: string, outputDir: string) { + const archive = await this.container.getArchive({ + path: remotePath + }); + + await new Promise((resolve, reject) => { + try { + const stream = archive.pipe(tarFs.extract(outputDir, { + map (header) { + const parts = header.name.split('/'); + parts.shift(); + header.name = parts.join('/'); + return header; + } + })); + + stream.on('finish', () => { + resolve(); + }); + stream.on('error', (err: unknown) => { + reject(err); + }); + } catch (err) { + reject(err); + } + }); + } + + async exec(command: string, env: { [p: string]: string}) { + const cancelTimeout = new AbortController(); + + const exec = await this.container.exec({ + Cmd: command.split(' '), + AttachStdin: false, + AttachStdout: true, + AttachStderr: true, + Tty: true, + Env: Object.keys(env).map(key => `${key}=${env[key]}`), + //WorkingDir + abortSignal: cancelTimeout.signal, + }); + + const stream = await exec.start({}); + + const stdout = new PassThrough(); + const stderr = new PassThrough(); + this.container.modem.demuxStream(stream, stdout, stderr); + + stdout.on('data', (chunk: Buffer) => { + this.writable.write(chunk); + }); + + await new Promise(resolve => stream.on('end', () => { + resolve(0); + })); + + const inspectInfo = await exec.inspect(); + + return inspectInfo.ExitCode; + } + +} From 0aad5af6c23876fe1108c5a38278ebf26999ae61 Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Fri, 18 Oct 2024 14:28:09 +0200 Subject: [PATCH 03/16] Fix pull request create --- .github/workflows/feat-deploy.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/feat-deploy.yml b/.github/workflows/feat-deploy.yml index 8ebba47a..f7159483 100644 --- a/.github/workflows/feat-deploy.yml +++ b/.github/workflows/feat-deploy.yml @@ -36,10 +36,6 @@ jobs: steps: - name: Create pull request - id: open-pr - uses: repo-sync/pull-request@v2 - with: - destination_branch: "master" - pr_title: "${BRANCH_NAME}" - pr_template: ".github/pull_request.md" - pr_draft: true + run: | + gh_pr_up() { gh pr create $* || gh pr edit $* } + gh_pr_up --draft --branch master --template ".github/pull_request.md" --title "${BRANCH_NAME}" --body "Description" From fbc189c787ef5afe1dfaa6b95c53e22caba5fd36 Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Fri, 18 Oct 2024 14:42:54 +0200 Subject: [PATCH 04/16] Update workflows --- .github/workflows/feat-deploy.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/feat-deploy.yml b/.github/workflows/feat-deploy.yml index f7159483..57f2451d 100644 --- a/.github/workflows/feat-deploy.yml +++ b/.github/workflows/feat-deploy.yml @@ -37,5 +37,7 @@ jobs: steps: - name: Create pull request run: | - gh_pr_up() { gh pr create $* || gh pr edit $* } + gh_pr_up() { + gh pr create $* || gh pr edit $* + } gh_pr_up --draft --branch master --template ".github/pull_request.md" --title "${BRANCH_NAME}" --body "Description" From 1b99427fe3082751d8bc11dff99af31ef3a80dd3 Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Fri, 18 Oct 2024 14:45:38 +0200 Subject: [PATCH 05/16] Update workflows --- .github/workflows/feat-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/feat-deploy.yml b/.github/workflows/feat-deploy.yml index 57f2451d..c40d237e 100644 --- a/.github/workflows/feat-deploy.yml +++ b/.github/workflows/feat-deploy.yml @@ -40,4 +40,4 @@ jobs: gh_pr_up() { gh pr create $* || gh pr edit $* } - gh_pr_up --draft --branch master --template ".github/pull_request.md" --title "${BRANCH_NAME}" --body "Description" + gh_pr_up --draft --base master --template ".github/pull_request.md" --title "${BRANCH_NAME}" --body "Description" From f9b6172c92fc144567b8a5762e5a284517212701 Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Fri, 18 Oct 2024 14:49:26 +0200 Subject: [PATCH 06/16] Update workflows --- .github/workflows/feat-deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/feat-deploy.yml b/.github/workflows/feat-deploy.yml index c40d237e..20a3fb36 100644 --- a/.github/workflows/feat-deploy.yml +++ b/.github/workflows/feat-deploy.yml @@ -38,6 +38,6 @@ jobs: - name: Create pull request run: | gh_pr_up() { - gh pr create $* || gh pr edit $* + gh pr create --draft $* --template ".github/pull_request.md" || gh pr edit $* } - gh_pr_up --draft --base master --template ".github/pull_request.md" --title "${BRANCH_NAME}" --body "Description" + gh_pr_up --base master --title "${BRANCH_NAME}" --body "Description" From d47cd680785414f2976bc263d4132a81bf832566 Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Fri, 18 Oct 2024 14:52:51 +0200 Subject: [PATCH 07/16] Update workflows --- .github/workflows/feat-deploy.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/feat-deploy.yml b/.github/workflows/feat-deploy.yml index 20a3fb36..7ab0c2ea 100644 --- a/.github/workflows/feat-deploy.yml +++ b/.github/workflows/feat-deploy.yml @@ -33,11 +33,14 @@ jobs: build: needs: test runs-on: ubuntu-latest - + env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + GH_TOKEN: ${{ github.token }} steps: + - uses: actions/checkout@v4 - name: Create pull request run: | gh_pr_up() { - gh pr create --draft $* --template ".github/pull_request.md" || gh pr edit $* + gh pr create --draft $* || gh pr edit $* } gh_pr_up --base master --title "${BRANCH_NAME}" --body "Description" From 1079a27f8ba7d63db7b49cef924bf75ae26a5efd Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Fri, 18 Oct 2024 15:11:42 +0200 Subject: [PATCH 08/16] Add support for github actions --- .github/workflows/feat-deploy.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/feat-deploy.yml b/.github/workflows/feat-deploy.yml index 7ab0c2ea..fac6cdb9 100644 --- a/.github/workflows/feat-deploy.yml +++ b/.github/workflows/feat-deploy.yml @@ -36,11 +36,12 @@ jobs: env: BRANCH_NAME: ${{ github.head_ref || github.ref_name }} GH_TOKEN: ${{ github.token }} + BODY: ${{ github.event.head_commit.message }} steps: - uses: actions/checkout@v4 - name: Create pull request run: | gh_pr_up() { - gh pr create --draft $* || gh pr edit $* + gh pr create --draft $* --body "${BODY}" || gh pr edit $* --body "${BODY}" } - gh_pr_up --base master --title "${BRANCH_NAME}" --body "Description" + gh_pr_up --base master --title "${BRANCH_NAME}" From f4eab69854934fdb3409e1e63291cf52caf8a3f5 Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Fri, 1 Nov 2024 15:22:42 +0100 Subject: [PATCH 09/16] Update workflows --- .github/workflows/DevelopServerDeploy.yml | 1 + .github/workflows/ProdServerDeploy.yml | 1 + .github/workflows/pull-request.yml | 1 + Dockerfile | 2 +- apps/ui/src/components/DriveTools.vue | 16 +- apps/ui/src/components/JobsViewer.vue | 36 +- apps/ui/src/components/UtilsMixin.ts | 21 -- apps/ui/src/components/WorkflowsEditor.vue | 2 +- apps/ui/src/modals/ToastsMixin.ts | 12 +- apps/ui/src/pages/FolderView.vue | 4 +- apps/wgd-action-runner/Dockerfile | 4 +- .../action/ActionRunnerContainer.ts | 339 +++++++++++------- src/containers/action/ActionTransform.ts | 54 +++ src/containers/action/DockerContainer.ts | 70 +++- src/containers/action/OciContainer.ts | 12 + src/containers/action/PodmanContainer.ts | 151 ++++++++ .../changes/WatchChangesContainer.ts | 5 - .../google_folder/UserConfigService.ts | 8 +- src/containers/job/JobManagerContainer.ts | 165 +++------ src/containers/server/ServerContainer.ts | 53 +-- .../transform/TransformContainer.ts | 2 +- .../transform/frontmatters/frontmatter.ts | 2 +- src/git/GitScanner.ts | 50 +-- website/docs/developer-guide.md | 1 + 24 files changed, 620 insertions(+), 392 deletions(-) create mode 100644 src/containers/action/ActionTransform.ts create mode 100644 src/containers/action/OciContainer.ts create mode 100644 src/containers/action/PodmanContainer.ts diff --git a/.github/workflows/DevelopServerDeploy.yml b/.github/workflows/DevelopServerDeploy.yml index 320d46cd..cd290f75 100644 --- a/.github/workflows/DevelopServerDeploy.yml +++ b/.github/workflows/DevelopServerDeploy.yml @@ -74,6 +74,7 @@ jobs: -v /home/wikigdrive/service_account.json:/service_account.json \ -v /home/wikigdrive/env.develop:/usr/src/app/.env \ -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/run/podman/podman.sock:/var/run/podman/podman.sock \ -v "/var/www/dev.wikigdrive.com:/usr/src/app/dist/hugo" \ -e "GIT_SHA=${GITHUB_SHA}" \ -e "ZIPKIN_URL=https://dev.wikigdrive.com/zipkin" \ diff --git a/.github/workflows/ProdServerDeploy.yml b/.github/workflows/ProdServerDeploy.yml index 901350be..abf3fc05 100644 --- a/.github/workflows/ProdServerDeploy.yml +++ b/.github/workflows/ProdServerDeploy.yml @@ -73,6 +73,7 @@ jobs: -v /home/wikigdrive/service_account.json:/service_account.json \ -v /home/wikigdrive/env.prod:/usr/src/app/.env \ -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/run/podman/podman.sock:/var/run/podman/podman.sock \ -v "/var/www/wikigdrive.com:/usr/src/app/dist/hugo" \ -e "GIT_SHA=${GITHUB_SHA}" \ --publish 127.0.0.1:3000:3000 \ diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 5daea8ff..b759282e 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -83,6 +83,7 @@ jobs: -v /home/wikigdrive/service_account.json:/service_account.json \ -v /home/wikigdrive/env.pr:/usr/src/app/.env \ -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/run/podman/podman.sock:/var/run/podman/podman.sock \ -e "GIT_SHA=${{ github.sha }}" \ -v "/var/www/pr-${{ github.event.number }}.wikigdrive.com:/usr/src/app/website/.vitepress/dist" \ -e "ZIPKIN_URL=https://pr-${{ github.event.number }}.wikigdrive.com/zipkin" \ diff --git a/Dockerfile b/Dockerfile index 2f027c66..6c99ecd6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ VOLUME /data WORKDIR /usr/src/app RUN apt-get update -RUN apt-get install -yq bash git-lfs openssh-client curl unzip socat +RUN apt-get install -yq bash git-lfs openssh-client curl unzip socat podman-remote RUN curl -fsSL https://deno.land/install.sh | DENO_INSTALL=/usr/local sh COPY package.json package-lock.json ./ diff --git a/apps/ui/src/components/DriveTools.vue b/apps/ui/src/components/DriveTools.vue index ad815b5c..78a0ad65 100644 --- a/apps/ui/src/components/DriveTools.vue +++ b/apps/ui/src/components/DriveTools.vue @@ -55,7 +55,7 @@
diff --git a/apps/ui/src/components/JobsViewer.vue b/apps/ui/src/components/JobsViewer.vue index dda9a828..66b849f7 100644 --- a/apps/ui/src/components/JobsViewer.vue +++ b/apps/ui/src/components/JobsViewer.vue @@ -9,16 +9,17 @@
Sync Single Sync All - Transform Single Markdown - Transform All Markdown
-
+
Workflows
- Render Preview +
- {{actions}}
@@ -79,7 +80,7 @@
diff --git a/apps/ui/src/components/UtilsMixin.ts b/apps/ui/src/components/UtilsMixin.ts index e83f9c7d..aaa25a1c 100644 --- a/apps/ui/src/components/UtilsMixin.ts +++ b/apps/ui/src/components/UtilsMixin.ts @@ -212,20 +212,6 @@ export const UtilsMixin = { }); }); }, - async renderPreview(event) { - await disableElement(event, async () => { - await this.authenticatedClient.fetchApi(`/api/run_action/${this.driveId}/transform`, { - method: 'post' - }); - }); - }, - async transformAll(event) { - await disableElement(event, async () => { - await this.authenticatedClient.fetchApi(`/api/transform/${this.driveId}`, { - method: 'post' - }); - }); - }, async uploadGdrive(event) { await disableElement(event, async () => { const response = await this.authenticatedClient.fetchApi('/api/gdrive/' + this.driveId + '/upload', { method: 'get' }); @@ -235,13 +221,6 @@ export const UtilsMixin = { } }); }, - async transformSingle(event, selectedFile) { - await disableElement(event, async () => { - await this.authenticatedClient.fetchApi(`/api/transform/${this.driveId}/${selectedFile.id}`, { - method: 'post' - }); - }); - }, downloadOdt(fileId) { const odtPath = `/api/drive/${this.driveId}/file/${fileId}.odt`; window.open(odtPath, '_blank'); diff --git a/apps/ui/src/components/WorkflowsEditor.vue b/apps/ui/src/components/WorkflowsEditor.vue index 3e7863ec..108708ae 100644 --- a/apps/ui/src/components/WorkflowsEditor.vue +++ b/apps/ui/src/components/WorkflowsEditor.vue @@ -6,7 +6,7 @@
- +
diff --git a/apps/ui/src/modals/ToastsMixin.ts b/apps/ui/src/modals/ToastsMixin.ts index 76efa395..f9eeae96 100644 --- a/apps/ui/src/modals/ToastsMixin.ts +++ b/apps/ui/src/modals/ToastsMixin.ts @@ -11,14 +11,14 @@ export const ToastsMixin = { await this.$root.changeDrive(this.drive.id); window.location.reload(); break; - case 'transform:scheduled': - this.$removeToastMatching(item => item.type.startsWith('transform:')); + case 'action:scheduled': + this.$removeToastMatching(item => item.type.startsWith('action:')); break; - case 'transform:failed': - this.$removeToastMatching(item => item.type.startsWith('transform:')); + case 'action:failed': + this.$removeToastMatching(item => item.type.startsWith('action:')); break; - case 'transform:done': - this.$removeToastMatching(item => item.type.startsWith('transform:')); + case 'action:done': + this.$removeToastMatching(item => item.type.startsWith('action:')); break; case 'sync:done': } diff --git a/apps/ui/src/pages/FolderView.vue b/apps/ui/src/pages/FolderView.vue index d76242e3..92216694 100644 --- a/apps/ui/src/pages/FolderView.vue +++ b/apps/ui/src/pages/FolderView.vue @@ -23,8 +23,8 @@
- - + + diff --git a/apps/wgd-action-runner/Dockerfile b/apps/wgd-action-runner/Dockerfile index f4a52fa2..af65e180 100644 --- a/apps/wgd-action-runner/Dockerfile +++ b/apps/wgd-action-runner/Dockerfile @@ -8,9 +8,6 @@ ENV THEME_ID="" ENV THEME_URL="" ENV NO_COLOR="true" -VOLUME ["/site/content"] -VOLUME ["/site/public"] - RUN apt-get update RUN apt-get install -y git nodejs curl python3 python3-pip s3cmd yq RUN npm i -g postcss postcss-cli hugo-extended @@ -26,3 +23,4 @@ ADD steps/ /steps RUN chmod a+x /steps/step_* WORKDIR /site +ENTRYPOINT /bin/bash diff --git a/src/containers/action/ActionRunnerContainer.ts b/src/containers/action/ActionRunnerContainer.ts index fa9b1a09..c3de760b 100644 --- a/src/containers/action/ActionRunnerContainer.ts +++ b/src/containers/action/ActionRunnerContainer.ts @@ -10,74 +10,142 @@ import {BufferWritable} from '../../utils/BufferWritable.ts'; import {UserConfigService} from '../google_folder/UserConfigService.ts'; import {GitScanner} from '../../git/GitScanner.ts'; import {FileContentService} from '../../utils/FileContentService.ts'; +import {DockerContainer} from './DockerContainer.ts'; +import {PodmanContainer} from './PodmanContainer.ts'; +import {ActionTransform} from './ActionTransform.ts'; const __filename = import.meta.filename; export interface ActionStep { name?: string; - uses: string; + uses?: string; + run?: string; with?: {[key: string]: string}; env?: {[key: string]: string}; } -export interface ActionDefinition { +export interface ActionDefinitionLegacy { on: string; 'run-name'?: string; steps: Array; } -export const DEFAULT_ACTIONS: ActionDefinition[] = [ - { - on: 'transform', - 'run-name': 'AutoCommit and Render', - steps: [ - { - name: 'auto_commit', - uses: 'auto_commit', - }, - { - name: 'render_hugo', - uses: 'render_hugo', - } - ] - }, - { - on: 'branch', - 'run-name': 'Commit and Push branch', - steps: [ - { - uses: 'commit_branch' - }, - { - uses: 'push_branch' - } - ] - }, - { - on: 'git_reset', - 'run-name': 'Render', - steps: [ - { - name: 'render_hugo', - uses: 'render_hugo', - } - ] +export interface WorkflowJob { + name: string; + 'runs-on'?: string; + steps: Array; + hide_in_menu?: boolean; +} + +export interface WorkflowDefinition { + on: {[trigger: string]: string}; + + jobs: { + [key: string]: WorkflowJob + } +} + +export const DEFAULT_WORKFLOW: WorkflowDefinition = { + on: { + 'internal/sync': 'transform_all', + 'transform_all': 'autocommit_render', + 'internal/branch': 'commit_and_push_branch' }, - { - on: 'git_pull', - 'run-name': 'Render', - steps: [ - { - name: 'render_hugo', - uses: 'render_hugo', - } - ] + + jobs: { + 'transform_all': { + name: 'Transform All', + steps: [ + { + uses: 'internal/transform', + } + ] + }, + 'autocommit_render': { + name: 'AutoCommit and Render', + steps: [ + { + name: 'internal/auto_commit', + uses: 'internal/auto_commit', + }, + { + name: 'internal/render_hugo', + uses: 'internal/render_hugo', + }, + { + name: 'Export preview to nginx', + uses: 'internal/export_preview' + } + ] + }, + 'commit_and_push_branch': { + name: 'Commit and Push branch', + hide_in_menu: true, + steps: [ + { + uses: 'internal/commit_branch' + }, + { + uses: 'internal/push_branch' + } + ] + } + } + // name: 'Check PR Labels' + // runs-on: ubuntu-latest + +}; + +function migrateStep(step: ActionStep): ActionStep { + if (step.uses === 'exec' && step.env?.EXEC) { + return { + name: step.name, + run: step.env.EXEC + }; + } + + if (step.uses === 'auto_commit') { + step.uses = 'internal/auto_commit'; + } + if (step.uses === 'commit_branch') { + step.uses = 'internal/commit_branch'; } -]; -export async function convertActionYaml(actionYaml: string): Promise { - const actionDefs: ActionDefinition[] = actionYaml ? yaml.load(actionYaml) : DEFAULT_ACTIONS; - return actionDefs; + return step; +} + +export function migrateLegacy(actionDefs: ActionDefinitionLegacy[]): WorkflowDefinition { + const retVal: WorkflowDefinition = DEFAULT_WORKFLOW; + + for (const actionDef of actionDefs) { + switch (actionDef.on) { + case 'transform': + retVal.jobs['autocommit_render'].steps = actionDef.steps.map(step => migrateStep(step)); + retVal.jobs['autocommit_render'].steps.push( { + name: 'Export preview to nginx', + uses: 'internal/export_preview' + }); + break; + } + } + + for (const jobId in retVal.jobs) { + const job = retVal.jobs[jobId]; + job['runs-on'] = 'docker'; + } + + return retVal; +} + +export async function convertActionYaml(actionYaml: string): Promise { + if (!actionYaml) { + return DEFAULT_WORKFLOW; + } + + const yamlObj = yaml.load(actionYaml); + const workflow: WorkflowDefinition = (Array.isArray(yamlObj)) ? migrateLegacy(yamlObj) : yamlObj; + + return workflow; } function withToEnv(map: { [p: string]: string }) { @@ -141,23 +209,21 @@ export class ActionRunnerContainer extends Container { this.isErr = false; - const actionDefs = await convertActionYaml(config.actions_yaml); - for (const actionDef of actionDefs) { - if (actionDef.on !== this.params['trigger']) { - continue; - } + const workflow = await convertActionYaml(config.actions_yaml); + const workflowJobId = workflow.on[this.params['trigger']] || this.params['action_id']; - if (actionDef.on === 'commit') { - await gitScanner.pushToDir(this.tempFileService.getRealPath()); + if (workflow.jobs[workflowJobId]) { + const workflowJob = workflow.jobs[workflowJobId]; + if (this.params['trigger'] === 'commit') { + await gitScanner.pushToDir(this.tempFileService.getRealPath()); } - const generatedFileService = actionDef.on === 'commit' ? this.tempFileService : this.generatedFileService; + const generatedFileService = this.params['trigger'] === 'commit' ? this.tempFileService : this.generatedFileService; - if (!Array.isArray(actionDef.steps)) { + const steps = workflowJob.steps; + if (!Array.isArray(steps)) { throw new Error('No action steps'); } - const driveIdTransform: string = path.basename(generatedFileService.getRealPath()); - const committer = { name: this.params.user_name || 'WikiGDrive', email: this.params.user_email || 'wikigdrive@wikigdrive.com' @@ -167,17 +233,6 @@ export class ActionRunnerContainer extends Container { const writable = new BufferWritable(); - // await this.generatedFileService.remove('resources'); - -/* const env = ['render_hugo', 'exec', 'commit_branch'].includes(step.uses) ? Object.assign({ - CONFIG_TOML: '/site/tmp_dir/config.toml', - BASE_URL: `${process.env.DOMAIN}/preview/${driveId}/_manual/`, - GIT_AUTHOR_NAME: committer.name, - GIT_AUTHOR_EMAIL: committer.email, - GIT_COMMITTER_NAME: committer.name, - GIT_COMMITTER_EMAIL: committer.email - }, step.env, additionalEnv) : Object.assign({}, step.env, additionalEnv);*/ - const env = Object.assign({ CONFIG_TOML: '/site/tmp_dir/config.toml', BASE_URL: `${process.env.DOMAIN}/preview/${driveId}/_manual/`, @@ -187,37 +242,21 @@ export class ActionRunnerContainer extends Container { GIT_COMMITTER_EMAIL: committer.email }, additionalEnv); - //--user=$(id -u):$(getent group docker | cut -d: -f3) - this.logger.info(`DockerAPI:\ndocker start \\ - --user=${process.getuid()}:${process.getegid()} \\ - // -v "${process.env.VOLUME_DATA}/${driveId}_transform:/repo:ro" \\ - // -v "${process.env.VOLUME_DATA}/${driveIdTransform}:/site:rw" \\ - // --mount "type=tmpfs,destination=/site/resources" \\ - ${Object.keys(env).map(key => `--env ${key}="${env[key]}"`).join(' ')} \\ - ${process.env.ACTION_IMAGE} - `); + const container = workflowJob['runs-on'] === 'podman' ? + await PodmanContainer.create(this.logger, 'localhost/' + process.env.ACTION_IMAGE, env, `/${driveId}_transform`) : + await DockerContainer.create(this.logger, process.env.ACTION_IMAGE, env, generatedFileService.getRealPath()); - const container = new DockerContainer(process.env.ACTION_IMAGE); try { - await container.create(env, writable); + container.skipMount = (steps.length === 1 && steps[0].uses === 'internal/transform'); await container.start(); - this.logger.info('container created: ' + container.id); + // await container.mountOverlay(generatedFileService.getRealPath(), '/site'); - this.logger.info('docker cp . /site'); - await container.copy(generatedFileService.getRealPath(), '/site'); - - // Convert to step: - const configToml = config?.config_toml || '#relativeURLs = true\n' + - 'languageCode = "en-us"\n' + - 'title = "My New Hugo Site"\n'; - this.logger.info('docker write /site/tmp_dir/config.toml'); + // TODO: Convert to step + const configToml = config?.config_toml || 'languageCode = "en-us"\ntitle = "My New Hugo Site"\n'; await container.putFile(new TextEncoder().encode(configToml), '/site/tmp_dir/config.toml'); - // - - for (const step of actionDef.steps) { - this.logger.info('Step: ' + (step.name || step.uses)); + for (const step of steps) { if (!step.env) { step.env = {}; } @@ -225,39 +264,81 @@ export class ActionRunnerContainer extends Container { step.env['PAYLOAD'] = this.params.payload; let lastCode = 0; - switch (step.uses) { - case 'push_branch': - { - const additionalEnv = this.payloadToEnv(); - await gitScanner.pushBranch(`wgd/${additionalEnv['BRANCH']}`, { - privateKeyFile: await this.userConfigService.getDeployPrivateKeyPath() - }, `wgd/${additionalEnv['BRANCH']}`); - } - break; - case 'auto_commit': - { - await gitScanner.autoCommit(); + + if (step.run) { + this.logger.info('Step: ' + (step.name || step.run)); + + try { + lastCode = await container.exec(step.run, Object.assign(step.env, withToEnv(step.with)), writable); + if (lastCode > 0) { + this.logger.error('err: '+new TextDecoder().decode(writable.getBuffer())); + } else { + this.logger.info(new TextDecoder().decode(writable.getBuffer())); + } + } catch (err) { + this.logger.error(err.stack ? err.stack : err.message); + lastCode = 1; } - break; - default: - this.logger.info(`docker exec ${container.id} /steps/step_${step.uses}`); - try { - if (step.uses.indexOf('/') > -1 && step.uses.indexOf('@') > -1) { - const [action_repo, action_version] = step.uses.split('@'); - lastCode = await container.exec(`/steps/step_gh_action ${action_repo} ${action_version}`, Object.assign(step.env, withToEnv(step.with))); - } else { - lastCode = await container.exec(`/steps/step_${step.uses}`, Object.assign(step.env, withToEnv(step.with))); + } else + if (step.uses) { + this.logger.info('Step: ' + (step.name || step.uses)); + + switch (step.uses) { + case 'internal/transform': + try { + const action = new ActionTransform(this.engine, this.filesService, this.generatedFileService); + let selectedFileId = undefined; + try { + const payload = JSON.parse(this.params.payload); + selectedFileId = payload.selectedFileId; + } catch (ignore) { /* empty */ } + await action.execute(driveId, this.params.jobId, selectedFileId ? [ selectedFileId ] : [] ); + } catch (err) { + this.logger.error(err.stack ? err.stack : err.message); + lastCode = 1; } - if (lastCode > 0) { - this.logger.error(writable.getBuffer().toString()); - } else { - this.logger.info(writable.getBuffer().toString()); + break; + + case 'internal/push_branch': + { + const additionalEnv = this.payloadToEnv(); + await gitScanner.pushBranch(`wgd/${additionalEnv['BRANCH']}`, { + privateKeyFile: await this.userConfigService.getDeployPrivateKeyPath() + }, `wgd/${additionalEnv['BRANCH']}`); + } + break; + case 'internal/auto_commit': + { + gitScanner.debug = true; + await gitScanner.setSafeDirectory(); + await gitScanner.autoCommit(); + } + break; + default: + try { + if (step.uses.indexOf('/') > -1 && step.uses.indexOf('@') > -1) { + const [action_repo, action_version] = step.uses.split('@'); + lastCode = await container.exec(`/steps/step_gh_action ${action_repo} ${action_version}`, Object.assign(step.env, withToEnv(step.with)), writable); + } else { + lastCode = await container.exec(`/steps/step_${step.uses}`, Object.assign(step.env, withToEnv(step.with)), writable); + } + if (lastCode > 0) { + this.logger.error('err: '+new TextDecoder().decode(writable.getBuffer())); + } else { + this.logger.info(new TextDecoder().decode(writable.getBuffer())); + } + } catch (err) { + this.logger.error(err.stack ? err.stack : err.message); + lastCode = 1; } - } catch (err) { - this.logger.error(err.stack ? err.stack : err.message); - lastCode = 1; + break; + case 'internal/export_preview': + { + const previewOutput = `${process.env.VOLUME_PREVIEW}/${driveId}/_manual`; + await container.export('/site/public', previewOutput); } - break; + break; + } } if (0 !== lastCode) { this.isErr = true; @@ -265,12 +346,6 @@ export class ActionRunnerContainer extends Container { } } - // Convert to step - const previewOutput = `${process.env.VOLUME_PREVIEW}/${driveId}/_manual`; - this.logger.info('docker export /site/public ' + previewOutput); - await container.export('/site/public', previewOutput); - // - this.logger.info('Action completed'); } catch (err) { diff --git a/src/containers/action/ActionTransform.ts b/src/containers/action/ActionTransform.ts new file mode 100644 index 00000000..31bdf6c7 --- /dev/null +++ b/src/containers/action/ActionTransform.ts @@ -0,0 +1,54 @@ +import {FileId} from '../../model/model.ts'; +import {TransformContainer} from '../transform/TransformContainer.ts'; +import {UserConfigService} from '../google_folder/UserConfigService.ts'; +import {getContentFileService} from '../transform/utils.ts'; +import {MarkdownTreeProcessor} from '../transform/MarkdownTreeProcessor.ts'; +import {ContainerEngine} from '../../ContainerEngine.ts'; +import {clearCachedChanges, JobManagerContainer} from '../job/JobManagerContainer.ts'; +import {FileContentService} from '../../utils/FileContentService.ts'; + +export class ActionTransform { + + constructor(private engine: ContainerEngine, private googleFileSystem: FileContentService, private transformedFileSystem: FileContentService) { + } + + async execute(folderId: FileId, jobId: string, filesIds: FileId[] = []) { + const transformContainer = new TransformContainer({ + folderId, + name: jobId, + jobId + }, { filesIds }); + await transformContainer.mount2( + this.googleFileSystem, + this.transformedFileSystem + ); + + const userConfigService = new UserConfigService(this.googleFileSystem); + await userConfigService.load(); + + transformContainer.setUseGoogleMarkdowns(userConfigService.config.use_google_markdowns); + + const jobManager = this.engine.getContainer('job_manager'); + transformContainer.onProgressNotify(({ completed, total, warnings, failed }) => { + jobManager.progressJob(folderId, jobId, { completed, total, warnings, failed }); + }); + + await this.engine.registerContainer(transformContainer); + try { + await transformContainer.run(folderId); + if (transformContainer.failed()) { + throw new Error('Transform failed'); + } + + const contentFileService = await getContentFileService(this.transformedFileSystem, userConfigService); + const markdownTreeProcessor = new MarkdownTreeProcessor(contentFileService); + await markdownTreeProcessor.load(); + + } finally { + await this.engine.unregisterContainer(transformContainer.params.name); + } + + await clearCachedChanges(this.googleFileSystem); + } + +} diff --git a/src/containers/action/DockerContainer.ts b/src/containers/action/DockerContainer.ts index 486e7f23..9a51fc4c 100644 --- a/src/containers/action/DockerContainer.ts +++ b/src/containers/action/DockerContainer.ts @@ -3,20 +3,25 @@ import path from 'path'; import tarFs from 'tar-fs'; import tarStream from 'tar-stream'; import {PassThrough, Writable} from 'stream'; - -export class DockerContainer { - public id: string; - private docker: Docker; - private container: Docker.Container; - private writable: Writable; - constructor(private image: string) { - this.docker = new Docker({socketPath: '/var/run/docker.sock'}); +import winston from 'winston'; +import {OciContainer} from './OciContainer.ts'; +import {BufferWritable} from '../../utils/BufferWritable.ts'; + +export class DockerContainer implements OciContainer { + public skipMount: false; + + private constructor(private logger: winston.Logger, + public readonly id: string, + public readonly image: string, + private container: Docker.Container, + private repoSubDir: string) { } - async create(env: { [p: string]: string }, writable: Writable) { - this.writable = writable; - this.container = await this.docker.createContainer({ - Image: process.env.ACTION_IMAGE, + static async create(logger: winston.Logger, image: string, env: { [p: string]: string }, repoSubDir: string): Promise { + const dockerEngine = new Docker({socketPath: '/var/run/docker.sock'}); + + const container = await dockerEngine.createContainer({ + Image: image, AttachStdin: false, AttachStdout: true, AttachStderr: true, @@ -45,11 +50,27 @@ export class DockerContainer { Env: Object.keys(env).map(key => `${key}=${env[key]}`), User: String(process.getuid())+ ':' + String(process.getegid()) }); - this.id = this.container.id; + + //--user=$(id -u):$(getent group docker | cut -d: -f3) + // logger.info(`DockerAPI:\ndocker start \\ + // --user=${process.getuid()}:${process.getegid()} \\ + // // -v "${process.env.VOLUME_DATA}/${driveId}_transform:/repo:ro" \\ + // // -v "${process.env.VOLUME_DATA}/${driveIdTransform}:/site:rw" \\ + // // --mount "type=tmpfs,destination=/site/resources" \\ + // ${Object.keys(env).map(key => `--env ${key}="${env[key]}"`).join(' ')} \\ + // ${process.env.ACTION_IMAGE} + // `); + + return new DockerContainer(logger, container.id, image, container, repoSubDir); } async start() { - return this.container.start(); + await this.container.start(); + this.logger.info('docker started: ' + this.id); + + if (!this.skipMount) { + await this.copy(this.repoSubDir, '/site'); + } } async stop() { @@ -57,6 +78,8 @@ export class DockerContainer { } async copy(realPath: string, remotePath: string, ignoreGit = false) { + this.logger.info('docker cp into ' + remotePath); + const archive = tarFs.pack(realPath, { ignore (name) { if (ignoreGit && name.startsWith(path.join(realPath, '.git'))) { @@ -78,12 +101,20 @@ export class DockerContainer { const archive = tarStream.pack(); archive.entry({ name: remotePath }, configToml); archive.finalize(); - await this.container.putArchive(archive, { + + const writable = new BufferWritable(); + archive.pipe(writable); + + this.logger.info('docker write into ' + remotePath); + + await this.container.putArchive(writable.getBuffer(), { path: '/' }); } async export(remotePath: string, outputDir: string) { + this.logger.info('docker export /site/public'); + const archive = await this.container.getArchive({ path: remotePath }); @@ -111,7 +142,9 @@ export class DockerContainer { }); } - async exec(command: string, env: { [p: string]: string}) { + async exec(command: string, env: { [p: string]: string}, writable: Writable) { + this.logger.info(`docker exec ${this.id} ${command}`); + const cancelTimeout = new AbortController(); const exec = await this.container.exec({ @@ -132,7 +165,10 @@ export class DockerContainer { this.container.modem.demuxStream(stream, stdout, stderr); stdout.on('data', (chunk: Buffer) => { - this.writable.write(chunk); + writable.write(chunk); + }); + stderr.on('data', (chunk: Buffer) => { + writable.write(chunk); }); await new Promise(resolve => stream.on('end', () => { diff --git a/src/containers/action/OciContainer.ts b/src/containers/action/OciContainer.ts new file mode 100644 index 00000000..bd2386c9 --- /dev/null +++ b/src/containers/action/OciContainer.ts @@ -0,0 +1,12 @@ +import {Writable} from 'stream'; + +export interface OciContainer { + skipMount: boolean; + + start(): Promise; + copy(localPath: string, remotePath: string): Promise; + putFile(uint8Array: Uint8Array, remotePath: string): Promise; + export(remotePath: string, localPath: string): Promise; + exec(command: string, env: { [p: string]: string }, writable: Writable): Promise; + stop(): Promise; +} diff --git a/src/containers/action/PodmanContainer.ts b/src/containers/action/PodmanContainer.ts new file mode 100644 index 00000000..d799e5e5 --- /dev/null +++ b/src/containers/action/PodmanContainer.ts @@ -0,0 +1,151 @@ +import Docker from 'dockerode'; +import path from 'path'; +import tarFs from 'tar-fs'; +import tarStream from 'tar-stream'; +import {PassThrough, Writable} from 'stream'; +import winston from 'winston'; +import {OciContainer} from './OciContainer.ts'; +import {BufferWritable} from '../../utils/BufferWritable.ts'; + +export class PodmanContainer implements OciContainer { + public skipMount: false; + + private constructor(private logger: winston.Logger, public readonly id: string, public readonly image: string, private container: Docker.Container) { + } + + static async create(logger: winston.Logger, image: string, env: { [p: string]: string }, repoSubDir: string): Promise { + const podmanEngine = new Docker({socketPath: '/var/run/podman/podman.sock'}); + + const container = await podmanEngine.createContainer({ + Image: image, + AttachStdin: false, + AttachStdout: true, + AttachStderr: true, + Tty: true, + OpenStdin: false, + StdinOnce: false, + + HostConfig: { + Binds: [ // Unlike Mounts those are created if not existing in the host + `${process.env.VOLUME_DATA}/${repoSubDir}:/site:O`, + ], + }, + Env: Object.keys(env).map(key => `${key}=${env[key]}`), + User: String(process.getuid())+ ':' + String(process.getegid()) + }); + + return new PodmanContainer(logger, container.id, image, container); + } + + async start() { + await this.container.start(); + this.logger.info('podman started: ' + this.id); + } + + async stop() { + return this.container.stop(); + } + + async copy(realPath: string, remotePath: string, ignoreGit = false) { + this.logger.info('podman copy into ' + remotePath); + + const archive = tarFs.pack(realPath, { + ignore (name) { + if (ignoreGit && name.startsWith(path.join(realPath, '.git'))) { + return true; + } + if (name.startsWith(path.join(realPath, '.private'))) { + return true; + } + return false; + }, + }); + + await this.container.putArchive(archive, { + path: remotePath + }); + } + + async putFile(configToml: Uint8Array, remotePath: string) { + const archive = tarStream.pack(); + archive.entry({ name: remotePath }, configToml); + archive.finalize(); + + const writable = new BufferWritable(); + archive.pipe(writable); + + this.logger.info('podman write into ' + remotePath); + + await this.container.putArchive(writable.getBuffer(), { + path: '/' + }); + } + + async export(remotePath: string, outputDir: string) { + this.logger.info('podman export /site/public'); + + const archive = await this.container.getArchive({ + path: remotePath + }); + + await new Promise((resolve, reject) => { + try { + const stream = archive.pipe(tarFs.extract(outputDir, { + map (header) { + const parts = header.name.split('/'); + parts.shift(); + header.name = parts.join('/'); + return header; + } + })); + + stream.on('finish', () => { + resolve(); + }); + stream.on('error', (err: unknown) => { + reject(err); + }); + } catch (err) { + reject(err); + } + }); + } + + async exec(command: string, env: { [p: string]: string}, writable: Writable) { + this.logger.info(`podman exec ${this.id} ${command}`); + + const cancelTimeout = new AbortController(); + + const exec = await this.container.exec({ + Cmd: command.split(' '), + AttachStdin: false, + AttachStdout: true, + AttachStderr: true, + Env: Object.keys(env).map(key => `${key}=${env[key]}`), + //WorkingDir + abortSignal: cancelTimeout.signal, + }); + + const stream = await exec.start({hijack: true, Detach: false}); + + const stdout = new PassThrough(); + const stderr = new PassThrough(); + this.container.modem.demuxStream(stream, stdout, stderr); + + stdout.on('data', (chunk: Buffer) => { + writable.write(chunk); + }); + stderr.on('data', (chunk: Buffer) => { + writable.write(chunk); + }); + + await new Promise(resolve => stream.on('end', () => { + resolve(0); + })); + + const inspectInfo = await exec.inspect(); + + return inspectInfo.ExitCode; + } + +} diff --git a/src/containers/changes/WatchChangesContainer.ts b/src/containers/changes/WatchChangesContainer.ts index 88fd62ad..3f8ad69e 100644 --- a/src/containers/changes/WatchChangesContainer.ts +++ b/src/containers/changes/WatchChangesContainer.ts @@ -99,11 +99,6 @@ export class WatchChangesContainer extends Container { payload: fileIdsString, title: 'Syncing file: ' + fileIdsString }); - await jobManagerContainer.schedule(driveId, { - ...initJob(), - type: 'transform', - title: 'Transform markdown' - }); } } diff --git a/src/containers/google_folder/UserConfigService.ts b/src/containers/google_folder/UserConfigService.ts index 6b445196..f8eddd0d 100644 --- a/src/containers/google_folder/UserConfigService.ts +++ b/src/containers/google_folder/UserConfigService.ts @@ -5,7 +5,7 @@ import yaml from 'js-yaml'; import {FileContentService} from '../../utils/FileContentService.ts'; import {HugoTheme} from '../server/routes/ConfigController.ts'; import {FRONTMATTER_DUMP_OPTS} from '../transform/frontmatters/frontmatter.ts'; -import {DEFAULT_ACTIONS} from '../action/ActionRunnerContainer.ts'; +import {convertActionYaml} from '../action/ActionRunnerContainer.ts'; import {RewriteRule} from '../../odt/applyRewriteRule.ts'; async function execAsync(command: string) { @@ -84,9 +84,9 @@ export class UserConfigService { this.config = structuredClone(DEFAULT_CONFIG); await this.save(); } - if (!this.config.actions_yaml) { - this.config.actions_yaml = yaml.dump(DEFAULT_ACTIONS, FRONTMATTER_DUMP_OPTS); - } + + const workflow = await convertActionYaml(this.config.actions_yaml); + this.config.actions_yaml = yaml.dump(workflow, FRONTMATTER_DUMP_OPTS); if (!this.config.rewrite_rules || this.config.rewrite_rules.length === 0) { this.config.rewrite_rules = DEFAULT_REWRITE_RULES; } diff --git a/src/containers/job/JobManagerContainer.ts b/src/containers/job/JobManagerContainer.ts index a0b5c3a4..afef2652 100644 --- a/src/containers/job/JobManagerContainer.ts +++ b/src/containers/job/JobManagerContainer.ts @@ -6,7 +6,6 @@ import { randomUUID } from 'node:crypto'; import {Container, ContainerConfig, ContainerEngine} from '../../ContainerEngine.ts'; import {FileId} from '../../model/model.ts'; import {GoogleFolderContainer} from '../google_folder/GoogleFolderContainer.ts'; -import {TransformContainer} from '../transform/TransformContainer.ts'; import {UserConfigService} from '../google_folder/UserConfigService.ts'; import {MarkdownTreeProcessor} from '../transform/MarkdownTreeProcessor.ts'; import {WorkerPool} from './WorkerPool.ts'; @@ -37,6 +36,7 @@ export interface Job { type: JobType; title: string; trigger?: string; + action_id?: string; payload?: string; access_token?: string; ts?: number; // scheduled at @@ -127,7 +127,7 @@ export class JobManagerContainer extends Container { this.workerPool = new WorkerPool(os.cpus().length); } - async getDriveJobs(driveId): Promise { + async getDriveJobs(driveId: FileId): Promise { if (!this.driveJobsMap[driveId]) { const driveFileSystem = await this.filesService.getSubFileService(driveId, ''); const driveJobs = await driveFileSystem.readJson('.jobs.json'); @@ -141,7 +141,7 @@ export class JobManagerContainer extends Container { return this.driveJobsMap[driveId]; } - async setDriveJobs(driveId, driveJobs: DriveJobs) { + async setDriveJobs(driveId: FileId, driveJobs: DriveJobs) { if (driveJobs) { this.driveJobsMap[driveId] = driveJobs; } @@ -192,25 +192,23 @@ export class JobManagerContainer extends Container { const userConfigService = new UserConfigService(googleFileSystem); await userConfigService.load(); const config = userConfigService.config; - const actionDefs = await convertActionYaml(config.actions_yaml); - const action = actionDefs.find(action => action.on === job.trigger); - if (action && action['run-name']) { - job.title = action['run-name']; + const workflow = await convertActionYaml(config.actions_yaml); + + const actionId = job.action_id ? job.action_id : workflow.on[job.trigger]; + const workflowJob = workflow.jobs[actionId]; + if (workflowJob && workflowJob.name) { + job.title = workflowJob.name; + job.action_id = actionId; + driveJobs.jobs.push(job); + + this.engine.emit(driveId, 'toasts:added', { + title: 'Scheduled: ' + workflowJob.name, + message: JSON.stringify(job, null, 2), + type: 'action:scheduled', + payload: job.payload ? job.payload : 'all' + }); } } - driveJobs.jobs.push(job); - break; - case 'transform': - if (driveJobs.jobs.find(subJob => subJob.type === 'transform' && notCompletedJob(subJob))) { - return; - } - this.engine.emit(driveId, 'toasts:added', { - title: 'Transform scheduled', - message: JSON.stringify(job, null, 2), - type: 'transform:scheduled', - payload: job.payload ? job.payload : 'all' - }); - driveJobs.jobs.push(job); break; case 'git_fetch': if (driveJobs.jobs.find(subJob => subJob.type === 'git_fetch' && notCompletedJob(subJob))) { @@ -454,55 +452,6 @@ export class JobManagerContainer extends Container { }, 100); } - private async transform(folderId: FileId, jobId: string, filesIds: FileId[] = []) { - const transformContainer = new TransformContainer({ - name: folderId, - jobId - }, { filesIds }); - const transformedFileSystem = await this.filesService.getSubFileService(folderId + '_transform', '/'); - const googleFileSystem = await this.filesService.getSubFileService(folderId, '/'); - await transformContainer.mount2( - googleFileSystem, - transformedFileSystem - ); - - const userConfigService = new UserConfigService(googleFileSystem); - await userConfigService.load(); - - transformContainer.setUseGoogleMarkdowns(userConfigService.config.use_google_markdowns); - - transformContainer.onProgressNotify(({ completed, total, warnings, failed }) => { - if (!this.driveJobsMap[folderId]) { - return; - } - const jobs = this.driveJobsMap[folderId].jobs || []; - const job = jobs.find(j => j.state === 'running' && j.type === 'transform'); - if (job) { - job.progress = { - completed: completed, - total: total, - failed: failed, - warnings - }; - this.engine.emit(folderId, 'jobs:changed', this.driveJobsMap[folderId]); - } - }); - await this.engine.registerContainer(transformContainer); - try { - await transformContainer.run(folderId); - if (transformContainer.failed()) { - throw new Error('Transform failed'); - } - - const contentFileService = await getContentFileService(transformedFileSystem, userConfigService); - const markdownTreeProcessor = new MarkdownTreeProcessor(contentFileService); - await markdownTreeProcessor.load(); - - } finally { - await this.engine.unregisterContainer(transformContainer.params.name); - } - } - private async upload(folderId: FileId, jobId: string, access_token: string) { const uploadContainer = new UploadContainer({ cmd: 'pull', @@ -576,13 +525,20 @@ export class JobManagerContainer extends Container { } finally { await this.engine.unregisterContainer(downloadContainer.params.name); } + + await this.schedule(folderId, { + ...initJob(), + type: 'run_action', + title: 'Run action: on sync', + trigger: 'internal/sync' + }); } - private async runAction(folderId: FileId, jobId: string, trigger: string, payload: string, user?: { name: string, email: string }) { + private async runAction(folderId: FileId, jobId: string, action_id: string, payload: string, user?: { name: string, email: string }) { const runActionContainer = new ActionRunnerContainer({ name: folderId, jobId, - trigger, + action_id, payload, user_name: user?.name || 'WikiGDrive', user_email: user?.email || 'wikigdrive@wikigdrive.com' @@ -804,56 +760,25 @@ export class JobManagerContainer extends Container { case 'sync_all': await this.sync(driveId, currentJob.id); break; - case 'transform': - try { - await this.transform(driveId, currentJob.id, currentJob.payload ? [ currentJob.payload ] : [] ); - await this.clearGitCache(driveId); - - driveJobs.jobs = driveJobs.jobs.filter(removeOldByType('transform')); - this.engine.emit(driveId, 'toasts:added', { - title: 'Transform done', - type: 'transform:done', - payload: currentJob.payload || 'all' - }); - - await this.schedule(driveId, { - ...initJob(), - type: 'run_action', - title: 'Run action: on ' + currentJob.type, - payload: currentJob.payload || 'all', - trigger: currentJob.type - }); - } catch (err) { - driveJobs.jobs = driveJobs.jobs.filter(removeOldByType('transform')); - this.engine.emit(driveId, 'toasts:added', { - title: 'Transform failed', - type: 'transform:failed', - err: err.message, - links: { - ['#drive_logs:job-' + currentJob.id]: 'View logs' - }, - payload: currentJob.payload || 'all' - }); - throw err; - } - break; case 'run_action': try { - await this.runAction(driveId, currentJob.id, currentJob.trigger, currentJob.payload, currentJob.user); - await this.clearGitCache(driveId); + await this.runAction(driveId, currentJob.id, currentJob.action_id, currentJob.payload, currentJob.user); + await this.clearGitCache(driveId); // TODO: check if necessary? this.engine.emit(driveId, 'toasts:added', { - title: 'Action done', + title: 'Done: ' + currentJob.title, type: 'run_action:done', + payload: this.params.payload }); } catch (err) { this.engine.emit(driveId, 'toasts:added', { - title: 'Action failed', + title: 'Failed: ' + currentJob.title, type: 'run_action:failed', err: err.message, links: { ['#drive_logs:job-' + currentJob.id]: 'View logs' - } + }, + payload: this.params.payload }); throw err; } finally { @@ -897,12 +822,6 @@ export class JobManagerContainer extends Container { '#git_log': 'View git history' }, }); - - await this.schedule(driveId, { - ...initJob(), - type: 'transform', - title: 'Transform markdown' - }); } catch (err) { driveJobs.jobs = driveJobs.jobs.filter(removeOldByType('git_pull')); this.engine.emit(driveId, 'toasts:added', { @@ -947,4 +866,20 @@ export class JobManagerContainer extends Container { async destroy(): Promise { } + progressJob(folderId: FileId, jobId: string,{ completed, total, warnings, failed }) { + if (!this.driveJobsMap[folderId]) { + return; + } + const jobs = this.driveJobsMap[folderId].jobs || []; + const job = jobs.find(j => j.id === jobId); + if (job) { + job.progress = { + completed: completed, + total: total, + failed: failed, + warnings + }; + this.engine.emit(folderId, 'jobs:changed', this.driveJobsMap[folderId]); + } + } } diff --git a/src/containers/server/ServerContainer.ts b/src/containers/server/ServerContainer.ts index 6425ba59..455a7973 100644 --- a/src/containers/server/ServerContainer.ts +++ b/src/containers/server/ServerContainer.ts @@ -301,7 +301,7 @@ export class ServerContainer extends Container { } }); - app.post('/api/run_action/:driveId/:trigger', authenticate(this.logger, 2), async (req, res, next) => { + app.post('/api/run_action/:driveId/:action_id', authenticate(this.logger, 2), async (req, res, next) => { try { const driveId = urlToFolderId(req.params.driveId); @@ -309,8 +309,8 @@ export class ServerContainer extends Container { await jobManagerContainer.schedule(driveId, { ...initJob(), type: 'run_action', - title: 'Run action: on ' + req.params.trigger, - trigger: req.params.trigger, + title: 'Run action: ' + req.params.action_id, + action_id: req.params.action_id, payload: req.body ? JSON.stringify(req.body) : '', user: req.user }); @@ -321,42 +321,6 @@ export class ServerContainer extends Container { } }); - app.post('/api/transform/:driveId', authenticate(this.logger, 2), async (req, res, next) => { - try { - const driveId = urlToFolderId(req.params.driveId); - - const jobManagerContainer = this.engine.getContainer('job_manager'); - await jobManagerContainer.schedule(driveId, { - ...initJob(), - type: 'transform', - title: 'Transform Markdown' - }); - - res.json({ driveId }); - } catch (err) { - next(err); - } - }); - - app.post('/api/transform/:driveId/:fileId', authenticate(this.logger, 2), async (req, res, next) => { - try { - const driveId = urlToFolderId(req.params.driveId); - const fileId = req.params.fileId; - - const jobManagerContainer = this.engine.getContainer('job_manager'); - await jobManagerContainer.schedule(driveId, { - ...initJob(), - type: 'transform', - payload: fileId, - title: 'Transform Single' - }); - - res.json({ driveId }); - } catch (err) { - next(err); - } - }); - app.post('/api/sync/:driveId', authenticate(this.logger, 2), async (req, res, next) => { try { const driveId = urlToFolderId(req.params.driveId); @@ -367,11 +331,6 @@ export class ServerContainer extends Container { type: 'sync_all', title: 'Syncing all' }); - await jobManagerContainer.schedule(driveId, { - ...initJob(), - type: 'transform', - title: 'Transform markdown' - }); res.json({ driveId }); } catch (err) { @@ -401,12 +360,6 @@ export class ServerContainer extends Container { payload: fileId, title: 'Syncing file: ' + fileTitle }); - await jobManagerContainer.schedule(driveId, { - ...initJob(), - type: 'transform', - payload: fileId, - title: 'Transform markdown' - }); res.json({ driveId, fileId }); } catch (err) { diff --git a/src/containers/transform/TransformContainer.ts b/src/containers/transform/TransformContainer.ts index 3de71191..395617a4 100644 --- a/src/containers/transform/TransformContainer.ts +++ b/src/containers/transform/TransformContainer.ts @@ -225,7 +225,7 @@ export class TransformContainer extends Container { async init(engine: ContainerEngine): Promise { await super.init(engine); - this.logger = engine.logger.child({ filename: __filename, driveId: this.params.name, jobId: this.params.jobId }); + this.logger = engine.logger.child({ filename: __filename, driveId: this.params.folderId, jobId: this.params.jobId }); this.transformLog = new TransformLog(); this.logger.add(this.transformLog); } diff --git a/src/containers/transform/frontmatters/frontmatter.ts b/src/containers/transform/frontmatters/frontmatter.ts index a06d2021..50a8d346 100644 --- a/src/containers/transform/frontmatters/frontmatter.ts +++ b/src/containers/transform/frontmatters/frontmatter.ts @@ -27,7 +27,7 @@ export function frontmatter(string) { export const FRONTMATTER_DUMP_OPTS = { flowLevel: 9, - forceQuotes: true, + forceQuotes: false, styles: { '!!null' : 'camelcase' } diff --git a/src/git/GitScanner.ts b/src/git/GitScanner.ts index 96fbe0e0..285544f0 100644 --- a/src/git/GitScanner.ts +++ b/src/git/GitScanner.ts @@ -43,6 +43,8 @@ interface ExecOpts { export class GitScanner { + public debug = false; + constructor(private logger: Logger, public readonly rootPath: string, private email: string) { } @@ -132,7 +134,7 @@ export class GitScanner { 'git --no-pager diff HEAD --name-status -- \':!**/*.assets/*.png\'' : 'git --no-pager diff HEAD --name-status --'; - const result = await this.exec(cmd, { skipLogger: true }); + const result = await this.exec(cmd, { skipLogger: !this.debug }); for (const line of result.stdout.split('\n')) { const parts = line.split(/\s/); const path = parts[parts.length - 1].trim(); @@ -248,7 +250,7 @@ export class GitScanner { } }); - const res = await this.exec('git rev-parse HEAD', { skipLogger: true }); + const res = await this.exec('git rev-parse HEAD', { skipLogger: !this.debug }); return res.stdout.trim(); } @@ -283,7 +285,7 @@ export class GitScanner { } async pushToDir(dir: string) { - await this.exec(`git clone ${this.rootPath} ${dir}`, { skipLogger: true }); + await this.exec(`git clone ${this.rootPath} ${dir}`, { skipLogger: !this.debug }); } async pushBranch(remoteBranch: string, sshParams?: SshParams, localBranch = 'main') { @@ -367,13 +369,17 @@ export class GitScanner { remoteBranch = 'main'; } - await this.exec('git fetch origin', { + await this.exec(`git fetch origin ${remoteBranch}`, { env: { GIT_SSH_COMMAND: sshParams?.privateKeyFile ? `ssh -i ${sanitize(sshParams.privateKeyFile)} -o StrictHostKeyChecking=no -o IdentitiesOnly=yes` : undefined } }); - await this.exec(`git reset --hard refs/remotes/origin/${remoteBranch}`, { + try { + await this.exec('git rebase --abort', {}); + } catch (ignoredError) { /* empty */ } + + await this.exec(`git reset --hard origin/${remoteBranch}`, { env: { GIT_SSH_COMMAND: sshParams?.privateKeyFile ? `ssh -i ${sanitize(sshParams.privateKeyFile)} -o StrictHostKeyChecking=no -o IdentitiesOnly=yes` : undefined } @@ -397,7 +403,7 @@ export class GitScanner { async getRemoteUrl(): Promise { try { - const result = await this.exec('git remote get-url origin', { skipLogger: true }); + const result = await this.exec('git remote get-url origin', { skipLogger: !this.debug }); return result.stdout.trim(); } catch (e) { return null; @@ -406,10 +412,10 @@ export class GitScanner { async setRemoteUrl(url) { try { - await this.exec('git remote rm origin', { skipLogger: true }); + await this.exec('git remote rm origin', { skipLogger: !this.debug }); // eslint-disable-next-line no-empty } catch (ignore) {} - await this.exec(`git remote add origin "${sanitize(url)}"`, { skipLogger: true }); + await this.exec(`git remote add origin "${sanitize(url)}"`, { skipLogger: !this.debug }); } async diff(fileName: string) { @@ -418,7 +424,7 @@ export class GitScanner { } try { - const untrackedList = await this.exec('git -c core.quotepath=off ls-files --others --exclude-standard', { skipLogger: true }); + const untrackedList = await this.exec('git -c core.quotepath=off ls-files --others --exclude-standard', { skipLogger: !this.debug }); const list = untrackedList.stdout.trim().split('\n') .filter(fileName => !!fileName) @@ -443,7 +449,7 @@ export class GitScanner { fileName = fileName.substring(0, fileName.length - '.md'.length) + '.*' + ' ' + fileName.substring(0, fileName.length - '.md'.length) + '.*/*'; } - const result = await this.exec(`git diff --minimal ${sanitize(fileName)}`, { skipLogger: true }); + const result = await this.exec(`git diff --minimal ${sanitize(fileName)}`, { skipLogger: !this.debug }); const retVal = []; @@ -557,7 +563,7 @@ export class GitScanner { try { const result = await this.exec( `git log --source --pretty="commit %H%d\n\nAuthor: %an <%ae>\nDate: %ct\n\n%B\n" ${sanitize(fileName)}`, - { skipLogger: true } + { skipLogger: !this.debug } ); let remoteCommit; @@ -660,7 +666,7 @@ export class GitScanner { } if (!await this.isRepo()) { - await this.exec('git init -b main', { skipLogger: true }); + await this.exec('git init -b main', { skipLogger: !this.debug }); } } @@ -670,7 +676,7 @@ export class GitScanner { async getBranchCommit(branch: string): Promise { try { - const res = await this.exec(`git rev-parse ${branch}`, { skipLogger: true }); + const res = await this.exec(`git rev-parse ${branch}`, { skipLogger: !this.debug }); return res.stdout.trim(); } catch (err) { return null; @@ -678,11 +684,12 @@ export class GitScanner { } async autoCommit() { + this.logger.info('Auto commit'); const dontCommit = new Set(); const toCommit = new Set(); try { - const untrackedList = await this.exec('git -c core.quotepath=off ls-files --others --exclude-standard', { skipLogger: true }); + const untrackedList = await this.exec('git -c core.quotepath=off ls-files --others --exclude-standard', { skipLogger: !this.debug }); const list = untrackedList.stdout.trim().split('\n') .filter(fileName => !!fileName) @@ -702,7 +709,7 @@ export class GitScanner { const childProcess = spawn('git', ['diff', '--minimal', '--ignore-space-change'], - { cwd: this.rootPath, env: {} }); + { cwd: this.rootPath, env: { HOME: process.env.HOME } }); const promise = new Promise((resolve) => { childProcess.on('close', resolve); }); @@ -807,7 +814,8 @@ export class GitScanner { const exitCode = await promise; if (exitCode) { - throw new Error( `subprocess error exit ${exitCode}, ${error}`); + const cmd = 'git ' + ['diff', '--minimal', '--ignore-space-change'].join(' '); + throw new Error( `subprocess (${cmd}) in ${this.rootPath} error exit ${exitCode}, ${error}`); } current = flushCurrent(current); @@ -843,7 +851,7 @@ export class GitScanner { async countAheadBehind(remoteBranch: string) { try { const result = await this.exec(`git rev-list --left-right --count HEAD...origin/${remoteBranch}`, { - skipLogger: true + skipLogger: !this.debug }); const firstLine = result.stdout.split('\n')[0]; @@ -864,7 +872,7 @@ export class GitScanner { let unstaged = 0; try { - const untrackedResult = await this.exec('git status --short --untracked-files', { skipLogger: true }); + const untrackedResult = await this.exec('git status --short --untracked-files', { skipLogger: !this.debug }); for (const line of untrackedResult.stdout.split('\n')) { if (!line.trim()) { continue; @@ -880,7 +888,7 @@ export class GitScanner { } try { - const result = await this.exec('git --no-pager diff HEAD --name-status -- \':!**/*.assets/*.png\'', { skipLogger: true }); + const result = await this.exec('git --no-pager diff HEAD --name-status -- \':!**/*.assets/*.png\'', { skipLogger: !this.debug }); for (const line of result.stdout.split('\n')) { if (line.match(/^A\s/)) { unstaged++; @@ -906,7 +914,7 @@ export class GitScanner { } async removeUntracked() { - const result = await this.exec('git -c core.quotepath=off status', { skipLogger: true }); + const result = await this.exec('git -c core.quotepath=off status', { skipLogger: !this.debug }); let mode = 0; const untracked = []; @@ -949,7 +957,7 @@ export class GitScanner { throw new Error('Forbidden command'); } - const result = await this.exec('git ' + cmd + ' ' + (arg || ''), { skipLogger: true }); + const result = await this.exec('git ' + cmd + ' ' + (arg || ''), { skipLogger: !this.debug }); return { stdout: result.stdout, stderr: result.stderr }; } diff --git a/website/docs/developer-guide.md b/website/docs/developer-guide.md index 398aae39..ddefeac0 100644 --- a/website/docs/developer-guide.md +++ b/website/docs/developer-guide.md @@ -77,6 +77,7 @@ docker run --rm --user=$(id -u):$(getent group docker | cut -d: -f3) -it \ -v ~/workspaces/mieweb/wikigdrive-with-service-account.json:/service_account.json \ -v ~/workspaces/mieweb/wikiGDrive:/usr/src/app \ -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/run/podman/podman.sock:/var/run/podman/podman.sock \ -e VOLUME_DATA=$VOLUME_DATA \ -e VOLUME_PREVIEW=$VOLUME_PREVIEW \ --link zipkin:zipkin \ From 8417837a6e403db0532bdb945f306acef7eabd6a Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Thu, 7 Nov 2024 18:32:41 +0100 Subject: [PATCH 10/16] Add transform single --- .../action/ActionRunnerContainer.ts | 20 ++++++++++++++++++- .../transform/TransformContainer.ts | 2 ++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/containers/action/ActionRunnerContainer.ts b/src/containers/action/ActionRunnerContainer.ts index c3de760b..8face76e 100644 --- a/src/containers/action/ActionRunnerContainer.ts +++ b/src/containers/action/ActionRunnerContainer.ts @@ -49,6 +49,7 @@ export const DEFAULT_WORKFLOW: WorkflowDefinition = { on: { 'internal/sync': 'transform_all', 'transform_all': 'autocommit_render', + 'transform_single': 'autocommit_render', 'internal/branch': 'commit_and_push_branch' }, @@ -58,6 +59,20 @@ export const DEFAULT_WORKFLOW: WorkflowDefinition = { steps: [ { uses: 'internal/transform', + with: { + 'selectedFileId': null + } + } + ] + }, + 'transform_single': { + name: 'Transform Single', + steps: [ + { + uses: 'internal/transform', + with: { + 'selectedFileId': '$wgd.selectedFileId' + } } ] }, @@ -290,7 +305,10 @@ export class ActionRunnerContainer extends Container { let selectedFileId = undefined; try { const payload = JSON.parse(this.params.payload); - selectedFileId = payload.selectedFileId; + + if (step.with?.selectedFileId === '$wgd.selectedFileId') { + selectedFileId = payload.selectedFileId; + } } catch (ignore) { /* empty */ } await action.execute(driveId, this.params.jobId, selectedFileId ? [ selectedFileId ] : [] ); } catch (err) { diff --git a/src/containers/transform/TransformContainer.ts b/src/containers/transform/TransformContainer.ts index 395617a4..bca4f714 100644 --- a/src/containers/transform/TransformContainer.ts +++ b/src/containers/transform/TransformContainer.ts @@ -471,6 +471,8 @@ export class TransformContainer extends Container { return ''; } return retVal; + } else { + this.logger.warn(`In ${fileName} there is a link to ${fullLink} which can't be translated into bookmark link`); } return str; }); From 561e6516b540379d9daae2d7599cef2ab10d62f3 Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Thu, 7 Nov 2024 18:33:02 +0100 Subject: [PATCH 11/16] Fix names --- src/containers/action/DockerContainer.ts | 4 ++-- src/containers/action/PodmanContainer.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/containers/action/DockerContainer.ts b/src/containers/action/DockerContainer.ts index 9a51fc4c..77375cce 100644 --- a/src/containers/action/DockerContainer.ts +++ b/src/containers/action/DockerContainer.ts @@ -97,9 +97,9 @@ export class DockerContainer implements OciContainer { }); } - async putFile(configToml: Uint8Array, remotePath: string) { + async putFile(content: Uint8Array, remotePath: string) { const archive = tarStream.pack(); - archive.entry({ name: remotePath }, configToml); + archive.entry({ name: remotePath }, content); archive.finalize(); const writable = new BufferWritable(); diff --git a/src/containers/action/PodmanContainer.ts b/src/containers/action/PodmanContainer.ts index d799e5e5..cd0bb36b 100644 --- a/src/containers/action/PodmanContainer.ts +++ b/src/containers/action/PodmanContainer.ts @@ -66,9 +66,9 @@ export class PodmanContainer implements OciContainer { }); } - async putFile(configToml: Uint8Array, remotePath: string) { + async putFile(content: Uint8Array, remotePath: string) { const archive = tarStream.pack(); - archive.entry({ name: remotePath }, configToml); + archive.entry({ name: remotePath }, content); archive.finalize(); const writable = new BufferWritable(); From be0e4957ce36970a2fdb3f57ba8c981200190e4a Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Fri, 15 Nov 2024 16:15:41 +0100 Subject: [PATCH 12/16] Fix ownership warning --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 6c99ecd6..186fb81a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,4 +30,8 @@ WORKDIR "/usr/src/app" # Add the GIT_SSH_COMMAND to /etc/profile so that we can debug git issues from the command line RUN echo 'export GIT_SSH_COMMAND="ssh -i \$(pwd | sed s/_transform.*//)/.private/id_rsa"' >> /etc/profile +# Git 2.47: +RUN apk upgrade git --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main +RUN git config --global --add safe.directory /srv/wikigdrive/* + CMD [ "sh", "-c", "wikigdrive --workdir /data server 3000" ] From efad06f239e466647ae60bee7a510798afb57505 Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Fri, 15 Nov 2024 16:44:05 +0100 Subject: [PATCH 13/16] Add hash warning --- src/containers/transform/TransformContainer.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/containers/transform/TransformContainer.ts b/src/containers/transform/TransformContainer.ts index bca4f714..e843f7ef 100644 --- a/src/containers/transform/TransformContainer.ts +++ b/src/containers/transform/TransformContainer.ts @@ -481,11 +481,16 @@ export class TransformContainer extends Container { newContent = newContent.replace(/(gdoc:[A-Z0-9_-]+)(#[^'")\s]*)?/ig, (str: string) => { let fileId = str.substring('gdoc:'.length).replace(/#.*/, ''); let hash = getUrlHash(str) || ''; - if (hash && this.globalHeadersMap[str]) { - const idx = this.globalHeadersMap[str].indexOf('#'); - if (idx >= 0) { - fileId = this.globalHeadersMap[str].substring('gdoc:'.length, idx); - hash = this.globalHeadersMap[str].substring(idx); + if (hash) { + if (this.globalHeadersMap[str]) { + const idx = this.globalHeadersMap[str].indexOf('#'); + if (idx >= 0) { + fileId = this.globalHeadersMap[str].substring('gdoc:'.length, idx); + hash = this.globalHeadersMap[str].substring(idx); + } + } else { + const fullLink = str; + this.logger.warn(`In ${fileName} there is a link to ${fullLink} which can't be translated into bookmark link`); } } const lastLog = this.localLog.findLastFile(fileId); From cce6ecb3847a4973ec3ade439c19600a9c6f8a3a Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Wed, 15 Jan 2025 17:13:08 +0100 Subject: [PATCH 14/16] Fix deploy --- .github/workflows/feat-deploy.yml | 2 +- Dockerfile | 4 ++-- src/containers/action/DockerContainer.ts | 6 ++++-- src/containers/action/OciContainer.ts | 2 +- src/containers/action/PodmanContainer.ts | 5 +++-- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/feat-deploy.yml b/.github/workflows/feat-deploy.yml index fac6cdb9..09fb9807 100644 --- a/.github/workflows/feat-deploy.yml +++ b/.github/workflows/feat-deploy.yml @@ -42,6 +42,6 @@ jobs: - name: Create pull request run: | gh_pr_up() { - gh pr create --draft $* --body "${BODY}" || gh pr edit $* --body "${BODY}" + gh pr edit $* --body "${BODY}" || gh pr create --draft $* --body "${BODY}" } gh_pr_up --base master --title "${BRANCH_NAME}" diff --git a/Dockerfile b/Dockerfile index 186fb81a..949f8761 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ VOLUME /data WORKDIR /usr/src/app RUN apt-get update -RUN apt-get install -yq bash git-lfs openssh-client curl unzip socat podman-remote +RUN apt-get install -yq bash git-lfs openssh-client curl unzip socat podman RUN curl -fsSL https://deno.land/install.sh | DENO_INSTALL=/usr/local sh COPY package.json package-lock.json ./ @@ -31,7 +31,7 @@ WORKDIR "/usr/src/app" RUN echo 'export GIT_SSH_COMMAND="ssh -i \$(pwd | sed s/_transform.*//)/.private/id_rsa"' >> /etc/profile # Git 2.47: -RUN apk upgrade git --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main +#RUN apt upgrade git --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main RUN git config --global --add safe.directory /srv/wikigdrive/* CMD [ "sh", "-c", "wikigdrive --workdir /data server 3000" ] diff --git a/src/containers/action/DockerContainer.ts b/src/containers/action/DockerContainer.ts index 77375cce..ae092779 100644 --- a/src/containers/action/DockerContainer.ts +++ b/src/containers/action/DockerContainer.ts @@ -1,9 +1,11 @@ +import path from 'node:path'; +import {PassThrough, Writable} from 'node:stream'; + import Docker from 'dockerode'; -import path from 'path'; import tarFs from 'tar-fs'; import tarStream from 'tar-stream'; -import {PassThrough, Writable} from 'stream'; import winston from 'winston'; + import {OciContainer} from './OciContainer.ts'; import {BufferWritable} from '../../utils/BufferWritable.ts'; diff --git a/src/containers/action/OciContainer.ts b/src/containers/action/OciContainer.ts index bd2386c9..a62face8 100644 --- a/src/containers/action/OciContainer.ts +++ b/src/containers/action/OciContainer.ts @@ -1,4 +1,4 @@ -import {Writable} from 'stream'; +import {Writable} from 'node:stream'; export interface OciContainer { skipMount: boolean; diff --git a/src/containers/action/PodmanContainer.ts b/src/containers/action/PodmanContainer.ts index cd0bb36b..dead8509 100644 --- a/src/containers/action/PodmanContainer.ts +++ b/src/containers/action/PodmanContainer.ts @@ -1,8 +1,9 @@ +import path from 'node:path'; +import {PassThrough, Writable} from 'node:stream'; + import Docker from 'dockerode'; -import path from 'path'; import tarFs from 'tar-fs'; import tarStream from 'tar-stream'; -import {PassThrough, Writable} from 'stream'; import winston from 'winston'; import {OciContainer} from './OciContainer.ts'; import {BufferWritable} from '../../utils/BufferWritable.ts'; From 7ee1115278b187674ec062828115a501cd2f0b4a Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Thu, 16 Jan 2025 13:31:55 +0100 Subject: [PATCH 15/16] Fix docker for deno --- package-lock.json | 2 ++ src/containers/action/DockerContainer.ts | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3a78d6be..77e3aa5c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -60,6 +60,8 @@ "sharp-phash": "2.2.0", "slugify": "1.6.6", "stream": "^0.0.2", + "tar-fs": "3.0.6", + "tar-stream": "3.1.7", "ts-node": "10.9.2", "typescript": "5.3.3", "winston": "3.8.2", diff --git a/src/containers/action/DockerContainer.ts b/src/containers/action/DockerContainer.ts index ae092779..1d8e2082 100644 --- a/src/containers/action/DockerContainer.ts +++ b/src/containers/action/DockerContainer.ts @@ -1,5 +1,6 @@ import path from 'node:path'; -import {PassThrough, Writable} from 'node:stream'; +import {PassThrough, type Writable} from 'node:stream'; +import type { Buffer } from 'node:buffer'; import Docker from 'dockerode'; import tarFs from 'tar-fs'; @@ -20,7 +21,10 @@ export class DockerContainer implements OciContainer { } static async create(logger: winston.Logger, image: string, env: { [p: string]: string }, repoSubDir: string): Promise { - const dockerEngine = new Docker({socketPath: '/var/run/docker.sock'}); + // https://github.com/apocas/dockerode/issues/747 + // const dockerEngine = new Docker({socketPath: '/var/run/docker.sock'}); + + const dockerEngine = new Docker({ protocol: 'http', host: 'localhost', port: 5000 }); const container = await dockerEngine.createContainer({ Image: image, From 63baec35fefa2aa54d39d8c3d44f6e8f43111682 Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Thu, 16 Jan 2025 15:02:04 +0100 Subject: [PATCH 16/16] Fix schedule trigger --- src/containers/job/JobManagerContainer.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/containers/job/JobManagerContainer.ts b/src/containers/job/JobManagerContainer.ts index afef2652..6ce3563f 100644 --- a/src/containers/job/JobManagerContainer.ts +++ b/src/containers/job/JobManagerContainer.ts @@ -184,9 +184,6 @@ export class JobManagerContainer extends Container { driveJobs.jobs.push(job); break; case 'run_action': - if (driveJobs.jobs.find(subJob => subJob.type === 'run_action' && notCompletedJob(subJob))) { - return; - } { const googleFileSystem = await this.filesService.getSubFileService(driveId, '/'); const userConfigService = new UserConfigService(googleFileSystem); @@ -765,6 +762,13 @@ export class JobManagerContainer extends Container { await this.runAction(driveId, currentJob.id, currentJob.action_id, currentJob.payload, currentJob.user); await this.clearGitCache(driveId); // TODO: check if necessary? + await this.schedule(driveId, { + ...initJob(), + type: 'run_action', + title: 'Run action:', + trigger: currentJob.action_id + }); + this.engine.emit(driveId, 'toasts:added', { title: 'Done: ' + currentJob.title, type: 'run_action:done', @@ -781,8 +785,6 @@ export class JobManagerContainer extends Container { payload: this.params.payload }); throw err; - } finally { - driveJobs.jobs = driveJobs.jobs.filter(removeOldByType('run_action')); } break; case 'git_fetch':