Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: docker-based lensing #367

Draft
wants to merge 5 commits into
base: develop
Choose a base branch
from
Draft

feat: docker-based lensing #367

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
fb60ee5
feat: implement docker-based lensing
themightychris Nov 27, 2024
cd24d3c
chore(deps): bump git-client to 1.9.3
themightychris Dec 9, 2024
5c32999
feat(lens): add HOLO_DEBUG_PERSIST_CONTAINER option
themightychris Dec 9, 2024
eae1fc4
fix: force push to container and relay output
themightychris Dec 10, 2024
9d753f8
fix: pass through DEBUG
themightychris Dec 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
249 changes: 218 additions & 31 deletions lib/Lens.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,10 @@ class Lens extends Configurable {
const config = await super.getConfig();

// process lens configuration
if (!config.package) {
throw new Error(`hololens has no package defined: ${this.name}`);
if (config.package) {
config.command = config.command || 'lens-tree {{ input }}';
}

config.command = config.command || 'lens-tree {{ input }}';

if (config.before) {
config.before =
typeof config.before == 'string'
Expand Down Expand Up @@ -103,7 +101,62 @@ class Lens extends Configurable {
async buildSpec (inputTree) {
const config = await this.getCachedConfig();

if (config.container) {
return this.buildSpecForContainer(inputTree, config);
} else if (config.package) {
return this.buildSpecForHabitatPackage(inputTree, config);
} else {
throw new Error(`hololens has no package or container defined: ${this.name}`);
}
}

async buildSpecForContainer (inputTree, config) {
const { container: containerQuery } = config;

// check if image exists locally first
let imageHash;
try {
const inspectOutput = await Studio.execDocker(['inspect', containerQuery]);
const imageInfo = JSON.parse(inspectOutput)[0];
imageHash = imageInfo.Id;
logger.info(`found local image: ${containerQuery}@${imageHash}`);
} catch (err) {
// image doesn't exist locally or can't be inspected, try pulling
logger.info(`pulling image: ${containerQuery}`);

try {
await Studio.execDocker(['pull', containerQuery], { $relayStdout: true });
const inspectOutput = await Studio.execDocker(['inspect', containerQuery]);
const imageInfo = JSON.parse(inspectOutput)[0];
imageHash = imageInfo.Id;
} catch (err) {
throw new Error(`failed to pull container image ${containerQuery}: ${err.message}`);
}
}

if (!imageHash) {
throw new Error(`failed to get hash for container image ${containerQuery}`);
}

// build spec
const data = {
...config,
container: `${containerQuery.replace(/:.*$/, '')}@${imageHash}`,
input: await inputTree.write(),
output: null,
before: null,
after: null
};

// write spec and return packet
return {
...await SpecObject.write(this.workspace.getRepo(), 'lens', data),
data,
type: 'container'
};
}

async buildSpecForHabitatPackage (inputTree, config) {
// determine current package version
const { package: packageQuery } = config;
const [pkgOrigin, pkgName, pkgVersion, pkgBuild] = packageQuery.split('/');
Expand Down Expand Up @@ -163,14 +216,6 @@ class Lens extends Configurable {
}


// old studio method that might be useful as fallback/debug option
// const setupOutput = await studio.exec('hab', 'pkg', 'install', 'core/hab-plan-build');
// const originOutput = await studio.exec('hab', 'origin', 'key', 'generate', 'holo');
// const buildOutput = await studio.habPkgExec('core/hab-plan-build', 'hab-plan-build', '/src/lenses/compass');
// const studio = await Studio.get(this.workspace.getRepo().gitDir);
// let packageIdent = await studio.getPackage(packageQuery);


// build spec
const data = {
...config,
Expand All @@ -185,21 +230,21 @@ class Lens extends Configurable {
// write spec and return packet
return {
...await SpecObject.write(this.workspace.getRepo(), 'lens', data),
data
data,
type: 'habitat'
};
}

async executeSpec (specHash, options) {
return Lens.executeSpec(specHash, {...options, repo: this.workspace.getRepo()});
async executeSpec (specType, specHash, options) {
return Lens.executeSpec(specType, specHash, {...options, repo: this.workspace.getRepo()});
}

static async executeSpec (specHash, { refresh=false, save=true, repo=null, cacheFrom=null, cacheTo=null }) {
static async executeSpec (specType, specHash, options) {
const { refresh=false, cacheFrom=null, cacheTo=null, save=true } = options;

// load holorepo
if (!repo) {
repo = await Repo.getFromEnvironment();
}

// load holorepo
const repo = options.repo || await Repo.getFromEnvironment();
const git = await repo.getGit();


Expand Down Expand Up @@ -228,9 +273,161 @@ class Lens extends Configurable {
}


// ensure the rest runs inside a studio environment
// execute lens in container or with habitat package:
let lensedTreeHash;
if (specType == 'container') {
lensedTreeHash = await Lens.executeSpecForContainer(repo, specHash);
} else if (specType == 'habitat') {
lensedTreeHash = await Lens.executeSpecForHabitatPackage(repo, specHash);
}

// save ref to accelerate next projection
if (save) {
await git.updateRef(specRef, lensedTreeHash);

if (cacheTo) {
await _cacheResultTo(repo, specRef, cacheTo);
}
}

return lensedTreeHash;
}

static async executeSpecForContainer (repo, specHash) {
const git = await repo.getGit();

// read and parse spec file
const specToml = await git.catFile({ p: true }, specHash);
const {
holospec: {
lens: spec
}
} = TOML.parse(specToml);

// write commit with input tree and spec content
const commitHash = await git.commitTree(spec.input, {
p: [],
m: specToml
});

// extract repository and hash from container string
const containerMatch = spec.container.match(/^.+@sha256:([a-f0-9]{64})$/);
if (!containerMatch) {
throw new Error(`Invalid container format: ${spec.container}`);
}
const [, sha256Hash] = containerMatch;

// create and start container
const persistentDebugContainer = process.env.HOLO_DEBUG_PERSIST_CONTAINER;
let containerId;
try {
if (persistentDebugContainer) {
try {
const containerInfo = await Studio.execDocker(['inspect', persistentDebugContainer]);
const containerState = JSON.parse(containerInfo)[0].State;

if (containerState.Running) {
logger.info(`Found running debug container: ${persistentDebugContainer}`);
containerId = persistentDebugContainer;
}
} catch (error) {
containerId = null;
}
}

// create container
if (!containerId) {
containerId = await Studio.execDocker([
'create',
'-p', '9000:9000',
...(persistentDebugContainer ? ['--name', persistentDebugContainer] : []),
...(process.env.DEBUG ? ['-e', 'DEBUG=1'] : []),
sha256Hash
]);
containerId = containerId.trim();

logger.info('starting container');
await Studio.execDocker(['start', containerId]);
}

// wait for port 9000 to be available
let attempts = 0;
const maxAttempts = 30;
const waitTime = 1000; // 1 second

while (attempts < maxAttempts) {
try {
const containerInfo = await Studio.execDocker(['inspect', containerId]);
const containerState = JSON.parse(containerInfo)[0].State;

if (containerState.Running) {
// check if port 9000 is listening
try {
await Studio.execDocker([
'exec',
containerId,
'nc',
'-z',
'localhost',
'9000'
]);
break;
} catch (err) {
// ignore error and continue waiting
}
}
} catch (err) {
// ignore error and continue waiting
}

await new Promise(resolve => setTimeout(resolve, waitTime));
attempts++;
}

if (attempts >= maxAttempts) {
throw new Error('Timeout waiting for git server to be ready');
}

// push commit to git server
logger.info(`pushing and executing job: ${commitHash}`);
await git.push(`http://localhost:9000/`, `${commitHash}:refs/heads/lens-input`, {
force: true,
$wait: true,
$onStderr: (line) => process.stderr.write(`\x1b[90m${line}\x1b[0m\n`)
});

// fetch and verify output commit
const outputRef = `refs/lens-jobs/${specHash}`;
logger.info('fetching result');
await git.fetch('http://localhost:9000/', `+refs/heads/lens-input:${outputRef}`);

// verify the output commit's parent matches our input commit
const outputParent = await git.revParse(`${outputRef}^`);
if (outputParent !== commitHash) {
throw new Error(`Output commit parent ${outputParent} does not match input commit ${commitHash}`);
}

return await git.getTreeHash(outputRef);

} finally {
// cleanup
if (containerId && !persistentDebugContainer) {
try {
await Studio.execDocker(['stop', containerId]);
await Studio.execDocker(['rm', containerId]);
} catch (err) {
logger.warn(`Failed to cleanup container: ${err.message}`);
}
}
}
}

static async executeSpecForHabitatPackage (repo, specHash) {
const git = await repo.getGit();

let lensedTreeHash;

// ensure the rest runs inside a studio environment
if (!await Studio.isEnvironmentStudio()) {
const studio = await Studio.get(repo.gitDir);
lensedTreeHash = await studio.holoLensExec(specHash);
Expand Down Expand Up @@ -307,16 +504,6 @@ class Lens extends Configurable {
}


// save ref to accelerate next projection
if (save) {
await git.updateRef(specRef, lensedTreeHash);

if (cacheTo) {
await _cacheResultTo(repo, specRef, cacheTo);
}
}


// return tree hash
return lensedTreeHash;
}
Expand Down
4 changes: 2 additions & 2 deletions lib/Projection.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,11 +162,11 @@ class Projection {

// build tree of matching files to input to lens
logger.info(`building input tree for lens ${lens.name} from ${inputRoot == '.' ? '' : (path.join(inputRoot, '.')+'/')}{${inputFiles}}`);
const { hash: specHash } = await lens.buildSpec(await lens.buildInputTree(this.output.root));
const { hash: specHash, type: specType } = await lens.buildSpec(await lens.buildInputTree(this.output.root));


// check for existing output tree
const outputTreeHash = await lens.executeSpec(specHash, { cacheFrom, cacheTo });
const outputTreeHash = await lens.executeSpec(specType, specHash, { cacheFrom, cacheTo });


// verify output
Expand Down
Loading
Loading