From d8ecff762ee47716e564f6fc5a18a1d6a59e786f Mon Sep 17 00:00:00 2001 From: aavarghese Date: Wed, 13 Nov 2024 17:40:40 -0500 Subject: [PATCH] feat: ibmcloud be to use local setup rather than k8 kind cluster Signed-off-by: aavarghese --- cmd/subcommands/component.go | 1 + cmd/subcommands/component/run-locally.go | 42 ++++++ pkg/be/backend.go | 3 + pkg/be/ibmcloud/create.go | 181 +++++++++++++++++------ pkg/be/ibmcloud/image.go | 36 +++++ pkg/be/kubernetes/image.go | 12 ++ pkg/be/local/image.go | 12 ++ pkg/be/local/shell/job.go | 5 +- pkg/be/local/shell/spawn.go | 8 +- pkg/be/local/spawn.go | 6 +- pkg/be/local/up.go | 8 +- pkg/boot/up.go | 9 ++ pkg/runtime/run-locally.go | 28 ++++ 13 files changed, 291 insertions(+), 60 deletions(-) create mode 100644 cmd/subcommands/component/run-locally.go create mode 100644 pkg/be/ibmcloud/image.go create mode 100644 pkg/be/kubernetes/image.go create mode 100644 pkg/be/local/image.go create mode 100644 pkg/runtime/run-locally.go diff --git a/cmd/subcommands/component.go b/cmd/subcommands/component.go index aaca1826..934e6314 100644 --- a/cmd/subcommands/component.go +++ b/cmd/subcommands/component.go @@ -17,4 +17,5 @@ func init() { cmd.AddCommand(component.Minio()) cmd.AddCommand(component.Worker()) cmd.AddCommand(component.WorkStealer()) + cmd.AddCommand(component.RunLocally()) } diff --git a/cmd/subcommands/component/run-locally.go b/cmd/subcommands/component/run-locally.go new file mode 100644 index 00000000..1b262d28 --- /dev/null +++ b/cmd/subcommands/component/run-locally.go @@ -0,0 +1,42 @@ +package component + +import ( + "context" + + "github.com/spf13/cobra" + "lunchpail.io/cmd/options" + "lunchpail.io/pkg/build" + "lunchpail.io/pkg/runtime" +) + +type RunLocallyOptions struct { + Component string + LLIR string + build.LogOptions +} + +func AddRunLocallyOptions(cmd *cobra.Command) *RunLocallyOptions { + options := RunLocallyOptions{} + cmd.Flags().StringVarP(&options.Component, "component", "", "", "") + cmd.Flags().StringVar(&options.LLIR, "llir", "", "") + cmd.MarkFlagRequired("component") + cmd.MarkFlagRequired("llir") + return &options +} + +func RunLocally() *cobra.Command { + cmd := &cobra.Command{ + Use: "run-locally", + Short: "Commands for running a component locally", + Long: "Commands for running a component locally", + } + + runOpts := AddRunLocallyOptions(cmd) + options.AddLogOptions(cmd) + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + return runtime.RunLocally(context.Background(), runOpts.Component, runOpts.LLIR, runOpts.LogOptions) + } + + return cmd +} diff --git a/pkg/be/backend.go b/pkg/be/backend.go index c8ee5351..12acd254 100644 --- a/pkg/be/backend.go +++ b/pkg/be/backend.go @@ -38,4 +38,7 @@ type Backend interface { // Return a streamer Streamer(ctx context.Context, run queue.RunContext) streamer.Streamer + + // Build any images(s) needed for Up + CreateImage(ctx context.Context, linked llir.LLIR, opts llir.Options, destroy bool) (string, error) } diff --git a/pkg/be/ibmcloud/create.go b/pkg/be/ibmcloud/create.go index 6c9ba9cb..65815f17 100644 --- a/pkg/be/ibmcloud/create.go +++ b/pkg/be/ibmcloud/create.go @@ -2,10 +2,12 @@ package ibmcloud import ( "context" + "encoding/json" "errors" "fmt" "math" "net/http" + "os" "os/exec" "strconv" "strings" @@ -17,9 +19,8 @@ import ( "github.com/elotl/cloud-init/config" "golang.org/x/crypto/ssh" "golang.org/x/sync/errgroup" + q "lunchpail.io/pkg/ir/queue" - "lunchpail.io/pkg/be/kubernetes" - "lunchpail.io/pkg/be/kubernetes/common" "lunchpail.io/pkg/ir/llir" "lunchpail.io/pkg/lunchpail" "lunchpail.io/pkg/util" @@ -51,7 +52,7 @@ func (i *intCounter) inc() { i.lock.Unlock() } -func createInstance(vpcService *vpcv1.VpcV1, name string, ir llir.LLIR, c llir.Component, resourceGroupID string, vpcID string, keyID string, zone string, profile string, subnetID string, secGroupID string, imageID string, namespace string, copts llir.Options) (*vpcv1.Instance, error) { +func createInstance(vpcService *vpcv1.VpcV1, name string, resourceGroupID string, vpcID string, keyID string, zone string, profile string, subnetID string, secGroupID string, imageID string, namespace string, copts llir.Options, cc *config.CloudConfig) (*vpcv1.Instance, error) { networkInterfacePrototypeModel := &vpcv1.NetworkInterfacePrototype{ Name: &name, Subnet: &vpcv1.SubnetIdentityByID{ @@ -62,29 +63,6 @@ func createInstance(vpcService *vpcv1.VpcV1, name string, ir llir.LLIR, c llir.C }}, } - // TODO pass through actual Cli Options? - opts := common.Options{Options: copts} - - appYamlString, err := kubernetes.MarshalComponentAsStandalone(ir, c, namespace, opts) - if err != nil { - return nil, fmt.Errorf("failed to marshall yaml: %v", err) - } - cc := &config.CloudConfig{ - WriteFiles: []config.File{ - { - Path: "/app.yaml", - Content: appYamlString, - Owner: "root:root", - RawFilePermissions: "0644", - }}, - RunCmd: []string{"sleep 10", //Minimum of 10 seconds needed for cluster to be able to run `apply` - "while ! kind get clusters | grep lunchpail; do sleep 2; done", - "echo 'Kind cluster is ready'", - "env HOME=/root kubectl create ns " + namespace, - "n=0; until [ $n -ge 60 ]; do env HOME=/root kubectl get serviceaccount default -o name -n " + namespace + " && break; n=$((n + 1)); sleep 1; done", - "env HOME=/root kubectl create -f /app.yaml -n " + namespace}, - } - instancePrototypeModel := &vpcv1.InstancePrototypeInstanceByImage{ Name: &name, ResourceGroup: &vpcv1.ResourceGroupIdentity{ @@ -273,50 +251,131 @@ func createVPC(vpcService *vpcv1.VpcV1, name string, appName string, resourceGro return *vpc.ID, nil } -func createAndInitVM(ctx context.Context, vpcService *vpcv1.VpcV1, name string, ir llir.LLIR, resourceGroupID string, keyType string, publicKey string, zone string, profile string, imageID string, namespace string, opts llir.Options) error { +func createImage(vpcService *vpcv1.VpcV1, name string, resourceGroupID string, vmID string) (string, error) { + options := &vpcv1.CreateImageOptions{ + ImagePrototype: &vpcv1.ImagePrototype{ + Name: &name, + ResourceGroup: &vpcv1.ResourceGroupIdentity{ + ID: &resourceGroupID, + }, + SourceVolume: &vpcv1.VolumeIdentityByID{ + ID: &vmID, + }, + }, + } + image, response, err := vpcService.CreateImage(options) + if err != nil { + return "", fmt.Errorf("failed to create an Image: %v and the response is: %s", err, response) + } + return *image.ID, nil +} + +func createResources(ctx context.Context, vpcService *vpcv1.VpcV1, name string, ir llir.LLIR, resourceGroupID string, keyType string, publicKey string, zone string, profile string, imageID string, namespace string, opts llir.Options) (string, error) { + var instanceID string t1s := time.Now() vpcID, err := createVPC(vpcService, name, ir.AppName, resourceGroupID) if err != nil { - return err + return "", err } t1e := time.Now() t2s := t1e keyID, err := createSSHKey(vpcService, name, resourceGroupID, keyType, publicKey) if err != nil { - return err + return "", err } t2e := time.Now() t3s := t2e subnetID, err := createSubnet(vpcService, name, resourceGroupID, vpcID, zone) if err != nil { - return err + return "", err } t3e := time.Now() t4s := t3e secGroupID, err := createSecurityGroup(vpcService, name, resourceGroupID, vpcID) if err != nil { - return err + return "", err } t4e := time.Now() t5s := t4e if err = createSecurityGroupRule(vpcService, secGroupID); err != nil { - return err + return "", err } t5e := time.Now() - group, _ := errgroup.WithContext(ctx) t6s := time.Now() - // One Component for WorkStealer, one for Dispatcher, and each per WorkerPool + if len(ir.Components) > 0 { + if err = createVMForComponents(ctx, vpcService, name, ir, resourceGroupID, zone, profile, imageID, namespace, vpcID, keyID, subnetID, secGroupID, opts); err != nil { + return "", err + } + } else { + instanceID, err = createVMForImage(vpcService, name, resourceGroupID, zone, profile, imageID, namespace, vpcID, keyID, subnetID, secGroupID, opts) + if err != nil { + return "", err + } + } + t6e := time.Now() + + if opts.Log.Verbose { + fmt.Fprintf(os.Stderr, "Setup done %s\n", util.RelTime(t1s, t6e)) + fmt.Fprintf(os.Stderr, " - VPC %s\n", util.RelTime(t1s, t1e)) + fmt.Fprintf(os.Stderr, " - SSH %s\n", util.RelTime(t2s, t2e)) + fmt.Fprintf(os.Stderr, " - Subnet %s\n", util.RelTime(t3s, t3e)) + fmt.Fprintf(os.Stderr, " - SecurityGroup %s\n", util.RelTime(t4s, t4e)) + fmt.Fprintf(os.Stderr, " - SecurityGroupRule %s\n", util.RelTime(t5s, t5e)) + fmt.Fprintf(os.Stderr, " - VMs %s\n", util.RelTime(t6s, t6e)) + } + return instanceID, nil +} + +func createVMForComponents(ctx context.Context, vpcService *vpcv1.VpcV1, name string, ir llir.LLIR, resourceGroupID string, zone string, profile string, imageID string, namespace string, vpcID string, keyID string, subnetID string, secGroupID string, opts llir.Options) error { + group, _ := errgroup.WithContext(ctx) + var verboseFlag string poolCount := intCounter{} for _, c := range ir.Components { instanceName := name + "-" + string(c.C()) + if opts.Log.Verbose { + fmt.Fprintf(os.Stderr, "Creating VM %s\n", instanceName) + } + + component, err := json.Marshal(c) + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + return err + } + + llir, err := json.Marshal(ir) + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + return err + } + + if opts.Log.Verbose { + verboseFlag = "--verbose" + } + //TODO - find executable and run it for each component!!!!! + cc := &config.CloudConfig{ + WriteFiles: []config.File{ + { + Path: "/app.yaml", + Content: "", + Owner: "root:root", + RawFilePermissions: "0644", + }}, + RunCmd: []string{"curl https://dl.min.io/client/mc/release/linux-arm64/mc --create-dirs -o $HOME/minio-binaries/mc", + "chmod +x $HOME/minio-binaries/mc", + "export PATH=$PATH:$HOME/minio-binaries/", + "mc alias set myminio " + ir.Context.Queue.Endpoint + " " + ir.Context.Queue.AccessKey + " " + ir.Context.Queue.SecretKey, //Todo + "mc get " + ir.Context.Run.AsFile(q.Blobs) + " $HOME/lunchpail", //set path //use mc client to download binary + "$HOME/lunchpail component run-locally --component " + string(component) + " --llir " + string(llir) + " " + verboseFlag}, + } + group.Go(func() error { if c.C() == lunchpail.DispatcherComponent || c.C() == lunchpail.WorkStealerComponent { - instance, err := createInstance(vpcService, instanceName, ir, c, resourceGroupID, vpcID, keyID, zone, profile, subnetID, secGroupID, imageID, namespace, opts) + instance, err := createInstance(vpcService, instanceName, resourceGroupID, vpcID, keyID, zone, profile, subnetID, secGroupID, imageID, namespace, opts, cc) if err != nil { return err } @@ -350,7 +409,7 @@ func createAndInitVM(ctx context.Context, vpcService *vpcv1.VpcV1, name string, for i := 0; i < numInstances; i++ { workerName := poolName + "-" + strconv.Itoa(i) //multiple worker instances c = c.SetWorkers(int(parallelism[i])) - instance, err := createInstance(vpcService, workerName, ir, c, resourceGroupID, vpcID, keyID, zone, profile, subnetID, secGroupID, imageID, namespace, opts) + instance, err := createInstance(vpcService, workerName, resourceGroupID, vpcID, keyID, zone, profile, subnetID, secGroupID, imageID, namespace, opts, cc) if err != nil { return err } @@ -378,18 +437,48 @@ func createAndInitVM(ctx context.Context, vpcService *vpcv1.VpcV1, name string, if err := group.Wait(); err != nil { return err } - t6e := time.Now() - - fmt.Printf("Setup done %s\n", util.RelTime(t1s, t6e)) - fmt.Printf(" - VPC %s\n", util.RelTime(t1s, t1e)) - fmt.Printf(" - SSH %s\n", util.RelTime(t2s, t2e)) - fmt.Printf(" - Subnet %s\n", util.RelTime(t3s, t3e)) - fmt.Printf(" - SecurityGroup %s\n", util.RelTime(t4s, t4e)) - fmt.Printf(" - SecurityGroupRule %s\n", util.RelTime(t5s, t5e)) - fmt.Printf(" - VMs %s\n", util.RelTime(t6s, t6e)) return nil } +func createVMForImage(vpcService *vpcv1.VpcV1, name string, resourceGroupID string, zone string, profile string, imageID string, namespace string, vpcID string, keyID string, subnetID string, secGroupID string, opts llir.Options) (string, error) { + if opts.Log.Verbose { + fmt.Printf("Creating VM %s for custom image creations\n", name) + } + var rclone string + //TODO - download executable from S3 and store binary in image!!!!! + cc := &config.CloudConfig{ + WriteFiles: []config.File{ + { + Path: "/rclone.conf", + Content: rclone, + Owner: "root:root", + RawFilePermissions: "0644", + }}, + RunCmd: []string{"sleep 10", //Minimum of 10 seconds needed for cluster to be able to run `apply` + ""}, + } + instance, err := createInstance(vpcService, name, resourceGroupID, vpcID, keyID, zone, profile, subnetID, secGroupID, imageID, namespace, opts, cc) + if err != nil { + return "", err + } + + floatingIPID, err := createFloatingIP(vpcService, name, resourceGroupID, zone) + if err != nil { + return "", err + } + + options := &vpcv1.AddInstanceNetworkInterfaceFloatingIPOptions{ + ID: &floatingIPID, + InstanceID: instance.ID, + NetworkInterfaceID: instance.PrimaryNetworkInterface.ID, + } + _, response, err := vpcService.AddInstanceNetworkInterfaceFloatingIP(options) + if err != nil { + return "", fmt.Errorf("failed to add floating IP to network interface: %v and the response is: %s", err, response) + } + return *instance.ID, err +} + func (backend Backend) SetAction(ctx context.Context, opts llir.Options, ir llir.LLIR, action Action) error { runname := ir.RunName() @@ -406,7 +495,7 @@ func (backend Backend) SetAction(ctx context.Context, opts llir.Options, ir llir } zone = randomZone } - if err := createAndInitVM(ctx, backend.vpcService, runname, ir, backend.config.ResourceGroup.GUID, backend.sshKeyType, backend.sshPublicKey, zone, opts.Profile, opts.ImageID, backend.namespace, opts); err != nil { + if _, err := createResources(ctx, backend.vpcService, runname, ir, backend.config.ResourceGroup.GUID, backend.sshKeyType, backend.sshPublicKey, zone, opts.Profile, opts.ImageID, backend.namespace, opts); err != nil { return err } } diff --git a/pkg/be/ibmcloud/image.go b/pkg/be/ibmcloud/image.go new file mode 100644 index 00000000..2441e6bf --- /dev/null +++ b/pkg/be/ibmcloud/image.go @@ -0,0 +1,36 @@ +package ibmcloud + +import ( + "context" + + "lunchpail.io/pkg/ir/llir" +) + +func (backend Backend) CreateImage(ctx context.Context, ir llir.LLIR, opts llir.Options, destroy bool) (string, error) { + runname := ir.RunName() + + zone := opts.Zone //command line zone value + if zone == "" { //random zone value using config + randomZone, err := getRandomizedZone(backend.config, backend.vpcService) //Todo: spread among random zones with a subnet in each zone + if err != nil { + return "", err + } + zone = randomZone + } + instanceID, err := createResources(ctx, backend.vpcService, runname, ir, backend.config.ResourceGroup.GUID, backend.sshKeyType, backend.sshPublicKey, zone, opts.Profile, opts.ImageID, backend.namespace, opts) + if err != nil { + return "", err + } + + imageID, err := createImage(backend.vpcService, runname, backend.config.ResourceGroup.GUID, instanceID) + if err != nil { + return "", err + } + + if destroy { + if err := stopOrDeleteVM(backend.vpcService, runname, backend.config.ResourceGroup.GUID, true); err != nil { + return "", err + } + } + return imageID, nil +} diff --git a/pkg/be/kubernetes/image.go b/pkg/be/kubernetes/image.go new file mode 100644 index 00000000..22ff1049 --- /dev/null +++ b/pkg/be/kubernetes/image.go @@ -0,0 +1,12 @@ +package kubernetes + +import ( + "context" + "fmt" + + "lunchpail.io/pkg/ir/llir" +) + +func (backend Backend) CreateImage(ctx context.Context, ir llir.LLIR, opts llir.Options, destroy bool) (string, error) { + return "", fmt.Errorf("Unsupported operation: 'CreateImage'") +} diff --git a/pkg/be/local/image.go b/pkg/be/local/image.go new file mode 100644 index 00000000..4737ff34 --- /dev/null +++ b/pkg/be/local/image.go @@ -0,0 +1,12 @@ +package local + +import ( + "context" + "fmt" + + "lunchpail.io/pkg/ir/llir" +) + +func (backend Backend) CreateImage(ctx context.Context, ir llir.LLIR, opts llir.Options, destroy bool) (string, error) { + return "", fmt.Errorf("Unsupported operation: 'CreateImage'") +} diff --git a/pkg/be/local/shell/job.go b/pkg/be/local/shell/job.go index 4791f70f..7f1136b3 100644 --- a/pkg/be/local/shell/job.go +++ b/pkg/be/local/shell/job.go @@ -11,16 +11,15 @@ import ( ) // Run the component as a "job", with multiple workers -func SpawnJob(ctx context.Context, c llir.ShellComponent, ir llir.LLIR, logdir string, opts build.LogOptions) error { +func SpawnJob(ctx context.Context, c llir.ShellComponent, ir llir.LLIR, opts build.LogOptions) error { if c.InitialWorkers < 1 { return fmt.Errorf("Invalid worker count %d for %v", c.InitialWorkers, c.C()) } - group, jobCtx := errgroup.WithContext(ctx) for workerIdx := range c.InitialWorkers { group.Go(func() error { - return Spawn(jobCtx, c.WithInstanceName(fmt.Sprintf("w%d", workerIdx)), ir, logdir, opts) + return Spawn(jobCtx, c.WithInstanceName(fmt.Sprintf("w%d", workerIdx)), ir, opts) }) } diff --git a/pkg/be/local/shell/spawn.go b/pkg/be/local/shell/spawn.go index 794c2554..311fa9fa 100644 --- a/pkg/be/local/shell/spawn.go +++ b/pkg/be/local/shell/spawn.go @@ -17,7 +17,7 @@ import ( "lunchpail.io/pkg/ir/llir" ) -func Spawn(ctx context.Context, c llir.ShellComponent, ir llir.LLIR, logdir string, opts build.LogOptions) error { +func Spawn(ctx context.Context, c llir.ShellComponent, ir llir.LLIR, opts build.LogOptions) error { pidfile, err := files.Pidfile(ir.Context.Run, c.InstanceName, c.C(), true) if err != nil { return err @@ -29,6 +29,12 @@ func Spawn(ctx context.Context, c llir.ShellComponent, ir llir.LLIR, logdir stri } defer os.RemoveAll(workdir) + // This is where component logs will go + logdir, err := files.LogDir(ir.Context.Run, true) + if err != nil { + return err + } + // tee command output to the logdir instance := strings.Replace(strings.Replace(c.InstanceName, ir.RunName(), "", 1), "--", "-", 1) logfile := files.LogFileForComponent(c.C()) diff --git a/pkg/be/local/spawn.go b/pkg/be/local/spawn.go index c1351767..6917f17b 100644 --- a/pkg/be/local/spawn.go +++ b/pkg/be/local/spawn.go @@ -9,13 +9,13 @@ import ( "lunchpail.io/pkg/ir/llir" ) -func (backend Backend) spawn(ctx context.Context, c llir.Component, ir llir.LLIR, logdir string, opts build.LogOptions) error { +func (backend Backend) spawn(ctx context.Context, c llir.Component, ir llir.LLIR, opts build.LogOptions) error { switch cc := c.(type) { case llir.ShellComponent: if cc.RunAsJob { - return shell.SpawnJob(ctx, cc, ir, logdir, opts) + return shell.SpawnJob(ctx, cc, ir, opts) } else { - return shell.Spawn(ctx, cc, ir, logdir, opts) + return shell.Spawn(ctx, cc, ir, opts) } } diff --git a/pkg/be/local/up.go b/pkg/be/local/up.go index ece362a2..e6290d10 100644 --- a/pkg/be/local/up.go +++ b/pkg/be/local/up.go @@ -26,12 +26,6 @@ func (backend Backend) Up(octx context.Context, ir llir.LLIR, opts llir.Options, } } - // This is where component logs will go - logdir, err := files.LogDir(ir.Context.Run, true) - if err != nil { - return err - } - // Write a pid file to indicate the pid of this process if pidfile, err := files.PidfileForMain(ir.Context.Run); err != nil { return err @@ -47,7 +41,7 @@ func (backend Backend) Up(octx context.Context, ir llir.LLIR, opts llir.Options, // Launch each of the components group, ctx := errgroup.WithContext(octx) for _, c := range ir.Components { - group.Go(func() error { return backend.spawn(ctx, c, ir, logdir, *opts.Log) }) + group.Go(func() error { return backend.spawn(ctx, c, ir, *opts.Log) }) } // Indicate that we are off to the races diff --git a/pkg/boot/up.go b/pkg/boot/up.go index a2ba423e..8eb29512 100644 --- a/pkg/boot/up.go +++ b/pkg/boot/up.go @@ -228,6 +228,15 @@ func upLLIR(ctx context.Context, backend be.Backend, ir llir.LLIR, opts UpOption fmt.Fprintln(os.Stderr, err) } }() + + /* if opts.BuildOptions.Target.Platform == target.IBMCloud { + //use the uploaded executable to create an IBM Cloud custom image for VPC using a VSI boot volume + imageID, err := backend.CreateImage(cancellable, ir, opts.BuildOptions, true) // destroys resources after image creation TODO: reuse resources on Up + if err != nil { + return err + } + opts.BuildOptions.ImageID = imageID + } */ } defer cancel() diff --git a/pkg/runtime/run-locally.go b/pkg/runtime/run-locally.go new file mode 100644 index 00000000..0b9c2ee2 --- /dev/null +++ b/pkg/runtime/run-locally.go @@ -0,0 +1,28 @@ +package runtime + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "lunchpail.io/pkg/be/local/shell" + "lunchpail.io/pkg/build" + "lunchpail.io/pkg/ir/llir" +) + +func RunLocally(ctx context.Context, component string, lowerir string, opts build.LogOptions) error { + var c llir.ShellComponent + var ir llir.LLIR + + if err := json.Unmarshal([]byte(component), &c); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + return err + } + if err := json.Unmarshal([]byte(lowerir), &ir); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + return err + } + + return shell.Spawn(ctx, c, ir, opts) +}