Skip to content

Commit

Permalink
feat: ibmcloud be to use local setup rather than k8 kind cluster
Browse files Browse the repository at this point in the history
Signed-off-by: aavarghese <[email protected]>
  • Loading branch information
aavarghese committed Nov 20, 2024
1 parent e71352b commit d8ecff7
Show file tree
Hide file tree
Showing 13 changed files with 291 additions and 60 deletions.
1 change: 1 addition & 0 deletions cmd/subcommands/component.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,5 @@ func init() {
cmd.AddCommand(component.Minio())
cmd.AddCommand(component.Worker())
cmd.AddCommand(component.WorkStealer())
cmd.AddCommand(component.RunLocally())
}
42 changes: 42 additions & 0 deletions cmd/subcommands/component/run-locally.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package component

import (
"context"

"github.com/spf13/cobra"
"lunchpail.io/cmd/options"
"lunchpail.io/pkg/build"
"lunchpail.io/pkg/runtime"
)

type RunLocallyOptions struct {
Component string
LLIR string
build.LogOptions
}

func AddRunLocallyOptions(cmd *cobra.Command) *RunLocallyOptions {
options := RunLocallyOptions{}
cmd.Flags().StringVarP(&options.Component, "component", "", "", "")
cmd.Flags().StringVar(&options.LLIR, "llir", "", "")
cmd.MarkFlagRequired("component")
cmd.MarkFlagRequired("llir")
return &options
}

func RunLocally() *cobra.Command {
cmd := &cobra.Command{
Use: "run-locally",
Short: "Commands for running a component locally",
Long: "Commands for running a component locally",
}

runOpts := AddRunLocallyOptions(cmd)
options.AddLogOptions(cmd)

cmd.RunE = func(cmd *cobra.Command, args []string) error {
return runtime.RunLocally(context.Background(), runOpts.Component, runOpts.LLIR, runOpts.LogOptions)
}

return cmd
}
3 changes: 3 additions & 0 deletions pkg/be/backend.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,7 @@ type Backend interface {

// Return a streamer
Streamer(ctx context.Context, run queue.RunContext) streamer.Streamer

// Build any images(s) needed for Up
CreateImage(ctx context.Context, linked llir.LLIR, opts llir.Options, destroy bool) (string, error)
}
181 changes: 135 additions & 46 deletions pkg/be/ibmcloud/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@ package ibmcloud

import (
"context"
"encoding/json"
"errors"
"fmt"
"math"
"net/http"
"os"
"os/exec"
"strconv"
"strings"
Expand All @@ -17,9 +19,8 @@ import (
"github.com/elotl/cloud-init/config"
"golang.org/x/crypto/ssh"
"golang.org/x/sync/errgroup"
q "lunchpail.io/pkg/ir/queue"

"lunchpail.io/pkg/be/kubernetes"
"lunchpail.io/pkg/be/kubernetes/common"
"lunchpail.io/pkg/ir/llir"
"lunchpail.io/pkg/lunchpail"
"lunchpail.io/pkg/util"
Expand Down Expand Up @@ -51,7 +52,7 @@ func (i *intCounter) inc() {
i.lock.Unlock()
}

func createInstance(vpcService *vpcv1.VpcV1, name string, ir llir.LLIR, c llir.Component, resourceGroupID string, vpcID string, keyID string, zone string, profile string, subnetID string, secGroupID string, imageID string, namespace string, copts llir.Options) (*vpcv1.Instance, error) {
func createInstance(vpcService *vpcv1.VpcV1, name string, resourceGroupID string, vpcID string, keyID string, zone string, profile string, subnetID string, secGroupID string, imageID string, namespace string, copts llir.Options, cc *config.CloudConfig) (*vpcv1.Instance, error) {
networkInterfacePrototypeModel := &vpcv1.NetworkInterfacePrototype{
Name: &name,
Subnet: &vpcv1.SubnetIdentityByID{
Expand All @@ -62,29 +63,6 @@ func createInstance(vpcService *vpcv1.VpcV1, name string, ir llir.LLIR, c llir.C
}},
}

// TODO pass through actual Cli Options?
opts := common.Options{Options: copts}

appYamlString, err := kubernetes.MarshalComponentAsStandalone(ir, c, namespace, opts)
if err != nil {
return nil, fmt.Errorf("failed to marshall yaml: %v", err)
}
cc := &config.CloudConfig{
WriteFiles: []config.File{
{
Path: "/app.yaml",
Content: appYamlString,
Owner: "root:root",
RawFilePermissions: "0644",
}},
RunCmd: []string{"sleep 10", //Minimum of 10 seconds needed for cluster to be able to run `apply`
"while ! kind get clusters | grep lunchpail; do sleep 2; done",
"echo 'Kind cluster is ready'",
"env HOME=/root kubectl create ns " + namespace,
"n=0; until [ $n -ge 60 ]; do env HOME=/root kubectl get serviceaccount default -o name -n " + namespace + " && break; n=$((n + 1)); sleep 1; done",
"env HOME=/root kubectl create -f /app.yaml -n " + namespace},
}

instancePrototypeModel := &vpcv1.InstancePrototypeInstanceByImage{
Name: &name,
ResourceGroup: &vpcv1.ResourceGroupIdentity{
Expand Down Expand Up @@ -273,50 +251,131 @@ func createVPC(vpcService *vpcv1.VpcV1, name string, appName string, resourceGro
return *vpc.ID, nil
}

func createAndInitVM(ctx context.Context, vpcService *vpcv1.VpcV1, name string, ir llir.LLIR, resourceGroupID string, keyType string, publicKey string, zone string, profile string, imageID string, namespace string, opts llir.Options) error {
func createImage(vpcService *vpcv1.VpcV1, name string, resourceGroupID string, vmID string) (string, error) {
options := &vpcv1.CreateImageOptions{
ImagePrototype: &vpcv1.ImagePrototype{
Name: &name,
ResourceGroup: &vpcv1.ResourceGroupIdentity{
ID: &resourceGroupID,
},
SourceVolume: &vpcv1.VolumeIdentityByID{
ID: &vmID,
},
},
}
image, response, err := vpcService.CreateImage(options)
if err != nil {
return "", fmt.Errorf("failed to create an Image: %v and the response is: %s", err, response)
}
return *image.ID, nil
}

func createResources(ctx context.Context, vpcService *vpcv1.VpcV1, name string, ir llir.LLIR, resourceGroupID string, keyType string, publicKey string, zone string, profile string, imageID string, namespace string, opts llir.Options) (string, error) {
var instanceID string
t1s := time.Now()
vpcID, err := createVPC(vpcService, name, ir.AppName, resourceGroupID)
if err != nil {
return err
return "", err
}
t1e := time.Now()

t2s := t1e
keyID, err := createSSHKey(vpcService, name, resourceGroupID, keyType, publicKey)
if err != nil {
return err
return "", err
}
t2e := time.Now()

t3s := t2e
subnetID, err := createSubnet(vpcService, name, resourceGroupID, vpcID, zone)
if err != nil {
return err
return "", err
}
t3e := time.Now()

t4s := t3e
secGroupID, err := createSecurityGroup(vpcService, name, resourceGroupID, vpcID)
if err != nil {
return err
return "", err
}
t4e := time.Now()

t5s := t4e
if err = createSecurityGroupRule(vpcService, secGroupID); err != nil {
return err
return "", err
}
t5e := time.Now()

group, _ := errgroup.WithContext(ctx)
t6s := time.Now()
// One Component for WorkStealer, one for Dispatcher, and each per WorkerPool
if len(ir.Components) > 0 {
if err = createVMForComponents(ctx, vpcService, name, ir, resourceGroupID, zone, profile, imageID, namespace, vpcID, keyID, subnetID, secGroupID, opts); err != nil {
return "", err
}
} else {
instanceID, err = createVMForImage(vpcService, name, resourceGroupID, zone, profile, imageID, namespace, vpcID, keyID, subnetID, secGroupID, opts)
if err != nil {
return "", err
}
}
t6e := time.Now()

if opts.Log.Verbose {
fmt.Fprintf(os.Stderr, "Setup done %s\n", util.RelTime(t1s, t6e))
fmt.Fprintf(os.Stderr, " - VPC %s\n", util.RelTime(t1s, t1e))
fmt.Fprintf(os.Stderr, " - SSH %s\n", util.RelTime(t2s, t2e))
fmt.Fprintf(os.Stderr, " - Subnet %s\n", util.RelTime(t3s, t3e))
fmt.Fprintf(os.Stderr, " - SecurityGroup %s\n", util.RelTime(t4s, t4e))
fmt.Fprintf(os.Stderr, " - SecurityGroupRule %s\n", util.RelTime(t5s, t5e))
fmt.Fprintf(os.Stderr, " - VMs %s\n", util.RelTime(t6s, t6e))
}
return instanceID, nil
}

func createVMForComponents(ctx context.Context, vpcService *vpcv1.VpcV1, name string, ir llir.LLIR, resourceGroupID string, zone string, profile string, imageID string, namespace string, vpcID string, keyID string, subnetID string, secGroupID string, opts llir.Options) error {
group, _ := errgroup.WithContext(ctx)
var verboseFlag string
poolCount := intCounter{}
for _, c := range ir.Components {
instanceName := name + "-" + string(c.C())
if opts.Log.Verbose {
fmt.Fprintf(os.Stderr, "Creating VM %s\n", instanceName)
}

component, err := json.Marshal(c)
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
return err
}

llir, err := json.Marshal(ir)
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
return err
}

if opts.Log.Verbose {
verboseFlag = "--verbose"
}
//TODO - find executable and run it for each component!!!!!
cc := &config.CloudConfig{
WriteFiles: []config.File{
{
Path: "/app.yaml",
Content: "",
Owner: "root:root",
RawFilePermissions: "0644",
}},
RunCmd: []string{"curl https://dl.min.io/client/mc/release/linux-arm64/mc --create-dirs -o $HOME/minio-binaries/mc",
"chmod +x $HOME/minio-binaries/mc",
"export PATH=$PATH:$HOME/minio-binaries/",
"mc alias set myminio " + ir.Context.Queue.Endpoint + " " + ir.Context.Queue.AccessKey + " " + ir.Context.Queue.SecretKey, //Todo
"mc get " + ir.Context.Run.AsFile(q.Blobs) + " $HOME/lunchpail", //set path //use mc client to download binary
"$HOME/lunchpail component run-locally --component " + string(component) + " --llir " + string(llir) + " " + verboseFlag},
}

group.Go(func() error {
if c.C() == lunchpail.DispatcherComponent || c.C() == lunchpail.WorkStealerComponent {
instance, err := createInstance(vpcService, instanceName, ir, c, resourceGroupID, vpcID, keyID, zone, profile, subnetID, secGroupID, imageID, namespace, opts)
instance, err := createInstance(vpcService, instanceName, resourceGroupID, vpcID, keyID, zone, profile, subnetID, secGroupID, imageID, namespace, opts, cc)
if err != nil {
return err
}
Expand Down Expand Up @@ -350,7 +409,7 @@ func createAndInitVM(ctx context.Context, vpcService *vpcv1.VpcV1, name string,
for i := 0; i < numInstances; i++ {
workerName := poolName + "-" + strconv.Itoa(i) //multiple worker instances
c = c.SetWorkers(int(parallelism[i]))
instance, err := createInstance(vpcService, workerName, ir, c, resourceGroupID, vpcID, keyID, zone, profile, subnetID, secGroupID, imageID, namespace, opts)
instance, err := createInstance(vpcService, workerName, resourceGroupID, vpcID, keyID, zone, profile, subnetID, secGroupID, imageID, namespace, opts, cc)
if err != nil {
return err
}
Expand Down Expand Up @@ -378,18 +437,48 @@ func createAndInitVM(ctx context.Context, vpcService *vpcv1.VpcV1, name string,
if err := group.Wait(); err != nil {
return err
}
t6e := time.Now()

fmt.Printf("Setup done %s\n", util.RelTime(t1s, t6e))
fmt.Printf(" - VPC %s\n", util.RelTime(t1s, t1e))
fmt.Printf(" - SSH %s\n", util.RelTime(t2s, t2e))
fmt.Printf(" - Subnet %s\n", util.RelTime(t3s, t3e))
fmt.Printf(" - SecurityGroup %s\n", util.RelTime(t4s, t4e))
fmt.Printf(" - SecurityGroupRule %s\n", util.RelTime(t5s, t5e))
fmt.Printf(" - VMs %s\n", util.RelTime(t6s, t6e))
return nil
}

func createVMForImage(vpcService *vpcv1.VpcV1, name string, resourceGroupID string, zone string, profile string, imageID string, namespace string, vpcID string, keyID string, subnetID string, secGroupID string, opts llir.Options) (string, error) {
if opts.Log.Verbose {
fmt.Printf("Creating VM %s for custom image creations\n", name)
}
var rclone string
//TODO - download executable from S3 and store binary in image!!!!!
cc := &config.CloudConfig{
WriteFiles: []config.File{
{
Path: "/rclone.conf",
Content: rclone,
Owner: "root:root",
RawFilePermissions: "0644",
}},
RunCmd: []string{"sleep 10", //Minimum of 10 seconds needed for cluster to be able to run `apply`
""},
}
instance, err := createInstance(vpcService, name, resourceGroupID, vpcID, keyID, zone, profile, subnetID, secGroupID, imageID, namespace, opts, cc)
if err != nil {
return "", err
}

floatingIPID, err := createFloatingIP(vpcService, name, resourceGroupID, zone)
if err != nil {
return "", err
}

options := &vpcv1.AddInstanceNetworkInterfaceFloatingIPOptions{
ID: &floatingIPID,
InstanceID: instance.ID,
NetworkInterfaceID: instance.PrimaryNetworkInterface.ID,
}
_, response, err := vpcService.AddInstanceNetworkInterfaceFloatingIP(options)
if err != nil {
return "", fmt.Errorf("failed to add floating IP to network interface: %v and the response is: %s", err, response)
}
return *instance.ID, err
}

func (backend Backend) SetAction(ctx context.Context, opts llir.Options, ir llir.LLIR, action Action) error {
runname := ir.RunName()

Expand All @@ -406,7 +495,7 @@ func (backend Backend) SetAction(ctx context.Context, opts llir.Options, ir llir
}
zone = randomZone
}
if err := createAndInitVM(ctx, backend.vpcService, runname, ir, backend.config.ResourceGroup.GUID, backend.sshKeyType, backend.sshPublicKey, zone, opts.Profile, opts.ImageID, backend.namespace, opts); err != nil {
if _, err := createResources(ctx, backend.vpcService, runname, ir, backend.config.ResourceGroup.GUID, backend.sshKeyType, backend.sshPublicKey, zone, opts.Profile, opts.ImageID, backend.namespace, opts); err != nil {
return err
}
}
Expand Down
36 changes: 36 additions & 0 deletions pkg/be/ibmcloud/image.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package ibmcloud

import (
"context"

"lunchpail.io/pkg/ir/llir"
)

func (backend Backend) CreateImage(ctx context.Context, ir llir.LLIR, opts llir.Options, destroy bool) (string, error) {
runname := ir.RunName()

zone := opts.Zone //command line zone value
if zone == "" { //random zone value using config
randomZone, err := getRandomizedZone(backend.config, backend.vpcService) //Todo: spread among random zones with a subnet in each zone
if err != nil {
return "", err
}
zone = randomZone
}
instanceID, err := createResources(ctx, backend.vpcService, runname, ir, backend.config.ResourceGroup.GUID, backend.sshKeyType, backend.sshPublicKey, zone, opts.Profile, opts.ImageID, backend.namespace, opts)
if err != nil {
return "", err
}

imageID, err := createImage(backend.vpcService, runname, backend.config.ResourceGroup.GUID, instanceID)
if err != nil {
return "", err
}

if destroy {
if err := stopOrDeleteVM(backend.vpcService, runname, backend.config.ResourceGroup.GUID, true); err != nil {
return "", err
}
}
return imageID, nil
}
12 changes: 12 additions & 0 deletions pkg/be/kubernetes/image.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package kubernetes

import (
"context"
"fmt"

"lunchpail.io/pkg/ir/llir"
)

func (backend Backend) CreateImage(ctx context.Context, ir llir.LLIR, opts llir.Options, destroy bool) (string, error) {
return "", fmt.Errorf("Unsupported operation: 'CreateImage'")
}
12 changes: 12 additions & 0 deletions pkg/be/local/image.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package local

import (
"context"
"fmt"

"lunchpail.io/pkg/ir/llir"
)

func (backend Backend) CreateImage(ctx context.Context, ir llir.LLIR, opts llir.Options, destroy bool) (string, error) {
return "", fmt.Errorf("Unsupported operation: 'CreateImage'")
}
Loading

0 comments on commit d8ecff7

Please sign in to comment.