v0.6.0

CHANGELOG.md

@@ -9,13 +9,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added the machine id to the output of a pull request in the catalog
+- Added the ability do do a catalog pull request without the need to specify the
+  local machine path, this will be taken from the user configuration in PD
+- Added a spinner to the long running commands for pull and push to notify the
+  user that the command is still running
+- Added a new endpoint to easily clone a virtual machine `/api/v1/machines/{id}/clone`
+- Added the ability to **enable** and **disable** a host from the orchestrator
+- Added the ability to configure the API **CORS policies** by passing environment variables
+
 ### Fixed
 
-### Changed
+- Fixed an typo in the docker-compose file that would not allow the root password
+  to be updated
+- Fixed an issue in the pull from the catalog where if there was an error the
+  system would crash
+- Fixed an issue where the provider would not take into account the host with a
+  schema present
+- Fixed a bug where the system would crash with a waiting group being negative
+- Fixed a bug where queries could get stuck while saving to the database
+- Fixed an issue where some credentials would be left behind in temporary files
+- further security fixes to the codebase
 
-### Deprecated
+### Changed
 
-### Removed
+- Packer Templates and Vagrant box endpoints are now disabled by default due to security
+  concerns on remote execution of code, you can enable them by setting the environment
+  variable `ENABLE_PACKER_PLUGIN` and `ENABLE_VAGRANT_PLUGIN` to `true`
 
 ## [0.5.6] - 2024-03-14
 

docker-compose.yml

@@ -1,10 +1,8 @@
 version: '3.9'
-name: api
+name: devops-service
 services:
-  api:
+  devops:
     build: .
-    ports:
-      - "80:80"
     environment:
       HMAC_SECRET: ''
       LOG_LEVEL: 'info'
@@ -15,7 +13,7 @@ services:
       TLS_PRIVATE_KEY: ''
       API_PORT: '80'
       API_PREFIX: '/api'
-      ROOR_PASSWORD: ''
+      ROOT_PASSWORD: ''
       DISABLE_CATALOG_CACHING: 'false'
       TOKEN_DURATION_MINUTES: 60
       MODE: api

docs/docs/getting-started/configuration.md

@@ -56,6 +56,11 @@ The root object of the configuration file is the environment object, which conta
 | DISABLE_CATALOG_CACHING | Specifies whether the service should disable the catalog caching | false |
 | USE_ORCHESTRATOR_RESOURCES | Specifies whether the service is running in orchestrator mode, which allows the service to use the resources of the orchestrator | false |
 | ORCHESTRATOR_PULL_FREQUENCY_SECONDS | The frequency in seconds that the orchestrator will sync with the other hosts in seconds | 30 |
+| CORS_ALLOWED_HEADERS | The headers that are allowed in the cors policy | "X-Requested-With, authorization, content-type" |
+| CORS_ALLOWED_ORIGINS | The origins that are allowed in the cors policy | "*" |
+| CORS_ALLOWED_METHODS | The methods that are allowed in the cors policy | "GET, HEAD, POST, PUT, DELETE, OPTIONS" |
+| ENABLE_PACKER_PLUGIN | Specifies whether the service should enable the packer plugin | false |
+| ENABLE_VAGRANT_PLUGIN | Specifies whether the service should enable the vagrant plugin | false |
 
 ### Json Web Tokens
 

go.work.sum

@@ -1,6 +1,7 @@
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/briandowns/spinner v1.23.0/go.mod h1:rPG4gmXeN3wQV/TsAY4w8lPdIM6RX3yqeBQJSrbXjuE=
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cjlapao/common-go-identity v0.0.3/go.mod h1:xuNepNCHVI/51Q6DQgNPYvx3HS0VaeEhGnp8YcDO/+I=

src/basecontext/api_context.go

@@ -19,4 +19,5 @@ type ApiContext interface {
 	LogErrorf(format string, a ...interface{})
 	LogDebugf(format string, a ...interface{})
 	LogWarnf(format string, a ...interface{})
+	LogTracef(format string, a ...interface{})
 }

src/basecontext/main.go

@@ -173,3 +173,17 @@ func (c *BaseContext) LogWarnf(format string, a ...interface{}) {
 	msg += format
 	common.Logger.Warn(msg, a...)
 }
+
+func (c *BaseContext) LogTracef(format string, a ...interface{}) {
+	// log is disabled, returning
+	if !c.shouldLog {
+		return
+	}
+
+	msg := ""
+	if c.GetRequestId() != "" {
+		msg = "[" + c.GetRequestId() + "] "
+	}
+	msg += format
+	common.Logger.Trace(msg, a...)
+}

src/basecontext/test/mock_base_context.go

@@ -123,3 +123,10 @@ func (m *MockBaseContext) LogWarnf(format string, a ...interface{}) {
 		m.callbackFunctions["LogWarnf"](value)
 	}
 }
+
+func (m *MockBaseContext) LogTracef(format string, a ...interface{}) {
+	if m.callbackFunctions["LogTracef"] != nil {
+		value := fmt.Sprintf(format, a...)
+		m.callbackFunctions["LogTracef"](value)
+	}
+}

src/catalog/models/catalog_manifest_provider.go

@@ -146,6 +146,15 @@ func (m *CatalogManifestProvider) Parse(connection string) error {
 			}
 		}
 
+		var schema string
+		if strings.HasPrefix(m.Host, "http://") || strings.HasPrefix(m.Host, "https://") {
+			schemaParts := strings.Split(m.Host, "://")
+			if len(schemaParts) == 2 {
+				schema = schemaParts[0]
+				m.Host = schemaParts[1]
+			}
+		}
+
 		if strings.ContainsAny(m.Host, "@") {
 			parts := strings.Split(m.Host, "@")
 			if len(parts) == 2 {
@@ -175,6 +184,9 @@ func (m *CatalogManifestProvider) Parse(connection string) error {
 				m.Host = parts[1]
 			}
 		}
+		if schema != "" {
+			m.Host = schema + "://" + m.Host
+		}
 	}
 
 	return nil

src/catalog/models/pull_catalog_manifest.go

@@ -1,11 +1,14 @@
 package models
 
 import (
+	"fmt"
+
 	"github.com/Parallels/prl-devops-service/basecontext"
 	"github.com/Parallels/prl-devops-service/catalog/cleanupservice"
 	"github.com/Parallels/prl-devops-service/config"
 	"github.com/Parallels/prl-devops-service/constants"
 	"github.com/Parallels/prl-devops-service/errors"
+	"github.com/Parallels/prl-devops-service/serviceprovider"
 	"github.com/Parallels/prl-devops-service/serviceprovider/system"
 )
 
@@ -31,9 +34,6 @@ type PullCatalogManifestRequest struct {
 }
 
 func (r *PullCatalogManifestRequest) Validate() error {
-	if r.Path == "" {
-		return ErrPullMissingPath
-	}
 	if r.CatalogId == "" {
 		return ErrPullMissingCatalogId
 	}
@@ -60,17 +60,30 @@ func (r *PullCatalogManifestRequest) Validate() error {
 		r.Owner = cfg.GetKey(constants.CURRENT_USER_ENV_VAR)
 	}
 
+	if r.Path == "" {
+		prl := serviceprovider.Get().ParallelsDesktopService
+		if prl == nil {
+			return errors.New("Local Path is required and we are unable to determine it without Parallels Desktop Service")
+		}
+		userPath, err := prl.GetUserHome(ctx, r.Owner)
+		if err != nil {
+			return fmt.Errorf("unable to determine user %v home for path", r.Owner)
+		}
+		r.Path = userPath
+	}
+
 	return nil
 }
 
 type PullCatalogManifestResponse struct {
 	ID             string                         `json:"id"`
-	CatalogId      string                         `json:"catalog_id"`
-	Version        string                         `json:"version"`
-	Architecture   string                         `json:"architecture"`
-	LocalPath      string                         `json:"local_path"`
-	MachineName    string                         `json:"machine_name"`
-	Manifest       *VirtualMachineCatalogManifest `json:"manifest"`
+	MachineID      string                         `json:"machine_id,omitempty"`
+	CatalogId      string                         `json:"catalog_id,omitempty"`
+	Version        string                         `json:"version,omitempty"`
+	Architecture   string                         `json:"architecture,omitempty"`
+	LocalPath      string                         `json:"local_path,omitempty"`
+	MachineName    string                         `json:"machine_name,omitempty"`
+	Manifest       *VirtualMachineCatalogManifest `json:"manifest,omitempty"`
 	CleanupRequest *cleanupservice.CleanupRequest `json:"-"`
 	Errors         []error                        `json:"-"`
 }

src/catalog/providers/aws_s3_bucket/main.go

@@ -6,9 +6,11 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/Parallels/prl-devops-service/basecontext"
 	"github.com/Parallels/prl-devops-service/catalog/common"
+	"github.com/briandowns/spinner"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
@@ -137,6 +139,10 @@ func (s *AwsS3BucketProvider) PullFile(ctx basecontext.ApiContext, path string,
 	remoteFilePath := strings.TrimPrefix(filepath.Join(path, filename), "/")
 	destinationFilePath := filepath.Join(destination, filename)
 
+	loader := spinner.New(spinner.CharSets[9], 10*time.Second)
+	loader.Prefix = "Downloading file "
+	loader.Start()
+
 	// Create a new session using the default region and credentials.
 	var err error
 	session, err := s.createSession()
@@ -164,6 +170,8 @@ func (s *AwsS3BucketProvider) PullFile(ctx basecontext.ApiContext, path string,
 		return err
 	}
 
+	loader.Stop()
+
 	return nil
 }
 

src/catalog/pull.go

@@ -239,11 +239,13 @@ func (s *CatalogManifestService) Pull(ctx basecontext.ApiContext, r *models.Pull
 
 				cacheFileName := fmt.Sprintf("%s.pdpack", fileChecksum)
 				needsPulling := false
+				// checking for the caching system to see if we need to pull the file
 				if cfg.IsCatalogCachingEnable() {
 					destinationFolder, err = cfg.CatalogCacheFolder()
 					if err != nil {
 						destinationFolder = r.Path
 					}
+
 					if helper.FileExists(filepath.Join(destinationFolder, cacheFileName)) {
 						ctx.LogInfof("File %v already exists in cache", fileName)
 					} else {
@@ -353,7 +355,7 @@ func (s *CatalogManifestService) renameMachineWithParallelsDesktop(ctx baseconte
 
 	if !response.HasErrors() {
 		ctx.LogInfof("Renaming machine %v to %v", r.MachineName, r.MachineName)
-		filter := fmt.Sprintf("home=%s", r.LocalMachineFolder)
+		filter := fmt.Sprintf("name=%s", r.MachineName)
 		vms, err := parallelsDesktopSvc.GetVms(ctx, filter)
 		if err != nil {
 			ctx.LogErrorf("Error getting machine %v: %v", r.MachineName, err)
@@ -363,25 +365,31 @@ func (s *CatalogManifestService) renameMachineWithParallelsDesktop(ctx baseconte
 		}
 
 		if len(vms) != 1 {
-			ctx.LogErrorf("Error getting machine %v: %v", r.MachineName, err)
-			response.AddError(err)
+			notFoundError := errors.Newf("Machine %v not found", r.MachineName)
+			ctx.LogErrorf("Error getting machine %v: %v", r.MachineName, notFoundError)
+			response.AddError(notFoundError)
 			response.CleanupRequest.AddLocalFileCleanupOperation(r.LocalMachineFolder, true)
 			return
 		}
 
-		response.ID = vms[0].ID
-		renameRequest := api_models.RenameVirtualMachineRequest{
-			ID:          vms[0].ID,
-			CurrentName: vms[0].Name,
-			NewName:     r.MachineName,
-		}
+		// Renaming only if the name is different
+		if vms[0].Name != r.MachineName {
+			response.ID = vms[0].ID
+			renameRequest := api_models.RenameVirtualMachineRequest{
+				ID:          vms[0].ID,
+				CurrentName: vms[0].Name,
+				NewName:     r.MachineName,
+			}
 
-		if err := parallelsDesktopSvc.RenameVm(ctx, renameRequest); err != nil {
-			ctx.LogErrorf("Error renaming machine %v: %v", r.MachineName, err)
-			response.AddError(err)
-			response.CleanupRequest.AddLocalFileCleanupOperation(r.LocalMachineFolder, true)
-			return
+			if err := parallelsDesktopSvc.RenameVm(ctx, renameRequest); err != nil {
+				ctx.LogErrorf("Error renaming machine %v: %v", r.MachineName, err)
+				response.AddError(err)
+				response.CleanupRequest.AddLocalFileCleanupOperation(r.LocalMachineFolder, true)
+				return
+			}
 		}
+
+		response.MachineID = vms[0].ID
 	} else {
 		ctx.LogErrorf("Error renaming machine %v: %v", r.MachineName, response.Errors)
 	}
@@ -394,7 +402,7 @@ func (s *CatalogManifestService) startMachineWithParallelsDesktop(ctx basecontex
 
 	if !response.HasErrors() {
 		ctx.LogInfof("Starting machine %v to %v", r.MachineName, r.MachineName)
-		filter := fmt.Sprintf("home=%s", r.LocalMachineFolder)
+		filter := fmt.Sprintf("name=%s", r.MachineName)
 		vms, err := parallelsDesktopSvc.GetVms(ctx, filter)
 		if err != nil {
 			ctx.LogErrorf("Error getting machine %v: %v", r.MachineName, err)
@@ -404,8 +412,9 @@ func (s *CatalogManifestService) startMachineWithParallelsDesktop(ctx basecontex
 		}
 
 		if len(vms) != 1 {
-			ctx.LogErrorf("Error getting machine %v: %v", r.MachineName, err)
-			response.AddError(err)
+			notFoundError := errors.Newf("Machine %v not found", r.MachineName)
+			ctx.LogErrorf("Error getting machine %v: %v", r.MachineName, notFoundError)
+			response.AddError(notFoundError)
 			response.CleanupRequest.AddLocalFileCleanupOperation(r.LocalMachineFolder, true)
 			return
 		}

src/cmd/catalog.go

@@ -3,13 +3,15 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/Parallels/prl-devops-service/basecontext"
 	"github.com/Parallels/prl-devops-service/constants"
 	"github.com/Parallels/prl-devops-service/pdfile"
 	"github.com/Parallels/prl-devops-service/pdfile/diagnostics"
 	"github.com/Parallels/prl-devops-service/pdfile/models"
 	"github.com/Parallels/prl-devops-service/serviceprovider/system"
+	"github.com/briandowns/spinner"
 	"github.com/cjlapao/common-go/helper"
 )
 
@@ -41,10 +43,10 @@ func processCatalog(ctx basecontext.ApiContext, operation string, filePath strin
 	case "list":
 		processCatalogListCmd(ctx, filePath)
 	case "push":
-		fmt.Println("Starting push...")
+		fmt.Println("Starting push, this can take a while...")
 		processCatalogPushCmd(ctx, filePath)
 	case "pull":
-		fmt.Println("Starting pull...")
+		fmt.Println("Starting pull, this can take a while...")
 		processCatalogPullCmd(ctx, filePath)
 	case "delete":
 		fmt.Println("Not implemented yet")
@@ -208,14 +210,23 @@ func processCatalogListCmd(ctx basecontext.ApiContext, filepath string) {
 func processCatalogPushCmd(ctx basecontext.ApiContext, filePath string) {
 	svc := catalogInitPdFile(ctx, "push", filePath)
 
+	s := spinner.New(spinner.CharSets[9], 500*time.Millisecond)
+	s.Start()
+	time.Sleep(4 * time.Second)
+
 	out, diags := svc.Run(ctx)
+
+	s.Stop()
 	if diags.HasErrors() {
 		for _, err := range diags.Errors() {
 			fmt.Println(err)
 		}
 		os.Exit(1)
 	}
 
+	// Stop the progress bar by printing a new line
+	fmt.Println()
+
 	ctx.LogInfof("%v", out)
 }
 

src/constants/main.go

@@ -58,6 +58,11 @@ const (
 	ORCHESTRATOR_PULL_FREQUENCY_SECONDS_ENV_VAR = "ORCHESTRATOR_PULL_FREQUENCY_SECONDS"
 	DATABASE_FOLDER_ENV_VAR                     = "DATABASE_FOLDER"
 	CATALOG_CACHE_FOLDER_ENV_VAR                = "CATALOG_CACHE_FOLDER"
+	CORS_ALLOWED_HEADERS_ENV_VAR                = "CORS_ALLOWED_HEADERS"
+	CORS_ALLOWED_METHODS_ENV_VAR                = "CORS_ALLOWED_METHODS"
+	CORS_ALLOWED_ORIGINS_ENV_VAR                = "CORS_ALLOWED_ORIGINS"
+	ENABLE_PACKER_PLUGIN_ENV_VAR                = "ENABLE_PACKER_PLUGIN"
+	ENABLE_VAGRANT_PLUGIN_ENV_VAR               = "ENABLE_VAGRANT_PLUGIN"
 )
 
 const (