diff --git a/pkg/instance/build.go b/pkg/instance/build.go index aa1bb54..fecc13d 100644 --- a/pkg/instance/build.go +++ b/pkg/instance/build.go @@ -3,6 +3,7 @@ package instance import ( "context" "fmt" + "os" "path/filepath" "sync" @@ -14,8 +15,6 @@ import ( "github.com/celestiaorg/knuu/pkg/container" ) -const buildDirBase = "/tmp/knuu" - type build struct { instance *Instance imageName string @@ -25,6 +24,7 @@ type build struct { args []string env map[string]string imageCache *sync.Map + buildDir string } func (i *Instance) Build() *build { @@ -53,10 +53,15 @@ func (b *build) SetImage(ctx context.Context, image string, args ...builder.ArgI return ErrSettingImageNotAllowed.WithParams(b.instance.state.String()) } + buildDir, err := b.getBuildDir() + if err != nil { + return ErrGettingBuildDir.Wrap(err) + } + // Use the builder to build a new image factory, err := container.NewBuilderFactory(container.BuilderFactoryOptions{ ImageName: image, - BuildContext: b.getBuildDir(), + BuildContext: buildDir, ImageBuilder: b.instance.ImageBuilder, Args: args, Logger: b.instance.Logger, @@ -86,9 +91,14 @@ func (b *build) SetGitRepo(ctx context.Context, gitContext builder.GitContext, a return ErrGettingImageName.Wrap(err) } + buildDir, err := b.getBuildDir() + if err != nil { + return ErrGettingBuildDir.Wrap(err) + } + factory, err := container.NewBuilderFactory(container.BuilderFactoryOptions{ ImageName: imageName, - BuildContext: b.getBuildDir(), + BuildContext: buildDir, ImageBuilder: b.instance.ImageBuilder, Args: args, Logger: b.instance.Logger, @@ -225,8 +235,16 @@ func getImageRegistry(imageName string) (string, error) { } // getBuildDir returns the build directory for the instance -func (b *build) getBuildDir() string { - return filepath.Join(buildDirBase, b.instance.name) +func (b *build) getBuildDir() (string, error) { + if b.buildDir != "" { + return b.buildDir, nil + } + + tmpDir, err := os.MkdirTemp("", "knuu-build-*") + if err != nil { + return "", err + } + return filepath.Join(tmpDir, b.instance.name), nil } // addFileToBuilder adds a file to the builder diff --git a/pkg/instance/errors.go b/pkg/instance/errors.go index 1230ca7..6a87395 100644 --- a/pkg/instance/errors.go +++ b/pkg/instance/errors.go @@ -83,6 +83,7 @@ var ( ErrSettingGitRepo = errors.New("SettingGitRepo", "setting git repo is only allowed in state 'None'. Current state is '%s'") ErrGettingBuildContext = errors.New("GettingBuildContext", "error getting build context") ErrGettingImageName = errors.New("GettingImageName", "error getting image name") + ErrGettingBuildDir = errors.New("GettingBuildDir", "error getting build directory") ErrSettingImageNotAllowedForSidecars = errors.New("SettingImageNotAllowedForSidecars", "setting image is not allowed for sidecars") ErrSettingCommand = errors.New("SettingCommand", "setting command is only allowed in state 'Preparing' or 'Committed'. Current state is '%s") ErrSettingArgsNotAllowed = errors.New("SettingArgsNotAllowed", "setting args is only allowed in state 'Preparing' or 'Committed'. Current state is '%s") @@ -102,7 +103,9 @@ var ( ErrCreatingDirectory = errors.New("CreatingDirectory", "error creating directory") ErrFailedToCreateDestFile = errors.New("FailedToCreateDestFile", "failed to create destination file '%s'") ErrFailedToOpenSrcFile = errors.New("FailedToOpenSrcFile", "failed to open source file '%s'") + ErrFailedToGetSrcFileInfo = errors.New("FailedToGetSrcFileInfo", "failed to get source file info for %s") ErrFailedToCopyFile = errors.New("FailedToCopyFile", "failed to copy from source '%s' to destination '%s'") + ErrFailedToSetPermissions = errors.New("FailedToSetPermissions", "failed to set permissions for destination file") ErrSrcDoesNotExistOrIsDirectory = errors.New("SrcDoesNotExistOrIsDirectory", "src '%s' does not exist or is a directory") ErrInvalidFormat = errors.New("InvalidFormat", "invalid format") ErrFailedToConvertToInt64 = errors.New("FailedToConvertToInt64", "failed to convert to int64") diff --git a/pkg/instance/execution.go b/pkg/instance/execution.go index 534bb5f..f8733a5 100644 --- a/pkg/instance/execution.go +++ b/pkg/instance/execution.go @@ -415,7 +415,6 @@ func (e *execution) prepareReplicaSetConfig() k8s.ReplicaSetConfig { Name: e.instance.name, Labels: e.Labels(), ServiceAccountName: e.instance.name, - FsGroup: e.instance.storage.fsGroup, ContainerConfig: containerConfig, SidecarConfigs: sidecarConfigs, } diff --git a/pkg/instance/resources.go b/pkg/instance/resources.go index 9497b1f..bbe99e4 100644 --- a/pkg/instance/resources.go +++ b/pkg/instance/resources.go @@ -89,13 +89,12 @@ func (r *resources) deployStorage(ctx context.Context) error { return ErrDeployingVolumeForInstance.WithParams(r.instance.name).Wrap(err) } } - if len(r.instance.storage.files) == 0 { - return nil + if len(r.instance.storage.files) != 0 { + if err := r.instance.storage.deployFiles(ctx); err != nil { + return ErrDeployingFilesForInstance.WithParams(r.instance.name).Wrap(err) + } } - if err := r.instance.storage.deployFiles(ctx); err != nil { - return ErrDeployingFilesForInstance.WithParams(r.instance.name).Wrap(err) - } return nil } @@ -123,8 +122,7 @@ func (r *resources) destroyResources(ctx context.Context) error { } if len(r.instance.storage.files) != 0 { - err := r.instance.storage.destroyFiles(ctx) - if err != nil { + if err := r.instance.storage.destroyFiles(ctx); err != nil { return ErrDestroyingFilesForInstance.WithParams(r.instance.name).Wrap(err) } } diff --git a/pkg/instance/storage.go b/pkg/instance/storage.go index 18fad74..b2343ec 100644 --- a/pkg/instance/storage.go +++ b/pkg/instance/storage.go @@ -22,9 +22,10 @@ type storage struct { instance *Instance volumes []*k8s.Volume files []*k8s.File - fsGroup int64 } +const defaultFilePermission = 0644 + func (i *Instance) Storage() *storage { return i.storage } @@ -43,7 +44,7 @@ func (s *storage) AddFile(src string, dest string, chown string) error { return err } - dstPath, err := s.copyFileToBuildDir(src, dest) + buildDirPath, err := s.copyFileToBuildDir(src, dest) if err != nil { return err } @@ -60,14 +61,18 @@ func (s *storage) AddFile(src string, dest string, chown string) error { if srcInfo.Size() > maxTotalFilesBytes { return ErrFileTooLargeCommitted.WithParams(src) } - return s.addFileToInstance(dstPath, dest, chown) + return s.addFileToInstance(buildDirPath, dest, chown) } + buildDir, err := s.instance.build.getBuildDir() + if err != nil { + return ErrGettingBuildDir.Wrap(err) + } s.instance.Logger.WithFields(logrus.Fields{ "file": dest, "instance": s.instance.name, "state": s.instance.state, - "build_dir": s.instance.build.getBuildDir(), + "build_dir": buildDir, }).Debug("added file") return nil } @@ -101,7 +106,11 @@ func (s *storage) AddFolder(src string, dest string, chown string) error { if err != nil { return err } - dstPath := filepath.Join(s.instance.build.getBuildDir(), dest, relPath) + buildDir, err := s.instance.build.getBuildDir() + if err != nil { + return ErrGettingBuildDir.Wrap(err) + } + dstPath := filepath.Join(buildDir, dest, relPath) if info.IsDir() { // create directory at destination path @@ -115,11 +124,15 @@ func (s *storage) AddFolder(src string, dest string, chown string) error { return ErrCopyingFolderToInstance.WithParams(src, s.instance.name).Wrap(err) } + buildDir, err := s.instance.build.getBuildDir() + if err != nil { + return ErrGettingBuildDir.Wrap(err) + } s.instance.Logger.WithFields(logrus.Fields{ "folder": dest, "instance": s.instance.name, "state": s.instance.state, - "build_dir": s.instance.build.getBuildDir(), + "build_dir": buildDir, }).Debug("added folder") return nil } @@ -140,6 +153,9 @@ func (s *storage) AddFileBytes(bytes []byte, dest string, chown string) error { if _, err := tmpfile.Write(bytes); err != nil { return err } + if err := tmpfile.Chmod(defaultFilePermission); err != nil { + return err + } if err := tmpfile.Close(); err != nil { return err } @@ -249,40 +265,64 @@ func (s *storage) validateFileArgs(src, dest, chown string) error { if !strings.Contains(chown, ":") || len(strings.Split(chown, ":")) != 2 { return ErrChownMustBeInFormatUserGroup } + + parts := strings.Split(chown, ":") + for _, part := range parts { + if _, err := strconv.ParseInt(part, 10, 64); err != nil { + return ErrFailedToConvertToInt64.WithParams(part).Wrap(err) + } + } return nil } func (s *storage) copyFileToBuildDir(src, dest string) (string, error) { - dstPath := filepath.Join(s.instance.build.getBuildDir(), dest) + buildDir, err := s.instance.build.getBuildDir() + if err != nil { + return "", ErrGettingBuildDir.Wrap(err) + } + dstPath := filepath.Join(buildDir, dest) if err := os.MkdirAll(filepath.Dir(dstPath), os.ModePerm); err != nil { return "", ErrCreatingDirectory.Wrap(err) } - dst, err := os.Create(dstPath) - if err != nil { - return "", ErrFailedToCreateDestFile.WithParams(dstPath).Wrap(err) - } - defer dst.Close() - srcFile, err := os.Open(src) if err != nil { return "", ErrFailedToOpenSrcFile.WithParams(src).Wrap(err) } defer srcFile.Close() + srcInfo, err := srcFile.Stat() + if err != nil { + return "", ErrFailedToGetSrcFileInfo.WithParams(src).Wrap(err) + } + + dst, err := os.OpenFile(dstPath, os.O_CREATE|os.O_WRONLY, srcInfo.Mode().Perm()) + if err != nil { + return "", ErrFailedToCreateDestFile.WithParams(dstPath).Wrap(err) + } + defer dst.Close() + if _, err := io.Copy(dst, srcFile); err != nil { return "", ErrFailedToCopyFile.WithParams(src, dstPath).Wrap(err) } + // Ensure the destination file has the same permissions as the source file + if err := os.Chmod(dstPath, srcInfo.Mode().Perm()); err != nil { + return "", ErrFailedToSetPermissions.WithParams(dstPath).Wrap(err) + } + return dstPath, nil } -func (s *storage) addFileToInstance(dstPath, dest, chown string) error { - srcInfo, err := os.Stat(dstPath) +func (s *storage) addFileToInstance(srcPath, dest, chown string) error { + srcInfo, err := os.Stat(srcPath) if os.IsNotExist(err) || srcInfo.IsDir() { - return ErrSrcDoesNotExistOrIsDirectory.WithParams(dstPath).Wrap(err) + return ErrSrcDoesNotExistOrIsDirectory.WithParams(srcPath).Wrap(err) } + // get the permission of the src file + permission := fmt.Sprintf("%o", srcInfo.Mode().Perm()) + size := int64(0) for _, file := range s.files { srcInfo, err := os.Stat(file.Source) @@ -291,30 +331,17 @@ func (s *storage) addFileToInstance(dstPath, dest, chown string) error { } size += srcInfo.Size() } - srcInfo, err = os.Stat(dstPath) + srcInfo, err = os.Stat(srcPath) if err != nil { return ErrFailedToGetFileSize.Wrap(err) } size += srcInfo.Size() if size > maxTotalFilesBytes { - return ErrTotalFilesSizeTooLarge.WithParams(dstPath) + return ErrTotalFilesSizeTooLarge.WithParams(srcPath) } - file := s.instance.K8sClient.NewFile(dstPath, dest) - parts := strings.Split(chown, ":") - if len(parts) != 2 { - return ErrInvalidFormat - } + file := s.instance.K8sClient.NewFile(srcPath, dest, chown, permission) - group, err := strconv.ParseInt(parts[1], 10, 64) - if err != nil { - return ErrFailedToConvertToInt64.Wrap(err) - } - - if s.fsGroup != 0 && s.fsGroup != group { - return ErrAllFilesMustHaveSameGroup - } - s.fsGroup = group s.files = append(s.files, file) return nil } @@ -468,6 +495,5 @@ func (s *storage) clone() *storage { instance: nil, volumes: volumesCopy, files: filesCopy, - fsGroup: s.fsGroup, } } diff --git a/pkg/k8s/pod.go b/pkg/k8s/pod.go index 397110c..a0ee9dd 100644 --- a/pkg/k8s/pod.go +++ b/pkg/k8s/pod.go @@ -26,12 +26,10 @@ const ( // knuuPath is the path where the knuu volume is mounted knuuPath = "/knuu" - // 0777 is used so that the files are usable by any user in the container without needing to change permissions - defaultFileModeForVolume = 0777 - podFilesConfigmapNameSuffix = "-config" initContainerNameSuffix = "-init" + initContainerImage = "nicolaka/netshoot" defaultContainerUser = 0 ) @@ -73,8 +71,10 @@ type Volume struct { } type File struct { - Source string - Dest string + Source string + Dest string + Chown string + Permission string } // DeployPod creates a new pod in the namespace that k8s client is initiate with if it doesn't already exist. @@ -103,10 +103,12 @@ func (c *Client) NewVolume(path string, size resource.Quantity, owner int64) *Vo } } -func (c *Client) NewFile(source, dest string) *File { +func (c *Client) NewFile(source, dest, chown, permission string) *File { return &File{ - Source: source, - Dest: dest, + Source: source, + Dest: dest, + Chown: chown, + Permission: permission, } } @@ -383,6 +385,16 @@ func buildPodVolumes(name string, volumesAmount, filesAmount int) []v1.Volume { podVolumes = append(podVolumes, podVolume) } + if volumesAmount == 0 && filesAmount != 0 { + podVolume := v1.Volume{ + Name: name, + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + } + podVolumes = append(podVolumes, podVolume) + } + if filesAmount != 0 { podFiles := v1.Volume{ Name: name + podFilesConfigmapNameSuffix, @@ -391,7 +403,7 @@ func buildPodVolumes(name string, volumesAmount, filesAmount int) []v1.Volume { LocalObjectReference: v1.LocalObjectReference{ Name: name, }, - DefaultMode: ptr.To[int32](defaultFileModeForVolume), + DefaultMode: ptr.To[int32](0600), }, }, } @@ -416,25 +428,22 @@ func buildContainerVolumes(name string, volumes []*Volume, files []*File) []v1.V ) } - var containerFiles []v1.VolumeMount - - for n, file := range files { - shouldAddFile := true - for _, volume := range volumes { - if strings.HasPrefix(file.Dest, volume.Path) { - shouldAddFile = false - break - } + if len(volumes) == 0 && len(files) != 0 { + uniquePaths := make(map[string]bool) + for _, file := range files { + uniquePaths[filepath.Dir(file.Dest)] = true } - if shouldAddFile { - containerFiles = append(containerFiles, v1.VolumeMount{ - Name: name + podFilesConfigmapNameSuffix, - MountPath: file.Dest, - SubPath: fmt.Sprintf("%d", n), + for path := range uniquePaths { + containerVolumes = append(containerVolumes, v1.VolumeMount{ + Name: name, + MountPath: path, + SubPath: strings.TrimPrefix(path, "/"), }) } } + var containerFiles []v1.VolumeMount + return append(containerVolumes, containerFiles...) } @@ -444,11 +453,27 @@ func buildInitContainerVolumes(name string, volumes []*Volume, files []*File) [] return []v1.VolumeMount{} // return empty slice if no volumes are specified } - containerVolumes := []v1.VolumeMount{ - { + var containerVolumes []v1.VolumeMount + // if the user want do add volumes, we need to mount the knuu path + if len(volumes) != 0 { + containerVolumes = append(containerVolumes, v1.VolumeMount{ Name: name, - MountPath: knuuPath, // set the path to "/knuu" as per the requirements - }, + MountPath: knuuPath, + }) + } + // if the user don't want to add volumes, but want to add files, we need to mount the knuu path for the init container + if len(volumes) == 0 && len(files) != 0 { + uniquePaths := make(map[string]bool) + for _, file := range files { + uniquePaths[filepath.Dir(file.Dest)] = true + } + for path := range uniquePaths { + containerVolumes = append(containerVolumes, v1.VolumeMount{ + Name: name, + MountPath: knuuPath + path, + SubPath: strings.TrimPrefix(path, "/"), + }) + } } var containerFiles []v1.VolumeMount @@ -485,26 +510,33 @@ func (c *Client) buildInitContainerCommand(volumes []*Volume, files []*File) []s cmds = append(cmds, parentDirCmd) dirsProcessed[folder] = true } - copyFileToKnuu := fmt.Sprintf("cp %s %s && ", file.Dest, filepath.Join(knuuPath, file.Dest)) - cmds = append(cmds, copyFileToKnuu) + chown := file.Chown + permission := file.Permission + addFileToKnuu := fmt.Sprintf("cp %s %s && ", file.Dest, filepath.Join(knuuPath, file.Dest)) + if chown != "" { + addFileToKnuu += fmt.Sprintf("chown %s %s && ", chown, filepath.Join(knuuPath, file.Dest)) + } + if permission != "" { + addFileToKnuu += fmt.Sprintf("chmod %s %s && ", permission, filepath.Join(knuuPath, file.Dest)) + } + cmds = append(cmds, addFileToKnuu) } // for each volume, copy the contents of the volume to the knuu volume - for i, volume := range volumes { + for _, volume := range volumes { knuuVolumePath := fmt.Sprintf("%s%s", knuuPath, volume.Path) cmd := fmt.Sprintf("if [ -d %s ] && [ \"$(ls -A %s)\" ]; then mkdir -p %s && cp -r %s/* %s && chown -R %d:%d %s", volume.Path, volume.Path, knuuVolumePath, volume.Path, knuuVolumePath, volume.Owner, volume.Owner, knuuVolumePath) - if i < len(volumes)-1 { - cmd += " ;fi && " - } else { - cmd += " ;fi" - } + cmd += " ;fi && " cmds = append(cmds, cmd) } fullCommand := strings.Join(cmds, "") commands = append(commands, fullCommand) + if strings.HasSuffix(fullCommand, " && ") { + commands[len(commands)-1] = strings.TrimSuffix(commands[len(commands)-1], " && ") + } c.logger.WithField("command", fullCommand).Debug("init container command") return commands @@ -563,14 +595,14 @@ func prepareContainer(config ContainerConfig) v1.Container { // prepareInitContainers creates a slice of v1.Container as init containers. func (c *Client) prepareInitContainers(config ContainerConfig, init bool) []v1.Container { - if !init || len(config.Volumes) == 0 { + if !init || (len(config.Volumes) == 0 && len(config.Files) == 0) { return nil } return []v1.Container{ { Name: config.Name + initContainerNameSuffix, - Image: config.Image, + Image: initContainerImage, SecurityContext: &v1.SecurityContext{ RunAsUser: ptr.To[int64](defaultContainerUser), }, @@ -588,7 +620,6 @@ func preparePodVolumes(config ContainerConfig) []v1.Volume { func (c *Client) preparePodSpec(spec PodConfig, init bool) v1.PodSpec { podSpec := v1.PodSpec{ ServiceAccountName: spec.ServiceAccountName, - SecurityContext: &v1.PodSecurityContext{FSGroup: &spec.FsGroup}, InitContainers: c.prepareInitContainers(spec.ContainerConfig, init), Containers: []v1.Container{prepareContainer(spec.ContainerConfig)}, Volumes: preparePodVolumes(spec.ContainerConfig), @@ -596,9 +627,11 @@ func (c *Client) preparePodSpec(spec PodConfig, init bool) v1.PodSpec { // Prepare sidecar containers and append to the pod spec for _, sidecarConfig := range spec.SidecarConfigs { + sidecarInitContainer := c.prepareInitContainers(sidecarConfig, true) sidecarContainer := prepareContainer(sidecarConfig) sidecarVolumes := preparePodVolumes(sidecarConfig) + podSpec.InitContainers = append(podSpec.InitContainers, sidecarInitContainer...) podSpec.Containers = append(podSpec.Containers, sidecarContainer) podSpec.Volumes = append(podSpec.Volumes, sidecarVolumes...) } diff --git a/pkg/k8s/types.go b/pkg/k8s/types.go index 6e9cb0a..b42baf0 100644 --- a/pkg/k8s/types.go +++ b/pkg/k8s/types.go @@ -65,7 +65,7 @@ type KubeManager interface { Namespace() string NamespaceExists(ctx context.Context, name string) (bool, error) NetworkPolicyExists(ctx context.Context, name string) bool - NewFile(source, dest string) *File + NewFile(source, dest, chown, permission string) *File NewVolume(path string, size resource.Quantity, owner int64) *Volume PatchService(ctx context.Context, name string, opts ServiceOptions) (*corev1.Service, error) PortForwardPod(ctx context.Context, podName string, localPort, remotePort int) error diff --git a/pkg/sidecars/observability/obsy.go b/pkg/sidecars/observability/obsy.go index 413bb86..f96f4dc 100644 --- a/pkg/sidecars/observability/obsy.go +++ b/pkg/sidecars/observability/obsy.go @@ -20,7 +20,7 @@ const ( otelAgentName = "otel-agent" // %s will be replaced with the otelCollectorVersion otelAgentConfigFile = "/etc/otel-agent.yaml" - otelAgentConfigFilePermissions = "0:0" + otelAgentConfigFilePermissions = "10001:10001" otelCollectorCommand = "/otelcol-contrib" otelCollectorConfigArg = "--config=/etc/otel-agent.yaml"