From 8528704ed5e837030c92d3d78ce2328369eef590 Mon Sep 17 00:00:00 2001 From: Ian Kaneshiro Date: Thu, 11 Jan 2024 14:59:32 -0800 Subject: [PATCH 1/2] Beskar-yum: Add a sync handler that accepts a URL with credentials Credentials can be configured in repository properies in mirror urls, but these are presisted to the database. This new handler only sets the provided url in memory before performing the sync. --- integration/beskar_yum_test.go | 16 ++++++ integration/main_integration_test.go | 25 +++++++++ integration/pkg/repoconfig/parse.go | 11 ++-- integration/testdata/repoconfig.yaml | 1 + internal/plugins/yum/api.go | 7 +++ internal/plugins/yum/pkg/yumrepository/api.go | 33 ++++++++++++ pkg/plugins/yum/api/v1/api.go | 5 ++ pkg/plugins/yum/api/v1/endpoint.go | 39 ++++++++++++++ pkg/plugins/yum/api/v1/http.go | 30 +++++++++++ pkg/plugins/yum/api/v1/http_client.go | 51 +++++++++++++++++++ pkg/plugins/yum/api/v1/oas2.go | 20 ++++++++ 11 files changed, 233 insertions(+), 5 deletions(-) diff --git a/integration/beskar_yum_test.go b/integration/beskar_yum_test.go index 1aeee0f..af49cd4 100644 --- a/integration/beskar_yum_test.go +++ b/integration/beskar_yum_test.go @@ -247,6 +247,22 @@ var _ = Describe("Beskar YUM Plugin", func() { Expect(status.EndTime).ToNot(BeEmpty()) }) + It("Sync Repository With URL", func() { + err := beskarYUMClient().SyncRepositoryWithURL(context.Background(), repositoryAPIName, repo.AuthMirrorURL, true) + Expect(err).To(BeNil()) + }) + + It("Sync Status", func() { + status, err := beskarYUMClient().GetRepositorySyncStatus(context.Background(), repositoryAPIName) + Expect(err).To(BeNil()) + Expect(status.Syncing).To(BeFalse()) + Expect(status.SyncError).To(BeEmpty()) + Expect(status.SyncedPackages).To(Equal(len(repo.Files))) + Expect(status.TotalPackages).To(Equal(len(repo.Files))) + Expect(status.StartTime).ToNot(BeEmpty()) + Expect(status.EndTime).ToNot(BeEmpty()) + }) + It("Access Repository Artifacts", func() { for filename := range repo.Files { info, ok := repo.Files[filename] diff --git a/integration/main_integration_test.go b/integration/main_integration_test.go index 6c15174..90e8cb7 100644 --- a/integration/main_integration_test.go +++ b/integration/main_integration_test.go @@ -4,6 +4,7 @@ import ( _ "embed" "fmt" "net/http" + "net/url" "os" "os/exec" "syscall" @@ -38,6 +39,17 @@ func TestSubstrateIntegration(t *testing.T) { RunSpecs(t, "Beskar Integration Test Suite") } +func auth(username, password string, h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u, p, ok := r.BasicAuth() + if !ok || username != u || password != p { + http.Error(w, "401 unauthorized", http.StatusUnauthorized) + } + + h.ServeHTTP(w, r) + }) +} + var _ = SynchronizedBeforeSuite( // NOTE: This runs *only* on process #1 when run in parallel func() []byte { @@ -61,6 +73,19 @@ var _ = SynchronizedBeforeSuite( fs := http.FileServer(http.Dir(repo.LocalPath)) pathPrefix := fmt.Sprintf("/%s/", name) mux.Handle(pathPrefix, http.StripPrefix(pathPrefix, fs)) + + if repo.AuthMirrorURL != "" { + u, err := url.Parse(repo.AuthMirrorURL) + if err != nil { + continue + } + + username := u.User.Username() + password, _ := u.User.Password() + + pathPrefix := fmt.Sprintf("/auth/%s/", name) + mux.Handle(pathPrefix, auth(username, password, http.StripPrefix(pathPrefix, fs))) + } } } diff --git a/integration/pkg/repoconfig/parse.go b/integration/pkg/repoconfig/parse.go index a98cdeb..30bc9b9 100644 --- a/integration/pkg/repoconfig/parse.go +++ b/integration/pkg/repoconfig/parse.go @@ -8,11 +8,12 @@ type File struct { } type Repository struct { - URL string `yaml:"url"` - LocalPath string `yaml:"local-path"` - MirrorURL string `yaml:"mirror-url"` - GPGKey string `yaml:"gpgkey"` - Files map[string]File `yaml:"files"` + URL string `yaml:"url"` + LocalPath string `yaml:"local-path"` + MirrorURL string `yaml:"mirror-url"` + AuthMirrorURL string `yaml:"auth-mirror-url"` + GPGKey string `yaml:"gpgkey"` + Files map[string]File `yaml:"files"` } type Repositories map[string]Repository diff --git a/integration/testdata/repoconfig.yaml b/integration/testdata/repoconfig.yaml index 9f4d103..7a8e3c0 100644 --- a/integration/testdata/repoconfig.yaml +++ b/integration/testdata/repoconfig.yaml @@ -3,6 +3,7 @@ vault-rocky-8.3-ha-debug: local-path: testdata/vault-rocky-8.3-ha-debug url: http://127.0.0.1:8080/vault-rocky-8.3-ha-debug/Packages mirror-url: http://127.0.0.1:8080/vault-rocky-8.3-ha-debug + auth-mirror-url: http://test:test-password@127.0.0.1:8080/auth/vault-rocky-8.3-ha-debug gpgkey: | -----BEGIN PGP PUBLIC KEY BLOCK----- diff --git a/internal/plugins/yum/api.go b/internal/plugins/yum/api.go index 11e3ce7..c540fa2 100644 --- a/internal/plugins/yum/api.go +++ b/internal/plugins/yum/api.go @@ -53,6 +53,13 @@ func (p *Plugin) SyncRepository(ctx context.Context, repository string, wait boo return p.repositoryManager.Get(ctx, repository).SyncRepository(ctx, wait) } +func (p *Plugin) SyncRepositoryWithURL(ctx context.Context, repository, url string, wait bool) (err error) { + if err := checkRepository(repository); err != nil { + return err + } + return p.repositoryManager.Get(ctx, repository).SyncRepositoryWithURL(ctx, url, wait) +} + func (p *Plugin) GetRepositorySyncStatus(ctx context.Context, repository string) (syncStatus *apiv1.SyncStatus, err error) { if err := checkRepository(repository); err != nil { return nil, err diff --git a/internal/plugins/yum/pkg/yumrepository/api.go b/internal/plugins/yum/pkg/yumrepository/api.go index a014bbb..6192028 100644 --- a/internal/plugins/yum/pkg/yumrepository/api.go +++ b/internal/plugins/yum/pkg/yumrepository/api.go @@ -301,6 +301,39 @@ func (h *Handler) SyncRepository(_ context.Context, wait bool) (err error) { return nil } +func (h *Handler) SyncRepositoryWithURL(_ context.Context, url string, wait bool) (err error) { + if !h.Started() { + return werror.Wrap(gcode.ErrUnavailable, err) + } else if !h.getMirror() { + return werror.Wrap(gcode.ErrFailedPrecondition, errors.New("repository not setup as a mirror")) + } else if err := h.setMirrorURLs([]string{url}); err != nil { + return werror.Wrap(gcode.ErrInternal, err) + } else if h.delete.Load() { + return werror.Wrap(gcode.ErrAlreadyExists, fmt.Errorf("repository %s is being deleted", h.Repository)) + } else if h.syncing.Swap(true) { + return werror.Wrap(gcode.ErrAlreadyExists, errors.New("a repository sync is already running")) + } + + var waitErrCh chan error + + if wait { + waitErrCh = make(chan error, 1) + } + + select { + case h.syncCh <- waitErrCh: + if waitErrCh != nil { + if err := <-waitErrCh; err != nil { + return werror.Wrap(gcode.ErrInternal, fmt.Errorf("synchronization failed: %w", err)) + } + } + default: + return werror.Wrap(gcode.ErrUnavailable, errors.New("something goes wrong")) + } + + return nil +} + func (h *Handler) GetRepositorySyncStatus(context.Context) (syncStatus *apiv1.SyncStatus, err error) { reposync := h.getReposync() return &apiv1.SyncStatus{ diff --git a/pkg/plugins/yum/api/v1/api.go b/pkg/plugins/yum/api/v1/api.go index ab61427..8cfaa35 100644 --- a/pkg/plugins/yum/api/v1/api.go +++ b/pkg/plugins/yum/api/v1/api.go @@ -115,6 +115,11 @@ type YUM interface { //nolint:interfacebloat //kun:success statusCode=200 SyncRepository(ctx context.Context, repository string, wait bool) (err error) + // Sync YUM repository with an upstream repository using a specified URL. + //kun:op GET /repository/sync:url + //kun:success statusCode=200 + SyncRepositoryWithURL(ctx context.Context, repository, mirrorURL string, wait bool) (err error) + // Get YUM repository sync status. //kun:op GET /repository/sync:status //kun:success statusCode=200 diff --git a/pkg/plugins/yum/api/v1/endpoint.go b/pkg/plugins/yum/api/v1/endpoint.go index d49f21c..0d78642 100644 --- a/pkg/plugins/yum/api/v1/endpoint.go +++ b/pkg/plugins/yum/api/v1/endpoint.go @@ -426,6 +426,45 @@ func MakeEndpointOfSyncRepository(s YUM) endpoint.Endpoint { } } +type SyncRepositoryWithURLRequest struct { + Repository string `json:"repository"` + MirrorURL string `json:"mirror_url"` + Wait bool `json:"wait"` +} + +// ValidateSyncRepositoryWithURLRequest creates a validator for SyncRepositoryWithURLRequest. +func ValidateSyncRepositoryWithURLRequest(newSchema func(*SyncRepositoryWithURLRequest) validating.Schema) httpoption.Validator { + return httpoption.FuncValidator(func(value interface{}) error { + req := value.(*SyncRepositoryWithURLRequest) + return httpoption.Validate(newSchema(req)) + }) +} + +type SyncRepositoryWithURLResponse struct { + Err error `json:"-"` +} + +func (r *SyncRepositoryWithURLResponse) Body() interface{} { return r } + +// Failed implements endpoint.Failer. +func (r *SyncRepositoryWithURLResponse) Failed() error { return r.Err } + +// MakeEndpointOfSyncRepositoryWithURL creates the endpoint for s.SyncRepositoryWithURL. +func MakeEndpointOfSyncRepositoryWithURL(s YUM) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + req := request.(*SyncRepositoryWithURLRequest) + err := s.SyncRepositoryWithURL( + ctx, + req.Repository, + req.MirrorURL, + req.Wait, + ) + return &SyncRepositoryWithURLResponse{ + Err: err, + }, nil + } +} + type UpdateRepositoryRequest struct { Repository string `json:"repository"` Properties *RepositoryProperties `json:"properties"` diff --git a/pkg/plugins/yum/api/v1/http.go b/pkg/plugins/yum/api/v1/http.go index 2a436db..f0b0cdd 100644 --- a/pkg/plugins/yum/api/v1/http.go +++ b/pkg/plugins/yum/api/v1/http.go @@ -178,6 +178,20 @@ func NewHTTPRouter(svc YUM, codecs httpcodec.Codecs, opts ...httpoption.Option) ), ) + codec = codecs.EncodeDecoder("SyncRepositoryWithURL") + validator = options.RequestValidator("SyncRepositoryWithURL") + r.Method( + "GET", "/repository/sync:url", + kithttp.NewServer( + MakeEndpointOfSyncRepositoryWithURL(svc), + decodeSyncRepositoryWithURLRequest(codec, validator), + httpcodec.MakeResponseEncoder(codec, 200), + append(kitOptions, + kithttp.ServerErrorEncoder(httpcodec.MakeErrorEncoder(codec)), + )..., + ), + ) + codec = codecs.EncodeDecoder("UpdateRepository") validator = options.RequestValidator("UpdateRepository") r.Method( @@ -371,6 +385,22 @@ func decodeSyncRepositoryRequest(codec httpcodec.Codec, validator httpoption.Val } } +func decodeSyncRepositoryWithURLRequest(codec httpcodec.Codec, validator httpoption.Validator) kithttp.DecodeRequestFunc { + return func(_ context.Context, r *http.Request) (interface{}, error) { + var _req SyncRepositoryWithURLRequest + + if err := codec.DecodeRequestBody(r, &_req); err != nil { + return nil, err + } + + if err := validator.Validate(&_req); err != nil { + return nil, err + } + + return &_req, nil + } +} + func decodeUpdateRepositoryRequest(codec httpcodec.Codec, validator httpoption.Validator) kithttp.DecodeRequestFunc { return func(_ context.Context, r *http.Request) (interface{}, error) { var _req UpdateRepositoryRequest diff --git a/pkg/plugins/yum/api/v1/http_client.go b/pkg/plugins/yum/api/v1/http_client.go index 2e5cb9a..2c2dec7 100644 --- a/pkg/plugins/yum/api/v1/http_client.go +++ b/pkg/plugins/yum/api/v1/http_client.go @@ -599,6 +599,57 @@ func (c *HTTPClient) SyncRepository(ctx context.Context, repository string, wait return nil } +func (c *HTTPClient) SyncRepositoryWithURL(ctx context.Context, repository string, mirrorURL string, wait bool) (err error) { + codec := c.codecs.EncodeDecoder("SyncRepositoryWithURL") + + path := "/repository/sync:url" + u := &url.URL{ + Scheme: c.scheme, + Host: c.host, + Path: c.pathPrefix + path, + } + + reqBody := struct { + Repository string `json:"repository"` + MirrorURL string `json:"mirror_url"` + Wait bool `json:"wait"` + }{ + Repository: repository, + MirrorURL: mirrorURL, + Wait: wait, + } + reqBodyReader, headers, err := codec.EncodeRequestBody(&reqBody) + if err != nil { + return err + } + + _req, err := http.NewRequestWithContext(ctx, "GET", u.String(), reqBodyReader) + if err != nil { + return err + } + + for k, v := range headers { + _req.Header.Set(k, v) + } + + _resp, err := c.httpClient.Do(_req) + if err != nil { + return err + } + defer _resp.Body.Close() + + if _resp.StatusCode < http.StatusOK || _resp.StatusCode > http.StatusNoContent { + var respErr error + err := codec.DecodeFailureResponse(_resp.Body, &respErr) + if err == nil { + err = respErr + } + return err + } + + return nil +} + func (c *HTTPClient) UpdateRepository(ctx context.Context, repository string, properties *RepositoryProperties) (err error) { codec := c.codecs.EncodeDecoder("UpdateRepository") diff --git a/pkg/plugins/yum/api/v1/oas2.go b/pkg/plugins/yum/api/v1/oas2.go index 1c8402a..670f413 100644 --- a/pkg/plugins/yum/api/v1/oas2.go +++ b/pkg/plugins/yum/api/v1/oas2.go @@ -168,6 +168,18 @@ paths: schema: $ref: "#/definitions/SyncRepositoryRequestBody" %s + /repository/sync:url: + get: + description: "Sync YUM repository with an upstream repository using a specified URL." + operationId: "SyncRepositoryWithURL" + tags: + - yum + parameters: + - name: body + in: body + schema: + $ref: "#/definitions/SyncRepositoryWithURLRequestBody" + %s ` ) @@ -185,6 +197,7 @@ func getResponses(schema oas2.Schema) []oas2.OASResponses { oas2.GetOASResponses(schema, "ListRepositoryLogs", 200, &ListRepositoryLogsResponse{}), oas2.GetOASResponses(schema, "ListRepositoryPackages", 200, &ListRepositoryPackagesResponse{}), oas2.GetOASResponses(schema, "SyncRepository", 200, &SyncRepositoryResponse{}), + oas2.GetOASResponses(schema, "SyncRepositoryWithURL", 200, &SyncRepositoryWithURLResponse{}), } } @@ -255,6 +268,13 @@ func getDefinitions(schema oas2.Schema) map[string]oas2.Definition { }{})) oas2.AddResponseDefinitions(defs, schema, "SyncRepository", 200, (&SyncRepositoryResponse{}).Body()) + oas2.AddDefinition(defs, "SyncRepositoryWithURLRequestBody", reflect.ValueOf(&struct { + Repository string `json:"repository"` + MirrorURL string `json:"mirror_url"` + Wait bool `json:"wait"` + }{})) + oas2.AddResponseDefinitions(defs, schema, "SyncRepositoryWithURL", 200, (&SyncRepositoryWithURLResponse{}).Body()) + oas2.AddDefinition(defs, "UpdateRepositoryRequestBody", reflect.ValueOf(&struct { Repository string `json:"repository"` Properties *RepositoryProperties `json:"properties"` From 1a1f08ecc44944ce0c66957de96b1c101a69544f Mon Sep 17 00:00:00 2001 From: Ian Kaneshiro Date: Fri, 12 Jan 2024 09:40:59 -0800 Subject: [PATCH 2/2] Update dagger cache options --- build/mage/common.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/mage/common.go b/build/mage/common.go index 1479c12..7f49db5 100644 --- a/build/mage/common.go +++ b/build/mage/common.go @@ -99,8 +99,8 @@ func getChartsSource(client *dagger.Client) *dagger.Directory { func goCache(client *dagger.Client) func(dc *dagger.Container) *dagger.Container { return func(dc *dagger.Container) *dagger.Container { return dc. - WithMountedCache("/go", client.CacheVolume("go-mod-cache"), dagger.ContainerWithMountedCacheOpts{Sharing: dagger.Shared}). - WithMountedCache("/root/.cache", client.CacheVolume("go-build-cache"), dagger.ContainerWithMountedCacheOpts{Sharing: dagger.Shared}) + WithMountedCache("/go", client.CacheVolume("go-mod-cache"), dagger.ContainerWithMountedCacheOpts{Sharing: dagger.Locked}). + WithMountedCache("/root/.cache", client.CacheVolume("go-build-cache"), dagger.ContainerWithMountedCacheOpts{Sharing: dagger.Locked}) } }