Add support for KCL packages (#3434)

Signed-off-by: Sergio Castaño Arteaga <[email protected]> Signed-off-by: Cintia Sanchez Garcia <[email protected]> Co-authored-by: Sergio Castaño Arteaga <[email protected]> Co-authored-by: Cintia Sanchez Garcia <[email protected]>
artifacthub · Oct 18, 2023 · d6281cc · d6281cc
1 parent c1686f4
commit d6281cc
Show file tree

Hide file tree

Showing 42 changed files with 369 additions and 41 deletions.
diff --git a/README.md b/README.md
@@ -21,6 +21,7 @@ At the moment, the following artifacts kinds are supported *(with plans to suppo
 - [Gatekeeper policies](https://open-policy-agent.github.io/gatekeeper/website/docs/)
 - [Helm charts](https://helm.sh/)
 - [Helm plugins](https://helm.sh/docs/topics/plugins/)
+- [KCL packages](https://kcl-lang.io)
 - [KEDA scalers](https://keda.sh/)
 - [Keptn integrations](https://keptn.sh)
 - [Knative client plugins](https://knative.dev)

diff --git a/charts/artifact-hub/Chart.yaml b/charts/artifact-hub/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: artifact-hub
 description: Artifact Hub is a web-based application that enables finding, installing, and publishing Kubernetes packages.
 type: application
-version: 1.16.0
+version: 1.16.1-0
 appVersion: 1.16.0
 kubeVersion: ">= 1.19.0-0"
 home: https://artifacthub.io
@@ -27,6 +27,7 @@ keywords:
   - backstage
   - argo
   - kubearmor
+  - kcl
 maintainers:
   - name: Sergio
     email: [email protected]

diff --git a/charts/artifact-hub/values.schema.json b/charts/artifact-hub/values.schema.json
@@ -1140,7 +1140,7 @@
                 },
                 "repositoriesKinds": {
                     "title": "Repositories kinds to process ([] = all)",
-                    "description": "The following kinds are supported at the moment: falco, helm, olm, opa, tbaction, krew, helm-plugin, tekton-task, keda-scaler, coredns, keptn, tekton-pipeline, container, kubewarden, gatekeeper, kyverno, knative-client-plugin, backstage, argo-template, kubearmor",
+                    "description": "The following kinds are supported at the moment: falco, helm, olm, opa, tbaction, krew, helm-plugin, tekton-task, keda-scaler, coredns, keptn, tekton-pipeline, container, kubewarden, gatekeeper, kyverno, knative-client-plugin, backstage, argo-template, kubearmor, kcl",
                     "type": "array",
                     "items": {
                         "type": "string"

diff --git a/cmd/ah/lint.go b/cmd/ah/lint.go
@@ -90,7 +90,7 @@ func newLintCmd() *cobra.Command {
 			return lint(opts, &output{cmd.OutOrStdout()})
 		},
 	}
-	lintCmd.Flags().StringVarP(&opts.kind, "kind", "k", "helm", "repository kind: argo-template, backstage, coredns, falco, gatekeeper, helm, helm-plugin, keda-scaler, keptn, knative-client-plugin, krew, kubearmor, kubewarden, kyverno, olm, opa, tbaction, tekton-task, tekton-pipeline")
+	lintCmd.Flags().StringVarP(&opts.kind, "kind", "k", "helm", "repository kind: argo-template, backstage, coredns, falco, gatekeeper, helm, helm-plugin, kcl, keda-scaler, keptn, knative-client-plugin, krew, kubearmor, kubewarden, kyverno, olm, opa, tbaction, tekton-task, tekton-pipeline")
 	lintCmd.Flags().StringVarP(&opts.path, "path", "p", ".", "repository's packages path")
 	return lintCmd
 }
@@ -114,6 +114,7 @@ func lint(opts *lintOptions, out *output) error {
 		hub.CoreDNS,
 		hub.Falco,
 		hub.Gatekeeper,
+		hub.KCL,
 		hub.KedaScaler,
 		hub.Keptn,
 		hub.KnativeClientPlugin,
@@ -616,6 +617,7 @@ func (out *output) printPkgDetails(pkg *hub.Package) {
 		hub.CoreDNS,
 		hub.Falco,
 		hub.Gatekeeper,
+		hub.KCL,
 		hub.KedaScaler,
 		hub.Keptn,
 		hub.KnativeClientPlugin,

diff --git a/database/migrations/schema/049_kcl_packages.sql b/database/migrations/schema/049_kcl_packages.sql
@@ -0,0 +1,5 @@
+insert into repository_kind values (20, 'KCL packages');
+
+---- create above / drop below ----
+
+delete from repository_kind where repository_kind_id = 20;
diff --git a/database/tests/schema/schema.sql b/database/tests/schema/schema.sql
@@ -556,7 +556,8 @@ select results_eq(
         (16, 'Knative client plugins'),
         (17, 'Backstage plugins'),
         (18, 'Argo templates'),
-        (19, 'KubeArmor policies')
+        (19, 'KubeArmor policies'),
+        (20, 'KCL packages')
     $$,
     'Repository kinds should exist'
 );

diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml
@@ -1299,6 +1299,29 @@ paths:
           $ref: "#/components/responses/TooManyRequests"
         "500":
           $ref: "#/components/responses/InternalServerError"
+  "/packages/kcl/{repoName}/{packageName}":
+    get:
+      tags:
+        - Packages
+      summary: Get package details
+      description: Get package details
+      operationId: getKCLPackageDetails
+      parameters:
+        - $ref: "#/components/parameters/RepoNameParam"
+        - $ref: "#/components/parameters/PackageNameParam"
+      responses:
+        "200":
+          description: ""
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/KCLPackage"
+        "404":
+          $ref: "#/components/responses/NotFoundResponse"
+        "429":
+          $ref: "#/components/responses/TooManyRequests"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
   "/packages/keda-scaler/{repoName}/{packageName}":
     get:
       tags:
@@ -1767,6 +1790,30 @@ paths:
           $ref: "#/components/responses/TooManyRequests"
         "500":
           $ref: "#/components/responses/InternalServerError"
+  "/packages/KCL/{repoName}/{packageName}/{version}":
+    get:
+      tags:
+        - Packages
+      summary: Get package version details
+      description: Get package version details
+      operationId: getKCLVersionDetails
+      parameters:
+        - $ref: "#/components/parameters/RepoNameParam"
+        - $ref: "#/components/parameters/PackageNameParam"
+        - $ref: "#/components/parameters/VersionParam"
+      responses:
+        "200":
+          description: ""
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/KCLPackage"
+        "404":
+          $ref: "#/components/responses/NotFoundResponse"
+        "429":
+          $ref: "#/components/responses/TooManyRequests"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
   "/packages/keda-scaler/{repoName}/{packageName}/{version}":
     get:
       tags:
@@ -3920,6 +3967,8 @@ components:
                         example: "http://repo.url"
     HelmPluginPackage:
       $ref: "#/components/schemas/Package"
+    KCLPackage:
+      $ref: "#/components/schemas/Package"
     KedaScalerPackage:
       $ref: "#/components/schemas/Package"
     KeptnIntegrationsPackage:
@@ -4636,6 +4685,10 @@ components:
         - 14
         - 15
         - 16
+        - 17
+        - 18
+        - 19
+        - 20
       description: |
         Repository kind:
           * `0` - Helm charts
@@ -4658,6 +4711,7 @@ components:
           * `17` - Backstage plugins
           * `18` - Argo templates
           * `19` - KubeArmor templates
+          * `20` - KCL packages
     RepositoryKindParam:
       type: string
       enum:
@@ -4681,6 +4735,7 @@ components:
         - backstage
         - argo-template
         - kubearmor
+        - kcl
       description: |
         Repository kind name:
         * `helm` - Helm charts
@@ -4703,6 +4758,7 @@ components:
         * `backstage` - Backstage plugins
         * `argo-template` - Argo templates
         * `kubearmor` - KubeArmor policies
+        * `kcl` - KCL packages
     RepositorySummary:
       type: object
       required:
@@ -5242,6 +5298,7 @@ components:
           * `17` - Backstage plugins
           * `18` - Argo templates
           * `19` - KubeArmor templates
+          * `20` - KCL packages
     PackageNameParam:
       in: path
       name: packageName

diff --git a/docs/kcl_packages_repositories.md b/docs/kcl_packages_repositories.md
@@ -0,0 +1,49 @@
+## KCL packages repositories
+
+KCL packages repositories are expected to be hosted in GitHub, GitLab or Bitbucket repos. When adding your repository to Artifact Hub, the url used **must** follow the following format:
+
+- `https://github.com/user/repo[/path/to/packages]`
+- `https://gitlab.com/user/repo[/path/to/packages]`
+- `https://bitbucket.org/user/repo[/path/to/packages]`
+
+By default the `master` branch is used, but it's possible to specify a different one from the UI.
+
+*Please NOTE that the repository URL used when adding the repository to Artifact Hub **must NOT** contain the git hosting platform specific parts, like **tree/branch**, just the path to your packages like it would show in the filesystem.*
+
+The *path/to/packages* provided can contain metadata for one or more packages. Each package version **must** be on a separate folder, and it's up to you to decide if you want to publish one or multiple versions of your package.
+
+The structure of a repository with multiple packages and versions could look something like this:
+
+```sh
+$ tree path/to/packages
+path/to/packages
+├── artifacthub-repo.yml
+├── package1
+│   ├── 1.0.0
+│   │   ├── README.md
+│   │   └── artifacthub-pkg.yml
+│   └── 2.0.0
+│       ├── README.md
+│       └── artifacthub-pkg.yml
+└── package2
+    └── 1.0.0
+        ├── README.md
+        └── artifacthub-pkg.yml
+```
+
+This structure is flexible, and in some cases where you only have a package and a version it can be greatly simplified. In the case of a single package with a single version available at a time (the publisher doesn't want to make previous ones available, for example), the structure could look like this:
+
+```sh
+$ tree path/to/packages
+path/to/packages
+├── artifacthub-repo.yml
+└── package1
+    ├── README.md
+    └── artifacthub-pkg.yml
+```
+
+In the previous case, even the `package1` directory could be omitted. The reason is that both packages names and versions are read from the `artifacthub-pkg.yml` metadata file, so directories names are not used at all.
+
+Each package version **needs** an `artifacthub-pkg.yml` metadata file. Please see the file [spec](https://github.com/artifacthub/hub/blob/master/docs/metadata/artifacthub-pkg.yml) for more details. The [artifacthub-repo.yml](https://github.com/artifacthub/hub/blob/master/docs/metadata/artifacthub-repo.yml) repository metadata file shown above can be used to setup features like [Verified publisher](https://github.com/artifacthub/hub/blob/master/docs/repositories.md#verified-publisher) or [Ownership claim](https://github.com/artifacthub/hub/blob/master/docs/repositories.md#ownership-claim). This file must be located at `/path/to/packages`.
+
+Once you have added your repository, you are all set up. As you add new versions of your packages or new packages to your git repository, they'll be automatically indexed and listed in Artifact Hub.
diff --git a/docs/repositories.md b/docs/repositories.md
@@ -11,6 +11,7 @@ The following repositories kinds are supported at the moment:
 - [Falco rules repositories](https://github.com/artifacthub/hub/blob/master/docs/falco_rules_repositories.md)
 - [Helm charts repositories](https://github.com/artifacthub/hub/blob/master/docs/helm_charts_repositories.md)
 - [Helm plugins repositories](https://github.com/artifacthub/hub/blob/master/docs/helm_plugins_repositories.md)
+- [KCL packages repositories](https://github.com/artifacthub/hub/blob/master/docs/kcl_packages_repositories.md)
 - [KEDA scalers repositories](https://github.com/artifacthub/hub/blob/master/docs/keda_scalers_repositories.md)
 - [Keptn integrations repositories](https://github.com/artifacthub/hub/blob/master/docs/keptn_integrations_repositories.md)
 - [Knative client plugins repositories](https://github.com/artifacthub/hub/blob/master/docs/knative_client_plugins_repositories.md)

diff --git a/docs/www/headers/kcl_packages_repositories b/docs/www/headers/kcl_packages_repositories
@@ -0,0 +1,6 @@
+---
+title: "KCL packages"
+aliases: [
+  "/kcl_packages_repositories",
+]
+---
diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go
@@ -265,7 +265,7 @@ func (h *Handlers) setupRouter() {
 			r.Get("/stats", h.Packages.GetStats)
 			r.With(corsMW).Get("/search", h.Packages.Search)
 			r.With(h.Users.RequireLogin).Get("/starred", h.Packages.GetStarredByUser)
-			r.Route("/{^helm$|^falco$|^opa$|^olm|^tbaction|^krew|^helm-plugin|^tekton-task|^keda-scaler|^coredns|^keptn|^tekton-pipeline|^container|^kubewarden|^gatekeeper|^kyverno|^knative-client-plugin|^backstage|^argo-template|^kubearmor$}/{repoName}/{packageName}", func(r chi.Router) {
+			r.Route("/{^helm$|^falco$|^opa$|^olm|^tbaction|^krew|^helm-plugin|^tekton-task|^keda-scaler|^coredns|^keptn|^tekton-pipeline|^container|^kubewarden|^gatekeeper|^kyverno|^knative-client-plugin|^backstage|^argo-template|^kubearmor|^kcl$}/{repoName}/{packageName}", func(r chi.Router) {
 				r.Get("/feed/rss", h.Packages.RssFeed)
 				r.With(corsMW).Get("/summary", h.Packages.GetSummary)
 				r.Get("/{version}", h.Packages.Get)
@@ -430,7 +430,7 @@ func (h *Handlers) setupRouter() {
 
 	// Index special entry points
 	r.Route("/packages", func(r chi.Router) {
-		r.Route("/{^helm$|^falco$|^opa$|^olm|^tbaction|^krew|^helm-plugin|^tekton-task|^keda-scaler|^coredns|^keptn|^tekton-pipeline|^container|^kubewarden|^gatekeeper|^kyverno|^knative-client-plugin|^backstage|^argo-template|^kubearmor$}/{repoName}/{packageName}", func(r chi.Router) {
+		r.Route("/{^helm$|^falco$|^opa$|^olm|^tbaction|^krew|^helm-plugin|^tekton-task|^keda-scaler|^coredns|^keptn|^tekton-pipeline|^container|^kubewarden|^gatekeeper|^kyverno|^knative-client-plugin|^backstage|^argo-template|^kubearmor|^kcl$}/{repoName}/{packageName}", func(r chi.Router) {
 			r.With(h.Packages.InjectIndexMeta).Get("/{version}", h.Static.Index)
 			r.With(h.Packages.InjectIndexMeta).Get("/", h.Static.Index)
 		})

diff --git a/internal/handlers/pkg/handlers_test.go b/internal/handlers/pkg/handlers_test.go
@@ -2144,6 +2144,17 @@ func TestBuildURL(t *testing.T) {
 			"2.0.0",
 			baseURL + "/packages/kubearmor/repo1/pkg1/2.0.0",
 		},
+		{
+			&hub.Package{
+				NormalizedName: "pkg1",
+				Repository: &hub.Repository{
+					Kind: hub.KCL,
+					Name: "repo1",
+				},
+			},
+			"2.0.0",
+			baseURL + "/packages/kcl/repo1/pkg1/2.0.0",
+		},
 	}
 	for _, tc := range testCases {
 		tc := tc

diff --git a/internal/hub/repo.go b/internal/hub/repo.go
@@ -106,6 +106,9 @@ const (
 
 	// KubeArmor represents a repository with KubeArmor policies.
 	KubeArmor RepositoryKind = 19
+
+	// KCL represents a repository with KCL policies.
+	KCL RepositoryKind = 20
 )
 
 // GetKindName returns the name of the provided repository kind.
@@ -127,6 +130,8 @@ func GetKindName(kind RepositoryKind) string {
 		return "helm"
 	case HelmPlugin:
 		return "helm-plugin"
+	case KCL:
+		return "kcl"
 	case KedaScaler:
 		return "keda-scaler"
 	case Keptn:
@@ -176,6 +181,8 @@ func GetKindFromName(kind string) (RepositoryKind, error) {
 		return Helm, nil
 	case "helm-plugin":
 		return HelmPlugin, nil
+	case "kcl":
+		return KCL, nil
 	case "keda-scaler":
 		return KedaScaler, nil
 	case "keptn":

diff --git a/internal/repo/manager.go b/internal/repo/manager.go
@@ -86,6 +86,7 @@ var (
 		hub.Gatekeeper,
 		hub.Helm,
 		hub.HelmPlugin,
+		hub.KCL,
 		hub.KedaScaler,
 		hub.Keptn,
 		hub.KnativeClientPlugin,
@@ -283,6 +284,7 @@ func (m *Manager) ClaimOwnership(ctx context.Context, repoName, orgName string)
 		hub.Falco,
 		hub.Gatekeeper,
 		hub.HelmPlugin,
+		hub.KCL,
 		hub.KedaScaler,
 		hub.Keptn,
 		hub.KnativeClientPlugin,
@@ -460,6 +462,7 @@ func (m *Manager) locateMetadataFile(r *hub.Repository, basePath string) string
 		hub.Falco,
 		hub.Gatekeeper,
 		hub.HelmPlugin,
+		hub.KCL,
 		hub.KedaScaler,
 		hub.Keptn,
 		hub.KnativeClientPlugin,
@@ -821,6 +824,7 @@ func (m *Manager) validateURL(r *hub.Repository) error {
 		hub.Falco,
 		hub.Gatekeeper,
 		hub.HelmPlugin,
+		hub.KCL,
 		hub.KedaScaler,
 		hub.Keptn,
 		hub.KnativeClientPlugin,

diff --git a/internal/tracker/helpers.go b/internal/tracker/helpers.go
@@ -117,6 +117,7 @@ func SetupSource(i *hub.TrackerSourceInput) hub.TrackerSource {
 		hub.Backstage,
 		hub.CoreDNS,
 		hub.Gatekeeper,
+		hub.KCL,
 		hub.KedaScaler,
 		hub.Keptn,
 		hub.KnativeClientPlugin,

diff --git a/internal/tracker/tracker.go b/internal/tracker/tracker.go
@@ -190,6 +190,7 @@ func (t *Tracker) cloneRepository() (string, string, error) {
 		hub.Falco,
 		hub.Gatekeeper,
 		hub.HelmPlugin,
+		hub.KCL,
 		hub.KedaScaler,
 		hub.Keptn,
 		hub.KnativeClientPlugin,

diff --git a/scripts/prepare-docs.sh b/scripts/prepare-docs.sh
@@ -14,6 +14,7 @@ cat docs/www/headers/falco_rules_repositories docs/falco_rules_repositories.md >
 cat docs/www/headers/gatekeeper_policies_repositories docs/gatekeeper_policies_repositories.md > docs/www/content/topics/repositories/gatekeeper-policies.md
 cat docs/www/headers/helm_charts_repositories docs/helm_charts_repositories.md > docs/www/content/topics/repositories/helm-charts.md
 cat docs/www/headers/helm_plugins_repositories docs/helm_plugins_repositories.md > docs/www/content/topics/repositories/helm-plugins.md
+cat docs/www/headers/kcl_packages_repositories docs/kcl_packages_repositories.md > docs/www/content/topics/repositories/kcl-packages.md
 cat docs/www/headers/keda_scalers_repositories docs/keda_scalers_repositories.md > docs/www/content/topics/repositories/keda-scalers.md
 cat docs/www/headers/keptn_integrations_repositories docs/keptn_integrations_repositories.md > docs/www/content/topics/repositories/keptn-integrations.md
 cat docs/www/headers/knative_client_plugins_repositories docs/knative_client_plugins_repositories.md > docs/www/content/topics/repositories/knative-client-plugins.md