feat: add checksum validation to KCL packages

.github/workflows/include-kcl-checksums.yml

@@ -0,0 +1,48 @@
+name: Include KCL Modules Checksum
+
+on:
+  workflow_dispatch:
+    inputs:
+      kpm_reg:
+        description: "Enter the KPM Registry"
+        required: true
+        default: "localhost:5001"
+      kpm_repo:
+        description: "Enter the KPM Repository"
+        required: true
+        default: "test"
+
+jobs:
+  include_modules_checksum:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Install kcl
+        run: wget -q https://kcl-lang.io/script/install-cli.sh -O - | /bin/bash
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Run local registry for localhost
+        if: ${{ github.event.inputs.kpm_reg == 'localhost:5001' }}
+        run: ./scripts/reg.sh
+
+      - name: Login to registry
+        run: |
+          kcl registry login -u ${{ github.event.inputs.kpm_repo }} -p ${{ secrets.REGISTRY_ACCESS_TOKEN }} ${{ github.event.inputs.kpm_reg }}
+
+      - name: Get dependencies
+        run: go get -v ./...
+
+      - name: Run include checksum tool
+        env:
+          KPM_REG: ${{ github.event.inputs.kpm_reg }}
+          KPM_REPO: ${{ github.event.inputs.kpm_repo }}
+        run: go run ./Integrate-Checksum/main.go

Integrate-Checksum/main.go

@@ -0,0 +1,220 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"oras.land/oras-go/v2"
+	"oras.land/oras-go/v2/registry/remote"
+	"oras.land/oras-go/v2/registry/remote/auth"
+	"oras.land/oras-go/v2/registry/remote/retry"
+
+	"kcl-lang.io/kpm/pkg/client"
+	"kcl-lang.io/kpm/pkg/constants"
+	"kcl-lang.io/kpm/pkg/downloader"
+	"kcl-lang.io/kpm/pkg/opt"
+	pkg "kcl-lang.io/kpm/pkg/package"
+	"kcl-lang.io/kpm/pkg/utils"
+)
+
+const KCLModFile = "kcl.mod"
+
+// findKCLModFiles searches the specified root directory for all kcl.mod files and returns their paths.
+func findKCLModFiles(root string) ([]string, error) {
+	var modFilePaths []string
+
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if !info.IsDir() && info.Name() == KCLModFile {
+			modFilePaths = append(modFilePaths, filepath.Dir(path))
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error walking directory '%s': %w", root, err)
+	}
+	return modFilePaths, nil
+}
+
+// parseMediaType extracts the media type from the manifest content.
+func parseMediaType(content []byte) (string, error) {
+	var manifest struct {
+		MediaType string `json:"mediaType"`
+	}
+	if err := json.Unmarshal(content, &manifest); err != nil {
+		return "", fmt.Errorf("failed to unmarshal content for media type: %w", err)
+	}
+	if manifest.MediaType == "" {
+		return "", fmt.Errorf("media type is missing in manifest")
+	}
+	return manifest.MediaType, nil
+}
+
+// resolveDependency loads the KCL package from a directory and constructs a dependency object with OCI source information.
+func resolveDependency(kpmClient *client.KpmClient, packageDir string) (*pkg.Dependency, error) {
+	kclPkg, err := kpmClient.LoadPkgFromPath(packageDir)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load KCL package from '%s': %w", packageDir, err)
+	}
+
+	dependency := &pkg.Dependency{
+		Name: kclPkg.ModFile.Pkg.Name,
+		Source: downloader.Source{
+			Oci: &downloader.Oci{
+				Reg:  kpmClient.GetSettings().DefaultOciRegistry(),
+				Repo: utils.JoinPath(kpmClient.GetSettings().DefaultOciRepo(), kclPkg.GetPkgName()),
+				Tag:  kclPkg.GetPkgTag(),
+			},
+		},
+	}
+
+	if dependency.Sum, err = utils.HashDir(packageDir); err != nil {
+		return nil, fmt.Errorf("failed to hash directory '%s': %w", packageDir, err)
+	}
+	dependency.FromKclPkg(kclPkg)
+
+	return dependency, nil
+}
+
+// fetchManifest retrieves and unmarshals the OCI manifest for the given dependency.
+func fetchManifest(kpmClient *client.KpmClient, dependency *pkg.Dependency) (ocispec.Manifest, error) {
+	var manifest ocispec.Manifest
+
+	manifestJSON, err := kpmClient.FetchOciManifestIntoJsonStr(opt.OciFetchOptions{
+		FetchBytesOptions: oras.DefaultFetchBytesOptions,
+		OciOptions: opt.OciOptions{
+			Reg:  dependency.Source.Oci.Reg,
+			Repo: dependency.Source.Oci.Repo,
+			Tag:  dependency.Source.Oci.Tag,
+		},
+	})
+	if err != nil {
+		return manifest, fmt.Errorf("failed to fetch OCI manifest for '%s': %w", dependency.Name, err)
+	}
+
+	if err := json.Unmarshal([]byte(manifestJSON), &manifest); err != nil {
+		return manifest, fmt.Errorf("failed to unmarshal OCI manifest: %w", err)
+	}
+	return manifest, nil
+}
+
+// updateChecksum updates the checksum in the OCI manifest and pushes the manifest to the registry.
+func updateChecksum(manifest ocispec.Manifest, kpmClient *client.KpmClient, dependency *pkg.Dependency) error {
+	if manifest.Annotations == nil {
+		manifest.Annotations = make(map[string]string)
+	}
+	manifest.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM] = dependency.Sum
+
+	repo, err := configureRepository(dependency, kpmClient)
+	if err != nil {
+		return fmt.Errorf("failed to configure repository: %w", err)
+	}
+
+	manifestBytes, err := json.Marshal(manifest)
+	if err != nil {
+		return fmt.Errorf("failed to marshal updated manifest: %w", err)
+	}
+
+	return tagManifest(repo, manifestBytes, dependency)
+}
+
+// configureRepository initializes a repository reference and sets up the OCI client with credentials.
+func configureRepository(dependency *pkg.Dependency, kpmClient *client.KpmClient) (*remote.Repository, error) {
+	repoReference := utils.JoinPath(dependency.Source.Oci.Reg, dependency.Source.Oci.Repo)
+	repo, err := remote.NewRepository(repoReference)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create repository: %w", err)
+	}
+
+	cred, err := kpmClient.GetCredentials(dependency.Source.Oci.Reg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to retrieve credentials for registry '%s': %w", dependency.Source.Oci.Reg, err)
+	}
+
+	repo.Client = &auth.Client{
+		Client: &http.Client{
+			Transport: retry.NewTransport(http.DefaultTransport.(*http.Transport).Clone()),
+		},
+		Cache:  auth.NewCache(),
+		Header: http.Header{"Accept": []string{"application/vnd.oci.image.manifest.v1+json"}},
+		Credential: func(ctx context.Context, _ string) (auth.Credential, error) {
+			return *cred, nil
+		},
+	}
+
+	return repo, nil
+}
+
+// tagManifest tags the updated manifest in the OCI registry.
+func tagManifest(repo *remote.Repository, manifestBytes []byte, dependency *pkg.Dependency) error {
+	mediaType, err := parseMediaType(manifestBytes)
+	if err != nil {
+		return fmt.Errorf("failed to extract media type: %w", err)
+	}
+
+	if _, err := oras.TagBytes(context.Background(), repo.Manifests(), mediaType, manifestBytes, dependency.Source.Oci.Tag); err != nil {
+		return fmt.Errorf("failed to tag manifest in OCI registry: %w", err)
+	}
+
+	return nil
+}
+
+// processPackage processes the package directory and updates the OCI manifest if needed.
+func processPackage(packageDir string) error {
+	kpmClient, err := client.NewKpmClient()
+	if err != nil {
+		return fmt.Errorf("failed to create KPM client: %w", err)
+	}
+
+	dependency, err := resolveDependency(kpmClient, packageDir)
+	if err != nil {
+		return fmt.Errorf("failed to resolve dependency: %w", err)
+	}
+
+	manifest, err := fetchManifest(kpmClient, dependency)
+	if err != nil {
+		return fmt.Errorf("failed to fetch manifest: %w", err)
+	}
+
+	if existingSum, ok := manifest.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM]; ok && dependency.Sum == existingSum {
+		fmt.Println("Manifest already up to date with matching checksum.")
+		return nil
+	}
+
+	if err := updateChecksum(manifest, kpmClient, dependency); err != nil {
+		return fmt.Errorf("failed to update checksum in manifest: %w", err)
+	}
+
+	return nil
+}
+
+func main() {
+	currentDir, err := os.Getwd()
+	if err != nil {
+		fmt.Printf("Error getting current directory: %v\n", err)
+		return
+	}
+
+	modFilePaths, err := findKCLModFiles(currentDir)
+	if err != nil {
+		fmt.Printf("Error finding kcl.mod files: %v\n", err)
+		return
+	}
+
+	for _, packageDir := range modFilePaths {
+		if err := processPackage(packageDir); err != nil {
+			fmt.Printf("Error processing package at '%s': %v\n", packageDir, err)
+		}
+	}
+
+	fmt.Println("Checksum successfully included in all KCL packages")
+}

Integrate-Checksum/main_test.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"encoding/json"
+	"runtime"
+	"testing"
+
+	"artifacthub/mock"
+
+	"gotest.tools/assert"
+	"kcl-lang.io/kpm/pkg/client"
+	"kcl-lang.io/kpm/pkg/constants"
+	"kcl-lang.io/kpm/pkg/utils"
+)
+
+// TestMainFunc tests the main functionality of the integrate checksum tool
+func TestMainFunc(t *testing.T) {
+	// Skip the test on Windows due to platform-specific issues.
+	if runtime.GOOS == "windows" {
+		t.Skip("Skipping TestMainFunc on Windows due to platform-specific issues")
+	}
+
+	// Start the local Docker registry required for testing
+	err := mock.StartDockerRegistry()
+	assert.NilError(t, err)
+
+	// Push the test package to the local OCI registry.
+	pkgDir, err := mock.PushTestPkgToRegistry()
+	assert.NilError(t, err)
+
+	// Initialize the KPM client.
+	kpmClient, err := client.NewKpmClient()
+	assert.NilError(t, err, "Failed to initialize KPM client")
+
+	// Locate KCL module files in the current directory.
+	packageDirs, err := findKCLModFiles(pkgDir)
+	assert.NilError(t, err, "Failed to locate KCL module files")
+	assert.Assert(t, len(packageDirs) > 0, "No KCL module files found")
+
+	// Resolve the dependency for the first module found.
+	dependency, err := resolveDependency(kpmClient, packageDirs[0])
+	assert.NilError(t, err, "Failed to resolve dependency")
+
+	// Set custom OCI registry and repository for testing.
+	dependency.Source.Oci.Reg = "localhost:5001"
+	dependency.Source.Oci.Repo = "test"
+
+	// Fetch the original manifest.
+	originalManifest, err := fetchManifest(kpmClient, dependency)
+	assert.NilError(t, err, "Failed to fetch original manifest")
+
+	// Marshal the original manifest into JSON format.
+	originalManifestJSON, err := json.Marshal(originalManifest)
+	assert.NilError(t, err, "Failed to marshal original manifest to JSON")
+
+	// Configure the repository for testing purposes.
+	repository, err := configureRepository(dependency, kpmClient)
+	assert.NilError(t, err, "Failed to configure repository")
+	repository.PlainHTTP = true // Enable plain HTTP for local testing.
+
+	// Modify the manifest annotations for testing.
+	originalManifest.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM] = "changes-for-testing-purpose"
+
+	// Marshal the updated manifest into JSON format.
+	updatedManifestJSON, err := json.Marshal(originalManifest)
+	assert.NilError(t, err, "Failed to marshal updated manifest to JSON")
+
+	// Tag the updated manifest in the repository.
+	err = tagManifest(repository, updatedManifestJSON, dependency)
+	assert.NilError(t, err, "Failed to tag updated manifest in repository")
+
+	// Fetch the new manifest after tagging.
+	newManifest, err := fetchManifest(kpmClient, dependency)
+	assert.NilError(t, err, "Failed to fetch new manifest")
+
+	// Marshal the new manifest into JSON format for comparison.
+	newManifestJSON, err := json.Marshal(newManifest)
+	assert.NilError(t, err, "Failed to marshal new manifest to JSON")
+
+	// Check if the manifest was updated correctly.
+	assert.Assert(t, string(newManifestJSON) != string(originalManifestJSON), "Failed to update the manifest")
+
+	// Revert the `Sum` field to its original value to ensure only that was changed.
+	newManifest.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM] = dependency.Sum
+	newManifestJSON, err = json.Marshal(newManifest)
+	assert.NilError(t, err, "Failed to marshal reverted manifest to JSON")
+
+	// Compare the new manifest data with the expected manifest data.
+	assert.Equal(t, string(newManifestJSON), string(originalManifestJSON), "New manifest data mismatch")
+
+	// Pull the test package.
+	pkgPullPath, err := mock.PullTestPkg()
+	assert.NilError(t, err)
+
+	// Find KCL module files in the pulled package.
+	packagePullDir, err := findKCLModFiles(pkgPullPath)
+	assert.NilError(t, err, "Failed to locate KCL module files")
+
+	// Ensure that at least one KCL module file was found.
+	assert.Assert(t, len(packagePullDir) > 0, "No KCL module files found")
+
+	// Calculate the hash of the pulled KCL module directory to verify its integrity.
+	pulledSum, err := utils.HashDir(packagePullDir[0])
+	assert.NilError(t, err)
+
+	// Compare the hash of the pulled files with the expected dependency sum to check for unintended changes.
+	assert.Equal(t, pulledSum, dependency.Sum, "Unexpected changes detected in the package contents")
+
+	// Clean the environment after all tests have been run
+	err = mock.CleanTestEnv()
+	assert.NilError(t, err)
+}