From 456b934c493f611db2444c4af1815bcb8c87d62d Mon Sep 17 00:00:00 2001 From: Nejc Habjan Date: Sun, 23 Oct 2022 14:38:59 +0200 Subject: [PATCH] feat: add git submodule analyzer --- go.mod | 1 + go.sum | 2 + pkg/detector/library/driver.go | 4 +- pkg/fanal/analyzer/all/import.go | 1 + pkg/fanal/analyzer/const.go | 3 +- pkg/fanal/analyzer/git/submodule/submodule.go | 108 ++++++++++++ .../analyzer/git/submodule/submodule_test.go | 162 ++++++++++++++++++ .../analyzer/git/submodule/testdata/README.md | 11 ++ .../git/submodule/testdata/git-url.gitmodules | 3 + .../submodule/testdata/https-url.gitmodules | 3 + .../testdata/missing-submodule.gitmodules | 1 + .../testdata/relative-url.gitmodules | 3 + .../git/submodule/testdata/ssh-url.gitmodules | 3 + pkg/fanal/types/const.go | 6 + 14 files changed, 308 insertions(+), 3 deletions(-) create mode 100644 pkg/fanal/analyzer/git/submodule/submodule.go create mode 100644 pkg/fanal/analyzer/git/submodule/submodule_test.go create mode 100644 pkg/fanal/analyzer/git/submodule/testdata/README.md create mode 100644 pkg/fanal/analyzer/git/submodule/testdata/git-url.gitmodules create mode 100644 pkg/fanal/analyzer/git/submodule/testdata/https-url.gitmodules create mode 100644 pkg/fanal/analyzer/git/submodule/testdata/missing-submodule.gitmodules create mode 100644 pkg/fanal/analyzer/git/submodule/testdata/relative-url.gitmodules create mode 100644 pkg/fanal/analyzer/git/submodule/testdata/ssh-url.gitmodules diff --git a/go.mod b/go.mod index 77113a84de1d..91a2ccd16e1c 100644 --- a/go.mod +++ b/go.mod @@ -71,6 +71,7 @@ require ( github.com/testcontainers/testcontainers-go v0.15.0 github.com/tetratelabs/wazero v1.0.0-pre.4 github.com/twitchtv/twirp v8.1.2+incompatible + github.com/whilp/git-urls v1.0.0 github.com/xlab/treeprint v1.1.0 go.etcd.io/bbolt v1.3.6 go.uber.org/zap v1.23.0 diff --git a/go.sum b/go.sum index 9fbf629ead32..d01380679574 100644 --- a/go.sum +++ b/go.sum @@ -1532,6 +1532,8 @@ github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1 github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU= +github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= diff --git a/pkg/detector/library/driver.go b/pkg/detector/library/driver.go index 99f6e9513565..ebc21ce7a390 100644 --- a/pkg/detector/library/driver.go +++ b/pkg/detector/library/driver.go @@ -56,8 +56,8 @@ func NewDriver(libType string) (Driver, error) { // Only semver can be used for version ranges // https://docs.conan.io/en/latest/versioning/version_ranges.html comparer = compare.GenericComparer{} - case ftypes.Cocoapods: - log.Logger.Warn("CocoaPods is supported for SBOM, not for vulnerability scanning") + case ftypes.Cocoapods, ftypes.GitSubmodule: + log.Logger.Warnf("%s is supported for SBOM, not for vulnerability scanning", libType) return Driver{}, ErrSBOMSupportOnly default: return Driver{}, xerrors.Errorf("unsupported type %s", libType) diff --git a/pkg/fanal/analyzer/all/import.go b/pkg/fanal/analyzer/all/import.go index f6ea4bc7cd7d..78d030aa379a 100644 --- a/pkg/fanal/analyzer/all/import.go +++ b/pkg/fanal/analyzer/all/import.go @@ -5,6 +5,7 @@ import ( _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/command/apk" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/all" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/executable" + _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/git/submodule" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/c/conan" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/dotnet/deps" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/dotnet/nuget" diff --git a/pkg/fanal/analyzer/const.go b/pkg/fanal/analyzer/const.go index 218131ed8763..c2820440f1aa 100644 --- a/pkg/fanal/analyzer/const.go +++ b/pkg/fanal/analyzer/const.go @@ -80,7 +80,8 @@ const ( // ============ // Non-packaged // ============ - TypeExecutable Type = "executable" + TypeExecutable Type = "executable" + TypeGitSubmodule Type = "git-submodule" // ============ // Image Config diff --git a/pkg/fanal/analyzer/git/submodule/submodule.go b/pkg/fanal/analyzer/git/submodule/submodule.go new file mode 100644 index 000000000000..cb239327c9b3 --- /dev/null +++ b/pkg/fanal/analyzer/git/submodule/submodule.go @@ -0,0 +1,108 @@ +package submodule + +import ( + "context" + "fmt" + "net/url" + "os" + "strings" + + godeptypes "github.com/aquasecurity/go-dep-parser/pkg/types" + "github.com/aquasecurity/trivy/pkg/fanal/analyzer" + "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language" + "github.com/aquasecurity/trivy/pkg/fanal/types" + + "github.com/go-git/go-git/v5" + giturls "github.com/whilp/git-urls" + "golang.org/x/xerrors" +) + +func init() { + analyzer.RegisterAnalyzer(&gitSubmoduleAnalyzer{}) +} + +const version = 1 + +type gitSubmoduleAnalyzer struct{} + +func (a gitSubmoduleAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) { + libs, deps, err := parseGitmodules(input.Dir) + if err != nil { + return nil, xerrors.Errorf("git repo parse error: %w", err) + } + + return language.ToAnalysisResult(types.GitSubmodule, input.FilePath, "", libs, deps), nil +} + +func (a gitSubmoduleAnalyzer) Required(_ string, fileInfo os.FileInfo) bool { + return fileInfo.Name() == types.GitModules +} + +func (a gitSubmoduleAnalyzer) Type() analyzer.Type { + return analyzer.TypeGitSubmodule +} + +func (a gitSubmoduleAnalyzer) Version() int { + return version +} + +func parseGitmodules(inputDir string) ([]godeptypes.Library, []godeptypes.Dependency, error) { + repo, err := git.PlainOpen(inputDir) + if err != nil { + return nil, nil, err + } + + w, err := repo.Worktree() + if err != nil { + return nil, nil, err + } + + submodules, err := w.Submodules() + if err != nil { + return nil, nil, err + } + + libs, _ := parseSubmodules(repo, &submodules) + return libs, nil, nil +} + +func parseSubmodules(repo *git.Repository, submodules *git.Submodules) ([]godeptypes.Library, []godeptypes.Dependency) { + var libs []godeptypes.Library + var name *url.URL + + for _, submodule := range *submodules { + remote := submodule.Config().URL + + if strings.HasPrefix(remote, "../") { + // resolve relative URLs via root remote + rootRemote, err := getRemoteUrl(repo) + if err != nil { + return nil, nil + } + + baseUrl, _ := giturls.Parse(fmt.Sprintf("%s/", rootRemote)) + name, _ = baseUrl.Parse(remote) + } else { + name, _ = giturls.Parse(remote) + } + + status, _ := submodule.Status() + version := status.Expected.String() + + libs = append(libs, godeptypes.Library{ + Name: name.String(), + Version: version, + }) + } + + return libs, nil +} + +func getRemoteUrl(repo *git.Repository) (string, error) { + remote, err := repo.Remote("origin") + if err != nil { + return "", err + } + + return remote.Config().URLs[0], nil +} diff --git a/pkg/fanal/analyzer/git/submodule/submodule_test.go b/pkg/fanal/analyzer/git/submodule/submodule_test.go new file mode 100644 index 000000000000..70cf6bd36d49 --- /dev/null +++ b/pkg/fanal/analyzer/git/submodule/submodule_test.go @@ -0,0 +1,162 @@ +package submodule + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/aquasecurity/trivy/pkg/fanal/analyzer" + "github.com/aquasecurity/trivy/pkg/fanal/types" + "github.com/aquasecurity/trivy/pkg/utils" +) + +func Test_gitSubmoduleAnalyzer_Analyze(t *testing.T) { + tests := []struct { + name string + filePath string + want *analyzer.AnalysisResult + }{ + { + name: "https-url", + filePath: "testdata/https-url.gitmodules", + want: &analyzer.AnalysisResult{ + Applications: []types.Application{ + { + Type: types.GitSubmodule, + FilePath: types.GitModules, + Libraries: []types.Package{ + { + Name: "https://github.com/org/repository.git", + Version: "ca82a6dff817ec66f44342007202690a93763949", + }, + }, + }, + }, + }, + }, + { + name: "git-url", + filePath: "testdata/git-url.gitmodules", + want: &analyzer.AnalysisResult{ + Applications: []types.Application{ + { + Type: types.GitSubmodule, + FilePath: types.GitModules, + Libraries: []types.Package{ + { + Name: "ssh://git@github.com/org/repository.git", + Version: "ca82a6dff817ec66f44342007202690a93763949", + }, + }, + }, + }, + }, + }, + { + name: "ssh-url", + filePath: "testdata/ssh-url.gitmodules", + want: &analyzer.AnalysisResult{ + Applications: []types.Application{ + { + Type: types.GitSubmodule, + FilePath: types.GitModules, + Libraries: []types.Package{ + { + Name: "ssh://git@github.com/org/repository.git", + Version: "ca82a6dff817ec66f44342007202690a93763949", + }, + }, + }, + }, + }, + }, + { + name: "relative-url", + filePath: "testdata/relative-url.gitmodules", + want: &analyzer.AnalysisResult{ + Applications: []types.Application{ + { + Type: types.GitSubmodule, + FilePath: types.GitModules, + Libraries: []types.Package{ + { + Name: "https://github.com/org/repository.git", + Version: "ca82a6dff817ec66f44342007202690a93763949", + }, + }, + }, + }, + }, + }, + { + name: "missing-submodule", + filePath: "testdata/missing-submodule.gitmodules", + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + currentDir, err := os.Getwd() + require.NoError(t, err) + + dir := t.TempDir() + destFilePath := filepath.Join(dir, types.GitModules) + + _, err = utils.CopyFile(tt.filePath, destFilePath) + require.NoError(t, err) + + err = initRepoWithSubmodules(dir) + require.NoError(t, err) + + err = os.Chdir(dir) + require.NoError(t, err) + defer os.Chdir(currentDir) + + a := gitSubmoduleAnalyzer{} + got, err := a.Analyze(context.Background(), analyzer.AnalysisInput{ + Dir: dir, + FilePath: types.GitModules, + }) + assert.Equal(t, tt.want, got) + }) + } +} + +func initRepoWithSubmodules(dir string) error { + repo, err := git.PlainInit(dir, false) + if err != nil { + return err + } + + _, err = repo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{"https://github.com/org/repository.git"}, + }) + if err != nil { + return err + } + + updateIndexCmd := exec.Command( + "git", + "update-index", + "--add", + "--cacheinfo", + "160000", + "ca82a6dff817ec66f44342007202690a93763949", + "submodule", + ) + updateIndexCmd.Dir = dir + updateIndexCmd.Run() + if err != nil { + return err + } + + return nil +} diff --git a/pkg/fanal/analyzer/git/submodule/testdata/README.md b/pkg/fanal/analyzer/git/submodule/testdata/README.md new file mode 100644 index 000000000000..a68d61a7960d --- /dev/null +++ b/pkg/fanal/analyzer/git/submodule/testdata/README.md @@ -0,0 +1,11 @@ +# Git submodule testdata + +The examples in this testdata directory test a few common Git URL formats. For a full +list of supported Git URL formats, see: +https://stackoverflow.com/questions/31801271/what-are-the-supported-git-url-formats + +For the git plumbing commands involved in the faked remote submodule test setup, see +https://stackoverflow.com/questions/34562333/is-there-a-way-to-git-submodule-add-a-repo-without-cloning-it. + +Files here are not valid Git submodule configuration filenames. They are copied in each test case +to `t.TempDir` as `.gitmodules`. diff --git a/pkg/fanal/analyzer/git/submodule/testdata/git-url.gitmodules b/pkg/fanal/analyzer/git/submodule/testdata/git-url.gitmodules new file mode 100644 index 000000000000..fa8b67a038d7 --- /dev/null +++ b/pkg/fanal/analyzer/git/submodule/testdata/git-url.gitmodules @@ -0,0 +1,3 @@ +[submodule "submodule"] + path = submodule + url = git@github.com:org/repository.git diff --git a/pkg/fanal/analyzer/git/submodule/testdata/https-url.gitmodules b/pkg/fanal/analyzer/git/submodule/testdata/https-url.gitmodules new file mode 100644 index 000000000000..114a1b1835c3 --- /dev/null +++ b/pkg/fanal/analyzer/git/submodule/testdata/https-url.gitmodules @@ -0,0 +1,3 @@ +[submodule "submodule"] + path = submodule + url = https://github.com/org/repository.git diff --git a/pkg/fanal/analyzer/git/submodule/testdata/missing-submodule.gitmodules b/pkg/fanal/analyzer/git/submodule/testdata/missing-submodule.gitmodules new file mode 100644 index 000000000000..5902e1d0bff3 --- /dev/null +++ b/pkg/fanal/analyzer/git/submodule/testdata/missing-submodule.gitmodules @@ -0,0 +1 @@ +# This repository includes a valid entry in .git/index but no .gitmodules entry diff --git a/pkg/fanal/analyzer/git/submodule/testdata/relative-url.gitmodules b/pkg/fanal/analyzer/git/submodule/testdata/relative-url.gitmodules new file mode 100644 index 000000000000..d4e766f690c9 --- /dev/null +++ b/pkg/fanal/analyzer/git/submodule/testdata/relative-url.gitmodules @@ -0,0 +1,3 @@ +[submodule "submodule"] + path = submodule + url = ../repository.git diff --git a/pkg/fanal/analyzer/git/submodule/testdata/ssh-url.gitmodules b/pkg/fanal/analyzer/git/submodule/testdata/ssh-url.gitmodules new file mode 100644 index 000000000000..461af274c909 --- /dev/null +++ b/pkg/fanal/analyzer/git/submodule/testdata/ssh-url.gitmodules @@ -0,0 +1,3 @@ +[submodule "submodule"] + path = submodule + url = ssh://git@github.com/org/repository.git diff --git a/pkg/fanal/types/const.go b/pkg/fanal/types/const.go index 2ebac0647bdb..dffce9ee241b 100644 --- a/pkg/fanal/types/const.go +++ b/pkg/fanal/types/const.go @@ -31,6 +31,9 @@ const ( Conan = "conan" Cocoapods = "cocoapods" + // Non-packaged dependencies + GitSubmodule = "git-submodule" + // Config files YAML = "yaml" JSON = "json" @@ -73,4 +76,7 @@ const ( ConanLock = "conan.lock" CocoaPodsLock = "Podfile.lock" + + // Non-packaged file names + GitModules = ".gitmodules" )