Merge pull request #129 from debricked/fingerprinting

File fingerprinting and save to file

.github/workflows/test.yml

@@ -14,6 +14,11 @@ jobs:
         os: [ 'ubuntu-latest', 'windows-latest', 'macos-latest' ]
     runs-on: ${{ matrix.os }}
     steps:
+      - name: Set git to use LF
+        run: |
+          git config --global core.autocrlf input
+          git config --global core.eol lf
+
       - uses: actions/checkout@v3
 
       - name: Set up Go

.gitignore

@@ -15,3 +15,4 @@ internal/resolution/pm/gradle/.gradle-init-script.debricked.groovy
 test/resolve/testdata/npm/yarn.lock
 test/resolve/testdata/nuget/packages.lock.json
 test/resolve/testdata/nuget/obj
+.debricked.fingerprints.wfp

internal/cmd/fingerprint/fingerprint.go

@@ -0,0 +1,73 @@
+package fingerprint
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/debricked/cli/internal/file"
+	"github.com/debricked/cli/internal/fingerprint"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var exclusions = file.DefaultExclusionsFingerprint()
+
+const (
+	ExclusionFlag = "exclusion-fingerprint"
+)
+
+func NewFingerprintCmd(fingerprinter fingerprint.IFingerprint) *cobra.Command {
+
+	short := "Fingerprints files to match against the Debricked knowledge base. [beta feature]"
+	long := fmt.Sprintf("Fingerprint files for identification in a given path and writes it to %s. [beta feature]\nThis hashes all files and matches them against the Debricked knowledge base.", fingerprint.OutputFileNameFingerprints)
+	cmd := &cobra.Command{
+		Use:    "fingerprint [path]",
+		Short:  short,
+		Hidden: true,
+		Long:   long,
+		PreRun: func(cmd *cobra.Command, _ []string) {
+			_ = viper.BindPFlags(cmd.Flags())
+		},
+		RunE: RunE(fingerprinter),
+	}
+	fileExclusionExample := filepath.Join("*", "**.pyc")
+	dirExclusionExample := filepath.Join("**", "node_modules", "**")
+	exampleFlags := fmt.Sprintf("-%s \"%s\" -%s \"%s\"", ExclusionFlag, fileExclusionExample, ExclusionFlag, dirExclusionExample)
+	cmd.Flags().StringArrayVarP(&exclusions, ExclusionFlag, "", exclusions, `The following terms are supported to exclude paths:
+Special Terms | Meaning
+------------- | -------
+"*"           | matches any sequence of non-Separator characters 
+"/**/"        | matches zero or multiple directories
+"?"           | matches any single non-Separator character
+"[class]"     | matches any single non-Separator character against a class of characters ([see "character classes"])
+"{alt1,...}"  | matches a sequence of characters if one of the comma-separated alternatives matches
+
+Example: 
+$ debricked files fingerprint . `+exampleFlags)
+
+	viper.MustBindEnv(ExclusionFlag)
+
+	return cmd
+}
+
+func RunE(f fingerprint.IFingerprint) func(_ *cobra.Command, args []string) error {
+	return func(_ *cobra.Command, args []string) error {
+		path := ""
+		if len(args) > 0 {
+			path = args[0]
+		}
+
+		output, err := f.FingerprintFiles(path, exclusions)
+
+		if err != nil {
+			return err
+		}
+
+		err = output.ToFile(fingerprint.OutputFileNameFingerprints)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}

internal/cmd/fingerprint/fingerprint_test.go

@@ -0,0 +1,56 @@
+package fingerprint
+
+import (
+	"os"
+	"testing"
+
+	"github.com/debricked/cli/internal/fingerprint"
+	"github.com/debricked/cli/internal/fingerprint/testdata"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewFingerprintCmd(t *testing.T) {
+	var f fingerprint.IFingerprint
+	cmd := NewFingerprintCmd(f)
+
+	commands := cmd.Commands()
+	nbrOfCommands := 0
+	assert.Len(t, commands, nbrOfCommands)
+
+	flags := cmd.Flags()
+	flagAssertions := map[string]string{}
+	for name, shorthand := range flagAssertions {
+		flag := flags.Lookup(name)
+		assert.NotNil(t, flag)
+		assert.Equal(t, shorthand, flag.Shorthand)
+	}
+
+	var flagKeys = []string{
+		ExclusionFlag,
+	}
+	viperKeys := viper.AllKeys()
+	for _, flagKey := range flagKeys {
+		match := false
+		for _, key := range viperKeys {
+			if key == flagKey {
+				match = true
+			}
+		}
+		assert.Truef(t, match, "failed to assert that flag was present: "+flagKey)
+	}
+
+}
+
+func TestRunE(t *testing.T) {
+	defer func() {
+		os.Remove(fingerprint.OutputFileNameFingerprints)
+	}()
+	fingerprintMock := testdata.NewFingerprintMock()
+	runE := RunE(fingerprintMock)
+
+	err := runE(nil, []string{"."})
+
+	assert.NoError(t, err)
+
+}

internal/cmd/root/root.go

@@ -2,6 +2,7 @@ package root
 
 import (
 	"github.com/debricked/cli/internal/cmd/files"
+	"github.com/debricked/cli/internal/cmd/fingerprint"
 	"github.com/debricked/cli/internal/cmd/report"
 	"github.com/debricked/cli/internal/cmd/resolve"
 	"github.com/debricked/cli/internal/cmd/scan"
@@ -42,6 +43,7 @@ Read more: https://portal.debricked.com/administration-47/how-do-i-generate-an-a
 	rootCmd.AddCommand(report.NewReportCmd(container.LicenseReporter(), container.VulnerabilityReporter()))
 	rootCmd.AddCommand(files.NewFilesCmd(container.Finder()))
 	rootCmd.AddCommand(scan.NewScanCmd(container.Scanner()))
+	rootCmd.AddCommand(fingerprint.NewFingerprintCmd(container.Fingerprinter()))
 	rootCmd.AddCommand(resolve.NewResolveCmd(container.Resolver()))
 
 	rootCmd.CompletionOptions.DisableDefaultCmd = true

internal/cmd/root/root_test.go

@@ -11,7 +11,7 @@ import (
 func TestNewRootCmd(t *testing.T) {
 	cmd := NewRootCmd("v0.0.0", wire.GetCliContainer())
 	commands := cmd.Commands()
-	nbrOfCommands := 4
+	nbrOfCommands := 5
 	if len(commands) != nbrOfCommands {
 		t.Errorf("failed to assert that there were %d sub commands connected", nbrOfCommands)
 	}
@@ -31,7 +31,7 @@ func TestNewRootCmd(t *testing.T) {
 		}
 	}
 	assert.Truef(t, match, "failed to assert that flag was present: "+AccessTokenFlag)
-	assert.Len(t, viperKeys, 13)
+	assert.Len(t, viperKeys, 14)
 }
 
 func TestPreRun(t *testing.T) {

internal/cmd/scan/scan.go

@@ -20,6 +20,7 @@ var repositoryUrl string
 var integrationName string
 var exclusions = file.DefaultExclusions()
 var noResolve bool
+var noFingerprint bool
 var passOnDowntime bool
 
 const (
@@ -31,6 +32,7 @@ const (
 	IntegrationFlag   = "integration"
 	ExclusionFlag     = "exclusion"
 	NoResolveFlag     = "no-resolve"
+	FingerprintFlag   = "fingerprint"
 	PassOnTimeOut     = "pass-on-timeout"
 )
 
@@ -82,6 +84,8 @@ $ debricked scan . `+exampleFlags)
 	cmd.Flags().BoolVarP(&passOnDowntime, PassOnTimeOut, "p", false, "pass scan if there is a service access timeout")
 	cmd.Flags().BoolVar(&noResolve, NoResolveFlag, false, `disables resolution of manifest files that lack lock files. Resolving manifest files enables more accurate dependency scanning since the whole dependency tree will be analysed.
 For example, if there is a "go.mod" in the target path, its dependencies are going to get resolved onto a lock file, and latter scanned.`)
+	cmd.Flags().BoolVar(&noFingerprint, FingerprintFlag, false, "enables fingerprinting for undeclared component identification. Can be run as a standalone command [files fingerprint] with more granular options. [beta feature]")
+	cmd.Flags().MarkHidden(FingerprintFlag) //nolint:errcheck
 
 	viper.MustBindEnv(RepositoryFlag)
 	viper.MustBindEnv(CommitFlag)
@@ -103,6 +107,7 @@ func RunE(s *scan.IScanner) func(_ *cobra.Command, args []string) error {
 		options := scan.DebrickedOptions{
 			Path:            path,
 			Resolve:         !viper.GetBool(NoResolveFlag),
+			Fingerprint:     viper.GetBool(FingerprintFlag),
 			Exclusions:      viper.GetStringSlice(ExclusionFlag),
 			RepositoryName:  viper.GetString(RepositoryFlag),
 			CommitName:      viper.GetString(CommitFlag),

internal/file/default_exclusion.go

@@ -10,3 +10,22 @@ func DefaultExclusions() []string {
 		filepath.Join("**", "obj", "**"), // nuget
 	}
 }
+
+var EXCLUDED_DIRS_FINGERPRINT = []string{
+	"nbproject", "nbbuild", "nbdist", "node_modules",
+	"__pycache__", "_yardoc", "eggs",
+	"wheels", "htmlcov", "__pypackages__"}
+
+var EXCLUDED_DIRS_FINGERPRINT_RAW = []string{"**/*.egg-info/**", "**/*venv/**"}
+
+func DefaultExclusionsFingerprint() []string {
+	output := []string{}
+
+	for _, pattern := range EXCLUDED_DIRS_FINGERPRINT {
+		output = append(output, filepath.Join("**", pattern, "**"))
+	}
+
+	output = append(output, EXCLUDED_DIRS_FINGERPRINT_RAW...)
+
+	return output
+}

internal/file/default_exclusion_test.go

@@ -2,6 +2,7 @@ package file
 
 import (
 	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -15,3 +16,23 @@ func TestDefaultExclusions(t *testing.T) {
 		assert.Greaterf(t, len(exParts), 0, "failed to assert that %s used correct separator. Proper separator %s", ex, separator)
 	}
 }
+func TestDefaultExclusionsFingerprint(t *testing.T) {
+	expectedExclusions := []string{
+		filepath.Join("**", "nbproject", "**"),
+		filepath.Join("**", "nbbuild", "**"),
+		filepath.Join("**", "nbdist", "**"),
+		filepath.Join("**", "node_modules", "**"),
+		filepath.Join("**", "__pycache__", "**"),
+		filepath.Join("**", "_yardoc", "**"),
+		filepath.Join("**", "eggs", "**"),
+		filepath.Join("**", "wheels", "**"),
+		filepath.Join("**", "htmlcov", "**"),
+		filepath.Join("**", "__pypackages__", "**"),
+		"**/*.egg-info/**",
+		"**/*venv/**",
+	}
+
+	exclusions := DefaultExclusionsFingerprint()
+
+	assert.ElementsMatch(t, expectedExclusions, exclusions, "DefaultExclusionsFingerprint did not return the expected exclusions")
+}

internal/file/finder.go

@@ -49,7 +49,7 @@ func (finder *Finder) GetGroups(rootPath string, exclusions []string, lockfileOn
 			if err != nil {
 				return err
 			}
-			if !fileInfo.IsDir() && !excluded(exclusions, path) {
+			if !fileInfo.IsDir() && !Excluded(exclusions, path) {
 				for _, format := range formats {
 					if groups.Match(format, path, lockfileOnly) {
 
@@ -97,7 +97,7 @@ func (finder *Finder) GetSupportedFormats() ([]*CompiledFormat, error) {
 	return compiledDependencyFileFormats, nil
 }
 
-func excluded(exclusions []string, path string) bool {
+func Excluded(exclusions []string, path string) bool {
 	for _, exclusion := range exclusions {
 		ex := filepath.Clean(exclusion)
 		matched, _ := doublestar.PathMatch(ex, path)

internal/file/finder_test.go

@@ -226,7 +226,7 @@ func TestExclude(t *testing.T) {
 		t.Run(c.name, func(t *testing.T) {
 			var excludedFiles []string
 			for _, file := range files {
-				if excluded(c.exclusions, file) {
+				if Excluded(c.exclusions, file) {
 					excludedFiles = append(excludedFiles, file)
 				}
 			}