Vulnerable functionality for java (maven focused) (#81)

* Add vf-cli for java (mvn) to cli

* Refactor find-files and finder logic

* Add callgraph cmd test

* Refactor MapFilesToDir

* Improve test coverage

* Rewrite tests to be OS-agnostic

* Add build flag to call graph command, without build functionality.

* Add viper binding for build flag and fix test not asserting flags as intended.

* Change build arg to no-build arg to indicate that auto build is default.

* Add build arg to callgraph config and strategy.

* Add build maven package cmd and call it if auto build is enabled (default true).

* Add functionality to run maven package compilation (build) step by default with a flag to disable it.

* Add mini java project for testing and test build on it.

* add functional test to make sure build works correctly. this test is slow and should possibly be separated in the future.

* Fix linting issues, including a cyclic complexity reduction (at a cost).

* Ignore the build target location in case it was not properly deleted in a failed test run.

* Add spinner UI in CLI for the build step of Maven projects in call graph. Add rich error logging.

* Improve testing of error logging.

* Add a Command struct that wraps os/exec.Cmd to allow mocking for test purposes.

* Implement command mock. Refactor command methods to allow better testing.

* Add upload + postprocess logic to java callgraph and improve tests

* Improve testing in io, command and callgraph

* Cleanup

* Move mvn test to correct folder

* Remove gradle callgraph logic from groovy file

* Fix windows problem

* Fix an issue where the zipping tool was looking in root instead of the real location of the call graph.

* Fix mapping pom files to class directories

* Increase time before timeout to 60min

* Change build to only fail if all build pom.xml fails

* Increase upload time for callgraphs to 10min

* Add fix for zipped filename being a long location

* Rewrite java-callgraph-files logic to handle each .class-dir + .jar-files

* Update sootWrapper to support .jar-files

* Change to refresh files of interest after build

* Add functionality to stop run preemptively if error occours

* Add --generate-timeout flag to callgraph command

* Add callgraph generation timeout and upload timeout flags to scan command

* Add e2e test of callgraph generation

* Move exclude logic to its own module

* Move callgraph finder from io to callgraph package

* Utilise Excluded in callgraph finder

* Add test coverage for callgraph finder file exclusion

* Update callgraph documentation

* Fix failing tests

* Change deps of pom.xml file

* Update SootWrapper

* Change name of .debricked-call-graph to debricked-call-graph (#138)

* Change to scan for groups after callgraph generated

* Update documentation link

* Add windows-latest to e2e tests again

* Fix general cleanup

---------

Co-authored-by: AntonDebricked <[email protected]>
Co-authored-by: filip-debricked <[email protected]>
Co-authored-by: jonnadebricked <[email protected]>
Co-authored-by: klaradebricked <[email protected]>

.github/workflows/test.yml

@@ -54,6 +54,48 @@ jobs:
       - name: E2E - scan
         run: go run cmd/debricked/main.go scan internal/file/testdata/misc  -e requirements.txt -t ${{ secrets.DEBRICKED_TOKEN }} -r debricked/cli-test -c E2E-test-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
 
+  vulnfunc-e2e:
+    name: Vulnfunc E2E
+
+    strategy:
+      matrix: 
+        os: [ 'ubuntu-latest', 'macos-latest', 'windows-latest' ]
+        java: [11, 17, 20]
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.20'
+
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/Library/Caches/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ matrix.os }}-go
+
+      - name: Build
+        run: go build -v ./...
+
+      - name: Set up Java ${{matrix.java}}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{matrix.java}}
+          distribution: 'temurin'
+
+      - name: Install Debricked CLI
+        run: go install -ldflags "-X main.version=${GITHUB_REF#refs/heads/}" ./cmd/debricked
+
+      - name: Callgraph E2E
+        run: ./scripts/test_e2e_callgraph_java_version.sh ${{matrix.java}}
+
   golangci:
     name: Lint
     runs-on: ubuntu-latest

.gitignore

@@ -12,9 +12,13 @@ test/resolve/testdata/pip/requirements.txt.venv/
 test/resolve/testdata/pip/requirements.txt.pip.debricked.lock
 internal/cmd/scan/testdata/npm/yarn.lock
 internal/resolution/pm/gradle/.gradle-init-script.debricked.groovy
+internal/callgraph/language/java11/testdata/mvnproj/target
 test/resolve/testdata/npm/yarn.lock
+test/resolve/testdata/nuget/packages.lock.json
 test/resolve/testdata/nuget/csproj/packages.lock.json
 test/resolve/testdata/nuget/packagesconfig/packages.config.nuget.debricked.lock
+test/resolve/testdata/nuget/obj
 test/resolve/testdata/nuget/**/obj
 debricked.fingerprints.wfp
 test/resolve/testdata/gomod/gomod.debricked.lock
+/mvnproj/target

Makefile

@@ -12,7 +12,7 @@ test:
 
 .PHONY: test-e2e
 test-e2e:
-	bash scripts/test_e2e.sh
+	bash scripts/test_e2e.sh $(type)
 
 .PHONY: test-e2e-docker
 docker-build-dev:

internal/callgraph/cgexec/command.go

@@ -0,0 +1,150 @@
+package cgexec
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+type ICommand interface {
+	CombinedOutput() ([]byte, error)
+	Start() error
+	Wait() error
+	GetProcess() *os.Process
+	SetStderr(*bytes.Buffer)
+	SetStdout(*bytes.Buffer)
+	GetArgs() []string
+	GetDir() string
+	Signal(process *os.Process, signal os.Signal) error
+}
+
+type Command struct {
+	osCmd *exec.Cmd
+}
+
+func NewCommand(osCmd *exec.Cmd) *Command {
+	return &Command{osCmd}
+}
+
+func (cmd Command) SetStderr(stderr *bytes.Buffer) {
+	cmd.osCmd.Stderr = stderr
+}
+
+func (cmd Command) SetStdout(stdout *bytes.Buffer) {
+	cmd.osCmd.Stdout = stdout
+}
+
+func (cmd Command) GetArgs() []string {
+	return cmd.osCmd.Args
+}
+
+func (cmd Command) CombinedOutput() ([]byte, error) {
+	return cmd.osCmd.CombinedOutput()
+}
+
+func (cmd Command) Start() error {
+	return cmd.osCmd.Start()
+}
+
+func (cmd Command) Wait() error {
+	return cmd.osCmd.Wait()
+}
+
+func (cmd Command) GetProcess() *os.Process {
+	return cmd.osCmd.Process
+}
+
+func (cmd Command) GetDir() string {
+	return cmd.osCmd.Dir
+}
+
+func (cmd Command) Signal(process *os.Process, signal os.Signal) error {
+	return process.Signal(signal)
+}
+
+func RunCommand(cmd ICommand, ctx IContext) error {
+	args := strings.Join(cmd.GetArgs(), " ")
+	var stdoutBuf, stderrBuf bytes.Buffer
+	var err error
+	var outputCmd []byte
+	if ctx == nil {
+		outputCmd, err = cmd.CombinedOutput()
+		if _, ok := err.(*exec.ExitError); ok {
+			err = fmt.Errorf("Command '%s' executed in folder '%s' gave the following error:\n%s", args, cmd.GetDir(), outputCmd)
+		}
+
+		return err
+	}
+
+	cmd.SetStderr(&stderrBuf)
+	cmd.SetStdout(&stdoutBuf)
+
+	// Start the external process
+	if err := cmd.Start(); err != nil {
+
+		return err
+	}
+
+	// Channel to signal when the external process completes
+	done := make(chan error, 1)
+
+	// Goroutine to wait for the process to complete
+	go func() {
+		err := cmd.Wait()
+		if err != nil {
+			err = fmt.Errorf("Command '%s' executed in folder '%s' gave the following error: \n%s\n%s", args, cmd.GetDir(), stdoutBuf.String(), stderrBuf.String())
+		}
+
+		done <- err
+	}()
+
+	select {
+	case <-ctx.Context().Done():
+
+		if ctx.Context().Err() == context.DeadlineExceeded {
+			// Context timeout occurred before the process started
+			return fmt.Errorf("Timeout error: Set timeout duration for Callgraph jobs reached")
+		}
+
+		// The context was canceled, handle cancellation if needed
+		// Send a signal to the process to terminate
+		process := cmd.GetProcess()
+		if process != nil {
+			err := cmd.Signal(process, os.Interrupt)
+			if err != nil {
+				return err
+			}
+		}
+
+		// Wait for the process to exit
+		<-done
+
+		return fmt.Errorf("Timeout error: Set timeout duration for Callgraph jobs reached")
+	case err := <-done:
+		// The external process completed before the context was canceled
+		return err
+	}
+}
+
+func MakeCommand(workingDir string, path string, args []string, ctx IContext) *exec.Cmd {
+	var cmd *exec.Cmd
+
+	if ctx == nil {
+		cmd = &exec.Cmd{
+			Path: path,
+			Args: args,
+			Dir:  workingDir,
+		}
+	} else {
+		command := args[0]
+		arguments := args[1:]
+		cmd = exec.CommandContext(ctx.Context(), command, arguments...)
+		cmd.Path = path
+		cmd.Dir = workingDir
+	}
+
+	return cmd
+}

internal/callgraph/cgexec/command_test.go

@@ -0,0 +1,118 @@
+package cgexec
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	execTestdata "github.com/debricked/cli/internal/callgraph/cgexec/testdata"
+)
+
+func TestMakeCommandWithContext(t *testing.T) {
+	cmdName := "echo"
+	path, _ := exec.LookPath(cmdName)
+	ctx, cancel := NewContext(100)
+	defer cancel()
+	osCmd := MakeCommand(".", path, []string{
+		"echo",
+		"package",
+	}, ctx)
+	cmd := NewCommand(osCmd)
+	err := RunCommand(*cmd, ctx)
+	assert.Nil(t, err)
+}
+
+func TestMakeCommandWithNoContext(t *testing.T) {
+	cmdName := "echo"
+	path, _ := exec.LookPath(cmdName)
+	osCmd := MakeCommand(".", path, []string{
+		"echo",
+		"package",
+		"-q",
+		"-DskipTests",
+		"-e",
+	}, nil)
+	cmd := NewCommand(osCmd)
+	err := RunCommand(*cmd, nil)
+	assert.Nil(t, err)
+}
+
+func TestMakeCommandDeadlineExceeded(t *testing.T) {
+	ctx, _ := execTestdata.NewContextMockDeadlineReached()
+	cmd := execTestdata.NewCommandMock()
+	err := RunCommand(cmd, ctx)
+	t.Log(err)
+	if err == nil {
+		assert.FailNow(t, "Error was unexpectedly nil.")
+	}
+	assert.Contains(t, err.Error(), "Timeout error: Set timeout duration for Callgraph jobs reached")
+}
+
+func TestMakeCommandCancelled(t *testing.T) {
+	ctx, _ := execTestdata.NewContextMockCancelled()
+	cmd := execTestdata.NewCommandMock()
+	err := RunCommand(cmd, ctx)
+	t.Log(err)
+	assert.Contains(t, err.Error(), "Timeout error: Set timeout duration for Callgraph jobs reached")
+}
+
+func TestMakeCommandCancelledInteruptedError(t *testing.T) {
+	ctx, _ := execTestdata.NewContextMockCancelled()
+	cmd := execTestdata.NewCommandMock()
+	cmd.Process = &os.Process{}
+	cmd.SignalError = fmt.Errorf("error")
+	err := RunCommand(cmd, ctx)
+	t.Log(err)
+	assert.Contains(t, err.Error(), "error")
+}
+
+func TestMakeCommandStartFailure(t *testing.T) {
+	ctx, _ := execTestdata.NewContextMock()
+	cmdConfig := execTestdata.NewCmdConfig()
+	cmd := execTestdata.NewCommandMockWithConfig(*cmdConfig)
+	err := RunCommand(cmd, ctx)
+	t.Log(err)
+	assert.Contains(t, err.Error(), "test error")
+}
+
+func TestWaitExecutionFail(t *testing.T) {
+	ctx, _ := execTestdata.NewContextMock()
+	cmd := execTestdata.NewCommandMock()
+	cmd.WaitError = fmt.Errorf("error")
+	err := RunCommand(cmd, ctx)
+	t.Log(err)
+	assert.Contains(t, err.Error(), "error")
+}
+
+func TestNoContextCommandFail(t *testing.T) {
+	cmd := execTestdata.NewCommandMock()
+	cmd.CombinedOutputError = &exec.ExitError{}
+	err := RunCommand(cmd, nil)
+	t.Log(err)
+	assert.Contains(t, err.Error(), "executed in folder")
+}
+
+func TestMakeCommandSuccess(t *testing.T) {
+	ctx, _ := execTestdata.NewContextMock()
+	cmd := execTestdata.NewCommandMock()
+	err := RunCommand(cmd, ctx)
+	t.Log(err)
+	assert.Nil(t, err)
+}
+
+func TestCmdGetProcess(t *testing.T) {
+	cmd := NewCommand(exec.Command("echo", "GetProcess"))
+	process := cmd.GetProcess()
+
+	assert.Nil(t, process)
+}
+
+func TestCmdGetDir(t *testing.T) {
+	cmd := NewCommand(exec.Command("echo", "GetProcess"))
+	dir := cmd.GetDir()
+
+	assert.NotNil(t, dir)
+}

internal/callgraph/cgexec/context.go

@@ -0,0 +1,29 @@
+package cgexec
+
+import (
+	"context"
+	"time"
+)
+
+type IContext interface {
+	Context() context.Context
+	Done() <-chan struct{}
+}
+
+type Context struct {
+	ctx context.Context
+}
+
+func NewContext(timer int) (Context, context.CancelFunc) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timer)*time.Second)
+
+	return Context{ctx}, cancel
+}
+
+func (c Context) Context() context.Context {
+	return c.ctx
+}
+
+func (c Context) Done() <-chan struct{} {
+	return c.ctx.Done()
+}

internal/callgraph/cgexec/context_test.go

@@ -0,0 +1,15 @@
+package cgexec
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDone(t *testing.T) {
+
+	ctx, cancel := NewContext(0)
+	defer cancel()
+	val := <-ctx.Done()
+	assert.NotNil(t, val)
+}